1 /*- 2 * Copyright (c) 2010-2012 Citrix Inc. 3 * Copyright (c) 2009-2012,2016-2017 Microsoft Corp. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /*- 30 * Copyright (c) 2004-2006 Kip Macy 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 45 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 52 * SUCH DAMAGE. 53 */ 54 55 #include <sys/cdefs.h> 56 #include "opt_hn.h" 57 #include "opt_inet6.h" 58 #include "opt_inet.h" 59 #include "opt_rss.h" 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/bus.h> 64 #include <sys/counter.h> 65 #include <sys/kernel.h> 66 #include <sys/limits.h> 67 #include <sys/malloc.h> 68 #include <sys/mbuf.h> 69 #include <sys/module.h> 70 #include <sys/queue.h> 71 #include <sys/lock.h> 72 #include <sys/proc.h> 73 #include <sys/rmlock.h> 74 #include <sys/sbuf.h> 75 #include <sys/sched.h> 76 #include <sys/smp.h> 77 #include <sys/socket.h> 78 #include <sys/sockio.h> 79 #include <sys/sx.h> 80 #include <sys/sysctl.h> 81 #include <sys/taskqueue.h> 82 #include <sys/buf_ring.h> 83 #include <sys/eventhandler.h> 84 #include <sys/epoch.h> 85 86 #include <vm/vm.h> 87 #include <vm/vm_extern.h> 88 #include <vm/pmap.h> 89 90 #include <machine/atomic.h> 91 #include <machine/in_cksum.h> 92 93 #include <net/bpf.h> 94 #include <net/ethernet.h> 95 #include <net/if.h> 96 #include <net/if_dl.h> 97 #include <net/if_media.h> 98 #include <net/if_types.h> 99 #include <net/if_var.h> 100 #include <net/if_vlan_var.h> 101 #include <net/rndis.h> 102 #include <net/rss_config.h> 103 104 #include <netinet/in_systm.h> 105 #include <netinet/in.h> 106 #include <netinet/ip.h> 107 #include <netinet/ip6.h> 108 #include <netinet/tcp.h> 109 #include <netinet/tcp_lro.h> 110 #include <netinet/udp.h> 111 112 #include <dev/hyperv/include/hyperv.h> 113 #include <dev/hyperv/include/hyperv_busdma.h> 114 #include <dev/hyperv/include/vmbus.h> 115 #include <dev/hyperv/include/vmbus_xact.h> 116 117 #include <dev/hyperv/netvsc/ndis.h> 118 #include <dev/hyperv/netvsc/if_hnreg.h> 119 #include <dev/hyperv/netvsc/if_hnvar.h> 120 #include <dev/hyperv/netvsc/hn_nvs.h> 121 #include <dev/hyperv/netvsc/hn_rndis.h> 122 123 #include "vmbus_if.h" 124 125 #define HN_IFSTART_SUPPORT 126 127 #define HN_RING_CNT_DEF_MAX 8 128 129 #define HN_VFMAP_SIZE_DEF 8 130 131 #define HN_XPNT_VF_ATTWAIT_MIN 2 /* seconds */ 132 133 /* YYY should get it from the underlying channel */ 134 #define HN_TX_DESC_CNT 512 135 136 #define HN_RNDIS_PKT_LEN \ 137 (sizeof(struct rndis_packet_msg) + \ 138 HN_RNDIS_PKTINFO_SIZE(HN_NDIS_HASH_VALUE_SIZE) + \ 139 HN_RNDIS_PKTINFO_SIZE(NDIS_VLAN_INFO_SIZE) + \ 140 HN_RNDIS_PKTINFO_SIZE(NDIS_LSO2_INFO_SIZE) + \ 141 HN_RNDIS_PKTINFO_SIZE(NDIS_TXCSUM_INFO_SIZE)) 142 #define HN_RNDIS_PKT_BOUNDARY PAGE_SIZE 143 #define HN_RNDIS_PKT_ALIGN CACHE_LINE_SIZE 144 145 #define HN_TX_DATA_BOUNDARY PAGE_SIZE 146 #define HN_TX_DATA_MAXSIZE IP_MAXPACKET 147 #define HN_TX_DATA_SEGSIZE PAGE_SIZE 148 /* -1 for RNDIS packet message */ 149 #define HN_TX_DATA_SEGCNT_MAX (HN_GPACNT_MAX - 1) 150 151 #define HN_DIRECT_TX_SIZE_DEF 128 152 153 #define HN_EARLY_TXEOF_THRESH 8 154 155 #define HN_PKTBUF_LEN_DEF (16 * 1024) 156 157 #define HN_LROENT_CNT_DEF 128 158 159 #define HN_LRO_LENLIM_MULTIRX_DEF (12 * ETHERMTU) 160 #define HN_LRO_LENLIM_DEF (25 * ETHERMTU) 161 /* YYY 2*MTU is a bit rough, but should be good enough. */ 162 #define HN_LRO_LENLIM_MIN(ifp) (2 * if_getmtu(ifp)) 163 164 #define HN_LRO_ACKCNT_DEF 1 165 166 #define HN_LOCK_INIT(sc) \ 167 sx_init(&(sc)->hn_lock, device_get_nameunit((sc)->hn_dev)) 168 #define HN_LOCK_DESTROY(sc) sx_destroy(&(sc)->hn_lock) 169 #define HN_LOCK_ASSERT(sc) sx_assert(&(sc)->hn_lock, SA_XLOCKED) 170 #define HN_LOCK(sc) \ 171 do { \ 172 while (sx_try_xlock(&(sc)->hn_lock) == 0) { \ 173 /* Relinquish cpu to avoid deadlock */ \ 174 sched_relinquish(curthread); \ 175 DELAY(1000); \ 176 } \ 177 } while (0) 178 #define HN_UNLOCK(sc) sx_xunlock(&(sc)->hn_lock) 179 180 #define HN_CSUM_IP_MASK (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP) 181 #define HN_CSUM_IP6_MASK (CSUM_IP6_TCP | CSUM_IP6_UDP) 182 #define HN_CSUM_IP_HWASSIST(sc) \ 183 ((sc)->hn_tx_ring[0].hn_csum_assist & HN_CSUM_IP_MASK) 184 #define HN_CSUM_IP6_HWASSIST(sc) \ 185 ((sc)->hn_tx_ring[0].hn_csum_assist & HN_CSUM_IP6_MASK) 186 187 /* Packet offloads can follow the VF; services requiring hn methods cannot. */ 188 #define HN_XPNT_VF_CAPS (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | \ 189 IFCAP_TSO | IFCAP_LRO | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 190 IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWFILTER | \ 191 IFCAP_JUMBO_MTU | IFCAP_LINKSTATE | IFCAP_HWSTATS | IFCAP_MEXTPG) 192 193 #define HN_PKTSIZE_MIN(align) \ 194 roundup2(ETHER_MIN_LEN + ETHER_VLAN_ENCAP_LEN - ETHER_CRC_LEN + \ 195 HN_RNDIS_PKT_LEN, (align)) 196 #define HN_PKTSIZE(m, align) \ 197 roundup2((m)->m_pkthdr.len + HN_RNDIS_PKT_LEN, (align)) 198 199 #ifdef RSS 200 #define HN_RING_IDX2CPU(sc, idx) rss_getcpu((idx) % rss_getnumbuckets()) 201 #else 202 #define HN_RING_IDX2CPU(sc, idx) (((sc)->hn_cpu + (idx)) % mp_ncpus) 203 #endif 204 205 struct hn_txdesc { 206 #ifndef HN_USE_TXDESC_BUFRING 207 SLIST_ENTRY(hn_txdesc) link; 208 #endif 209 STAILQ_ENTRY(hn_txdesc) agg_link; 210 211 /* Aggregated txdescs, in sending order. */ 212 STAILQ_HEAD(, hn_txdesc) agg_list; 213 214 /* The oldest packet, if transmission aggregation happens. */ 215 struct mbuf *m; 216 struct hn_tx_ring *txr; 217 int refs; 218 uint32_t flags; /* HN_TXD_FLAG_ */ 219 struct hn_nvs_sendctx send_ctx; 220 uint32_t chim_index; 221 int chim_size; 222 223 bus_dmamap_t data_dmap; 224 225 bus_addr_t rndis_pkt_paddr; 226 struct rndis_packet_msg *rndis_pkt; 227 bus_dmamap_t rndis_pkt_dmap; 228 }; 229 230 #define HN_TXD_FLAG_ONLIST 0x0001 231 #define HN_TXD_FLAG_DMAMAP 0x0002 232 #define HN_TXD_FLAG_ONAGG 0x0004 233 234 #define HN_NDIS_PKTINFO_SUBALLOC 0x01 235 #define HN_NDIS_PKTINFO_1ST_FRAG 0x02 236 #define HN_NDIS_PKTINFO_LAST_FRAG 0x04 237 238 struct packet_info_id { 239 uint8_t ver; 240 uint8_t flag; 241 uint16_t pkt_id; 242 }; 243 244 #define NDIS_PKTINFOID_SZ sizeof(struct packet_info_id) 245 246 247 struct hn_rxinfo { 248 const uint32_t *vlan_info; 249 const uint32_t *csum_info; 250 const uint32_t *hash_info; 251 const uint32_t *hash_value; 252 const struct packet_info_id *pktinfo_id; 253 }; 254 255 struct hn_rxvf_setarg { 256 struct hn_rx_ring *rxr; 257 if_t vf_ifp; 258 }; 259 260 #define HN_RXINFO_VLAN 0x0001 261 #define HN_RXINFO_CSUM 0x0002 262 #define HN_RXINFO_HASHINF 0x0004 263 #define HN_RXINFO_HASHVAL 0x0008 264 #define HN_RXINFO_PKTINFO_ID 0x0010 265 #define HN_RXINFO_ALL \ 266 (HN_RXINFO_VLAN | \ 267 HN_RXINFO_CSUM | \ 268 HN_RXINFO_HASHINF | \ 269 HN_RXINFO_HASHVAL | \ 270 HN_RXINFO_PKTINFO_ID) 271 272 static int hn_probe(device_t); 273 static int hn_attach(device_t); 274 static int hn_detach(device_t); 275 static int hn_shutdown(device_t); 276 static void hn_chan_callback(struct vmbus_channel *, 277 void *); 278 279 static void hn_init(void *); 280 static int hn_ioctl(if_t, u_long, caddr_t); 281 #ifdef HN_IFSTART_SUPPORT 282 static void hn_start(if_t); 283 #endif 284 static int hn_transmit(if_t, struct mbuf *); 285 static void hn_xmit_qflush(if_t); 286 static int hn_ifmedia_upd(if_t); 287 static void hn_ifmedia_sts(if_t, 288 struct ifmediareq *); 289 290 static void hn_ifnet_event(void *, if_t, int); 291 static void hn_ifaddr_event(void *, if_t); 292 static void hn_ifnet_attevent(void *, if_t); 293 static void hn_ifnet_detevent(void *, if_t); 294 static void hn_ifnet_lnkevent(void *, if_t, int); 295 296 static bool hn_ismyvf(const struct hn_softc *, 297 const if_t); 298 static void hn_rxvf_change(struct hn_softc *, 299 if_t); 300 static void hn_rxvf_set(struct hn_softc *, if_t); 301 static void hn_rxvf_change_locked(struct hn_softc *, if_t, 302 bool); 303 static void hn_rxvf_set_task(void *, int); 304 static void hn_xpnt_vf_input(if_t, struct mbuf *); 305 static int hn_xpnt_vf_iocsetflags(struct hn_softc *); 306 static int hn_xpnt_vf_iocsetcaps(struct hn_softc *, 307 struct ifreq *); 308 static void hn_xpnt_vf_synccaps(struct hn_softc *); 309 static void hn_xpnt_vf_vlancap_taskfunc(void *, int); 310 static void hn_xpnt_vf_saveifflags(struct hn_softc *); 311 static bool hn_xpnt_vf_isready(struct hn_softc *); 312 static bool hn_xpnt_vf_caninit(struct hn_softc *); 313 static void hn_xpnt_vf_setready(struct hn_softc *); 314 static void hn_xpnt_vf_restore(struct hn_softc *); 315 static void hn_xpnt_vf_deactivate(struct hn_softc *); 316 static void hn_xpnt_vf_init_taskfunc(void *, int); 317 static void hn_xpnt_vf_init(struct hn_softc *); 318 static bool hn_xpnt_vf_setenable(struct hn_softc *); 319 static void hn_xpnt_vf_setdisable(struct hn_softc *, bool); 320 static void hn_vf_rss_fixup(struct hn_softc *, bool); 321 static void hn_vf_rss_restore(struct hn_softc *); 322 323 static int hn_rndis_rxinfo(const void *, int, 324 struct hn_rxinfo *); 325 static void hn_rndis_rx_data(struct hn_rx_ring *, 326 const void *, int); 327 static void hn_rndis_rx_status(struct hn_softc *, 328 const void *, int); 329 static void hn_rndis_init_fixat(struct hn_softc *, int); 330 331 static void hn_nvs_handle_notify(struct hn_softc *, 332 const struct vmbus_chanpkt_hdr *); 333 static void hn_nvs_handle_comp(struct hn_softc *, 334 struct vmbus_channel *, 335 const struct vmbus_chanpkt_hdr *); 336 static void hn_nvs_handle_rxbuf(struct hn_rx_ring *, 337 struct vmbus_channel *, 338 const struct vmbus_chanpkt_hdr *); 339 static void hn_nvs_ack_rxbuf(struct hn_rx_ring *, 340 struct vmbus_channel *, uint64_t); 341 342 static int hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS); 343 static int hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS); 344 static int hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS); 345 static int hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS); 346 static int hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS); 347 static int hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); 348 static int hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); 349 static int hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS); 350 static int hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS); 351 static int hn_caps_sysctl(SYSCTL_HANDLER_ARGS); 352 static int hn_hwassist_sysctl(SYSCTL_HANDLER_ARGS); 353 static int hn_rxfilter_sysctl(SYSCTL_HANDLER_ARGS); 354 #ifndef RSS 355 static int hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS); 356 static int hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS); 357 #endif 358 static int hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS); 359 static int hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS); 360 static int hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS); 361 static int hn_txagg_size_sysctl(SYSCTL_HANDLER_ARGS); 362 static int hn_txagg_pkts_sysctl(SYSCTL_HANDLER_ARGS); 363 static int hn_txagg_pktmax_sysctl(SYSCTL_HANDLER_ARGS); 364 static int hn_txagg_align_sysctl(SYSCTL_HANDLER_ARGS); 365 static int hn_polling_sysctl(SYSCTL_HANDLER_ARGS); 366 static int hn_vf_sysctl(SYSCTL_HANDLER_ARGS); 367 static int hn_rxvf_sysctl(SYSCTL_HANDLER_ARGS); 368 static int hn_vflist_sysctl(SYSCTL_HANDLER_ARGS); 369 static int hn_vfmap_sysctl(SYSCTL_HANDLER_ARGS); 370 static int hn_xpnt_vf_accbpf_sysctl(SYSCTL_HANDLER_ARGS); 371 static int hn_xpnt_vf_enabled_sysctl(SYSCTL_HANDLER_ARGS); 372 373 static void hn_stop(struct hn_softc *, bool); 374 static void hn_init_locked(struct hn_softc *); 375 static int hn_chan_attach(struct hn_softc *, 376 struct vmbus_channel *); 377 static void hn_chan_detach(struct hn_softc *, 378 struct vmbus_channel *); 379 static int hn_attach_subchans(struct hn_softc *); 380 static void hn_detach_allchans(struct hn_softc *); 381 static void hn_chan_rollup(struct hn_rx_ring *, 382 struct hn_tx_ring *); 383 static void hn_set_ring_inuse(struct hn_softc *, int); 384 static int hn_synth_attach(struct hn_softc *, int); 385 static void hn_synth_detach(struct hn_softc *); 386 static int hn_synth_alloc_subchans(struct hn_softc *, 387 int *); 388 static bool hn_synth_attachable(const struct hn_softc *); 389 static void hn_suspend(struct hn_softc *); 390 static void hn_suspend_data(struct hn_softc *); 391 static void hn_suspend_mgmt(struct hn_softc *); 392 static void hn_resume(struct hn_softc *); 393 static void hn_resume_data(struct hn_softc *); 394 static void hn_resume_mgmt(struct hn_softc *); 395 static void hn_suspend_mgmt_taskfunc(void *, int); 396 static void hn_chan_drain(struct hn_softc *, 397 struct vmbus_channel *); 398 static void hn_disable_rx(struct hn_softc *); 399 static void hn_drain_rxtx(struct hn_softc *, int); 400 static void hn_polling(struct hn_softc *, u_int); 401 static void hn_chan_polling(struct vmbus_channel *, u_int); 402 static void hn_mtu_change_fixup(struct hn_softc *); 403 404 static void hn_update_link_status(struct hn_softc *); 405 static void hn_change_network(struct hn_softc *); 406 static void hn_link_taskfunc(void *, int); 407 static void hn_netchg_init_taskfunc(void *, int); 408 static void hn_netchg_status_taskfunc(void *, int); 409 static void hn_link_status(struct hn_softc *); 410 411 static int hn_create_rx_data(struct hn_softc *, int); 412 static void hn_destroy_rx_data(struct hn_softc *); 413 static int hn_check_iplen(const struct mbuf *, int); 414 static void hn_rxpkt_proto(const struct mbuf *, int *, int *); 415 static int hn_set_rxfilter(struct hn_softc *, uint32_t); 416 static int hn_rxfilter_config(struct hn_softc *); 417 static int hn_rss_reconfig(struct hn_softc *); 418 static void hn_rss_ind_fixup(struct hn_softc *); 419 static void hn_rss_mbuf_hash(struct hn_softc *, uint32_t); 420 static int hn_rxpkt(struct hn_rx_ring *); 421 static uint32_t hn_rss_type_fromndis(uint32_t); 422 static uint32_t hn_rss_type_tondis(uint32_t); 423 424 static int hn_tx_ring_create(struct hn_softc *, int); 425 static void hn_tx_ring_destroy(struct hn_tx_ring *); 426 static int hn_create_tx_data(struct hn_softc *, int); 427 static void hn_fixup_tx_data(struct hn_softc *); 428 static void hn_fixup_rx_data(struct hn_softc *); 429 static void hn_destroy_tx_data(struct hn_softc *); 430 static void hn_txdesc_dmamap_destroy(struct hn_txdesc *); 431 static void hn_txdesc_gc(struct hn_tx_ring *, 432 struct hn_txdesc *); 433 static int hn_encap(if_t, struct hn_tx_ring *, 434 struct hn_txdesc *, struct mbuf **); 435 static int hn_txpkt(if_t, struct hn_tx_ring *, 436 struct hn_txdesc *); 437 static void hn_set_chim_size(struct hn_softc *, int); 438 static void hn_set_tso_maxsize(struct hn_softc *, int, int); 439 static bool hn_tx_ring_pending(struct hn_tx_ring *); 440 static void hn_tx_ring_qflush(struct hn_tx_ring *); 441 static void hn_resume_tx(struct hn_softc *, int); 442 static void hn_set_txagg(struct hn_softc *); 443 static void *hn_try_txagg(if_t, 444 struct hn_tx_ring *, struct hn_txdesc *, 445 int); 446 static int hn_get_txswq_depth(const struct hn_tx_ring *); 447 static void hn_txpkt_done(struct hn_nvs_sendctx *, 448 struct hn_softc *, struct vmbus_channel *, 449 const void *, int); 450 static int hn_txpkt_sglist(struct hn_tx_ring *, 451 struct hn_txdesc *); 452 static int hn_txpkt_chim(struct hn_tx_ring *, 453 struct hn_txdesc *); 454 static int hn_xmit(struct hn_tx_ring *, int); 455 static void hn_xmit_taskfunc(void *, int); 456 static void hn_xmit_txeof(struct hn_tx_ring *); 457 static void hn_xmit_txeof_taskfunc(void *, int); 458 #ifdef HN_IFSTART_SUPPORT 459 static int hn_start_locked(struct hn_tx_ring *, int); 460 static void hn_start_taskfunc(void *, int); 461 static void hn_start_txeof(struct hn_tx_ring *); 462 static void hn_start_txeof_taskfunc(void *, int); 463 #endif 464 465 static int hn_rsc_sysctl(SYSCTL_HANDLER_ARGS); 466 467 SYSCTL_NODE(_hw, OID_AUTO, hn, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 468 "Hyper-V network interface"); 469 470 /* Trust tcp segment verification on host side. */ 471 static int hn_trust_hosttcp = 1; 472 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hosttcp, CTLFLAG_RDTUN, 473 &hn_trust_hosttcp, 0, 474 "Trust tcp segment verification on host side, " 475 "when csum info is missing (global setting)"); 476 477 /* Trust udp datagrams verification on host side. */ 478 static int hn_trust_hostudp = 1; 479 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostudp, CTLFLAG_RDTUN, 480 &hn_trust_hostudp, 0, 481 "Trust udp datagram verification on host side, " 482 "when csum info is missing (global setting)"); 483 484 /* Trust ip packets verification on host side. */ 485 static int hn_trust_hostip = 1; 486 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostip, CTLFLAG_RDTUN, 487 &hn_trust_hostip, 0, 488 "Trust ip packet verification on host side, " 489 "when csum info is missing (global setting)"); 490 491 /* 492 * Offload UDP/IPv4 checksum. 493 */ 494 static int hn_enable_udp4cs = 1; 495 SYSCTL_INT(_hw_hn, OID_AUTO, enable_udp4cs, CTLFLAG_RDTUN, 496 &hn_enable_udp4cs, 0, "Offload UDP/IPv4 checksum"); 497 498 /* 499 * Offload UDP/IPv6 checksum. 500 */ 501 static int hn_enable_udp6cs = 1; 502 SYSCTL_INT(_hw_hn, OID_AUTO, enable_udp6cs, CTLFLAG_RDTUN, 503 &hn_enable_udp6cs, 0, "Offload UDP/IPv6 checksum"); 504 505 /* Stats. */ 506 static counter_u64_t hn_udpcs_fixup; 507 SYSCTL_COUNTER_U64(_hw_hn, OID_AUTO, udpcs_fixup, CTLFLAG_RW, 508 &hn_udpcs_fixup, "# of UDP checksum fixup"); 509 510 /* 511 * See hn_set_hlen(). 512 * 513 * This value is for Azure. For Hyper-V, set this above 514 * 65536 to disable UDP datagram checksum fixup. 515 */ 516 static int hn_udpcs_fixup_mtu = 1420; 517 SYSCTL_INT(_hw_hn, OID_AUTO, udpcs_fixup_mtu, CTLFLAG_RWTUN, 518 &hn_udpcs_fixup_mtu, 0, "UDP checksum fixup MTU threshold"); 519 520 /* Limit TSO burst size */ 521 static int hn_tso_maxlen = IP_MAXPACKET; 522 SYSCTL_INT(_hw_hn, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, 523 &hn_tso_maxlen, 0, "TSO burst limit"); 524 525 /* Limit chimney send size */ 526 static int hn_tx_chimney_size = 0; 527 SYSCTL_INT(_hw_hn, OID_AUTO, tx_chimney_size, CTLFLAG_RDTUN, 528 &hn_tx_chimney_size, 0, "Chimney send packet size limit"); 529 530 /* Limit the size of packet for direct transmission */ 531 static int hn_direct_tx_size = HN_DIRECT_TX_SIZE_DEF; 532 SYSCTL_INT(_hw_hn, OID_AUTO, direct_tx_size, CTLFLAG_RDTUN, 533 &hn_direct_tx_size, 0, "Size of the packet for direct transmission"); 534 535 /* # of LRO entries per RX ring */ 536 #if defined(INET) || defined(INET6) 537 static int hn_lro_entry_count = HN_LROENT_CNT_DEF; 538 SYSCTL_INT(_hw_hn, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN, 539 &hn_lro_entry_count, 0, "LRO entry count"); 540 #endif 541 542 static int hn_tx_taskq_cnt = 1; 543 SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_cnt, CTLFLAG_RDTUN, 544 &hn_tx_taskq_cnt, 0, "# of TX taskqueues"); 545 546 #define HN_TX_TASKQ_M_INDEP 0 547 #define HN_TX_TASKQ_M_GLOBAL 1 548 #define HN_TX_TASKQ_M_EVTTQ 2 549 550 static int hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; 551 SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_mode, CTLFLAG_RDTUN, 552 &hn_tx_taskq_mode, 0, "TX taskqueue modes: " 553 "0 - independent, 1 - share global tx taskqs, 2 - share event taskqs"); 554 555 #ifndef HN_USE_TXDESC_BUFRING 556 static int hn_use_txdesc_bufring = 0; 557 #else 558 static int hn_use_txdesc_bufring = 1; 559 #endif 560 SYSCTL_INT(_hw_hn, OID_AUTO, use_txdesc_bufring, CTLFLAG_RD, 561 &hn_use_txdesc_bufring, 0, "Use buf_ring for TX descriptors"); 562 563 #ifdef HN_IFSTART_SUPPORT 564 /* Use ifnet.if_start instead of ifnet.if_transmit */ 565 static int hn_use_if_start = 0; 566 SYSCTL_INT(_hw_hn, OID_AUTO, use_if_start, CTLFLAG_RDTUN, 567 &hn_use_if_start, 0, "Use if_start TX method"); 568 #endif 569 570 /* # of channels to use */ 571 static int hn_chan_cnt = 0; 572 SYSCTL_INT(_hw_hn, OID_AUTO, chan_cnt, CTLFLAG_RDTUN, 573 &hn_chan_cnt, 0, 574 "# of channels to use; each channel has one RX ring and one TX ring"); 575 576 /* # of transmit rings to use */ 577 static int hn_tx_ring_cnt = 0; 578 SYSCTL_INT(_hw_hn, OID_AUTO, tx_ring_cnt, CTLFLAG_RDTUN, 579 &hn_tx_ring_cnt, 0, "# of TX rings to use"); 580 581 /* Software TX ring deptch */ 582 static int hn_tx_swq_depth = 0; 583 SYSCTL_INT(_hw_hn, OID_AUTO, tx_swq_depth, CTLFLAG_RDTUN, 584 &hn_tx_swq_depth, 0, "Depth of IFQ or BUFRING"); 585 586 /* Enable sorted LRO, and the depth of the per-channel mbuf queue */ 587 static u_int hn_lro_mbufq_depth = 0; 588 SYSCTL_UINT(_hw_hn, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN, 589 &hn_lro_mbufq_depth, 0, "Depth of LRO mbuf queue"); 590 591 /* Packet transmission aggregation size limit */ 592 static int hn_tx_agg_size = -1; 593 SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_size, CTLFLAG_RDTUN, 594 &hn_tx_agg_size, 0, "Packet transmission aggregation size limit"); 595 596 /* Packet transmission aggregation count limit */ 597 static int hn_tx_agg_pkts = -1; 598 SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_pkts, CTLFLAG_RDTUN, 599 &hn_tx_agg_pkts, 0, "Packet transmission aggregation packet limit"); 600 601 /* VF list */ 602 SYSCTL_PROC(_hw_hn, OID_AUTO, vflist, 603 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_NEEDGIANT, 0, 0, 604 hn_vflist_sysctl, "A", 605 "VF list"); 606 607 /* VF mapping */ 608 SYSCTL_PROC(_hw_hn, OID_AUTO, vfmap, 609 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_NEEDGIANT, 0, 0, 610 hn_vfmap_sysctl, "A", 611 "VF mapping"); 612 613 /* Transparent VF */ 614 static int hn_xpnt_vf = 1; 615 SYSCTL_INT(_hw_hn, OID_AUTO, vf_transparent, CTLFLAG_RDTUN, 616 &hn_xpnt_vf, 0, "Transparent VF mod"); 617 618 /* Accurate BPF support for Transparent VF */ 619 static int hn_xpnt_vf_accbpf = 0; 620 SYSCTL_INT(_hw_hn, OID_AUTO, vf_xpnt_accbpf, CTLFLAG_RDTUN, 621 &hn_xpnt_vf_accbpf, 0, "Accurate BPF for transparent VF"); 622 623 /* Extra wait for transparent VF attach routing; unit seconds. */ 624 static int hn_xpnt_vf_attwait = HN_XPNT_VF_ATTWAIT_MIN; 625 SYSCTL_INT(_hw_hn, OID_AUTO, vf_xpnt_attwait, CTLFLAG_RWTUN, 626 &hn_xpnt_vf_attwait, 0, 627 "Extra wait for transparent VF attach routing; unit: seconds"); 628 629 static u_int hn_cpu_index; /* next CPU for channel */ 630 static struct taskqueue **hn_tx_taskque;/* shared TX taskqueues */ 631 632 static struct rmlock hn_vfmap_lock; 633 static int hn_vfmap_size; 634 static if_t *hn_vfmap; 635 636 static const struct hyperv_guid hn_guid = { 637 .hv_guid = { 638 0x63, 0x51, 0x61, 0xf8, 0x3e, 0xdf, 0xc5, 0x46, 639 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e } 640 }; 641 642 static device_method_t hn_methods[] = { 643 /* Device interface */ 644 DEVMETHOD(device_probe, hn_probe), 645 DEVMETHOD(device_attach, hn_attach), 646 DEVMETHOD(device_detach, hn_detach), 647 DEVMETHOD(device_shutdown, hn_shutdown), 648 DEVMETHOD_END 649 }; 650 651 static driver_t hn_driver = { 652 "hn", 653 hn_methods, 654 sizeof(struct hn_softc) 655 }; 656 657 DRIVER_MODULE(hn, vmbus, hn_driver, 0, 0); 658 MODULE_VERSION(hn, 1); 659 MODULE_DEPEND(hn, vmbus, 1, 1, 1); 660 661 static void 662 hn_set_lro_lenlim(struct hn_softc *sc, int lenlim) 663 { 664 int i; 665 666 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 667 sc->hn_rx_ring[i].hn_lro.lro_length_lim = lenlim; 668 } 669 670 static int 671 hn_txpkt_sglist(struct hn_tx_ring *txr, struct hn_txdesc *txd) 672 { 673 674 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID && 675 txd->chim_size == 0, ("invalid rndis sglist txd")); 676 return (hn_nvs_send_rndis_sglist(txr->hn_chan, HN_NVS_RNDIS_MTYPE_DATA, 677 &txd->send_ctx, txr->hn_gpa, txr->hn_gpa_cnt)); 678 } 679 680 static int 681 hn_txpkt_chim(struct hn_tx_ring *txr, struct hn_txdesc *txd) 682 { 683 struct hn_nvs_rndis rndis; 684 685 KASSERT(txd->chim_index != HN_NVS_CHIM_IDX_INVALID && 686 txd->chim_size > 0, ("invalid rndis chim txd")); 687 688 rndis.nvs_type = HN_NVS_TYPE_RNDIS; 689 rndis.nvs_rndis_mtype = HN_NVS_RNDIS_MTYPE_DATA; 690 rndis.nvs_chim_idx = txd->chim_index; 691 rndis.nvs_chim_sz = txd->chim_size; 692 693 return (hn_nvs_send(txr->hn_chan, VMBUS_CHANPKT_FLAG_RC, 694 &rndis, sizeof(rndis), &txd->send_ctx)); 695 } 696 697 static __inline uint32_t 698 hn_chim_alloc(struct hn_softc *sc) 699 { 700 int i, bmap_cnt = sc->hn_chim_bmap_cnt; 701 u_long *bmap = sc->hn_chim_bmap; 702 uint32_t ret = HN_NVS_CHIM_IDX_INVALID; 703 704 for (i = 0; i < bmap_cnt; ++i) { 705 int idx; 706 707 idx = ffsl(~bmap[i]); 708 if (idx == 0) 709 continue; 710 711 --idx; /* ffsl is 1-based */ 712 KASSERT(i * LONG_BIT + idx < sc->hn_chim_cnt, 713 ("invalid i %d and idx %d", i, idx)); 714 715 if (atomic_testandset_long(&bmap[i], idx)) 716 continue; 717 718 ret = i * LONG_BIT + idx; 719 break; 720 } 721 return (ret); 722 } 723 724 static __inline void 725 hn_chim_free(struct hn_softc *sc, uint32_t chim_idx) 726 { 727 u_long mask; 728 uint32_t idx; 729 730 idx = chim_idx / LONG_BIT; 731 KASSERT(idx < sc->hn_chim_bmap_cnt, 732 ("invalid chimney index 0x%x", chim_idx)); 733 734 mask = 1UL << (chim_idx % LONG_BIT); 735 KASSERT(sc->hn_chim_bmap[idx] & mask, 736 ("index bitmap 0x%lx, chimney index %u, " 737 "bitmap idx %d, bitmask 0x%lx", 738 sc->hn_chim_bmap[idx], chim_idx, idx, mask)); 739 740 atomic_clear_long(&sc->hn_chim_bmap[idx], mask); 741 } 742 743 #if defined(INET6) || defined(INET) 744 745 #define PULLUP_HDR(m, len) \ 746 do { \ 747 if (__predict_false((m)->m_len < (len))) { \ 748 (m) = m_pullup((m), (len)); \ 749 if ((m) == NULL) \ 750 return (NULL); \ 751 } \ 752 } while (0) 753 754 /* 755 * NOTE: If this function failed, the m_head would be freed. 756 */ 757 static __inline struct mbuf * 758 hn_tso_fixup(struct mbuf *m_head) 759 { 760 struct ether_vlan_header *evl; 761 struct tcphdr *th; 762 int ehlen; 763 764 KASSERT(M_WRITABLE(m_head), ("TSO mbuf not writable")); 765 766 PULLUP_HDR(m_head, sizeof(*evl)); 767 evl = mtod(m_head, struct ether_vlan_header *); 768 if (evl->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) 769 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 770 else 771 ehlen = ETHER_HDR_LEN; 772 m_head->m_pkthdr.l2hlen = ehlen; 773 774 #ifdef INET 775 if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { 776 struct ip *ip; 777 int iphlen; 778 779 PULLUP_HDR(m_head, ehlen + sizeof(*ip)); 780 ip = mtodo(m_head, ehlen); 781 iphlen = ip->ip_hl << 2; 782 m_head->m_pkthdr.l3hlen = iphlen; 783 784 PULLUP_HDR(m_head, ehlen + iphlen + sizeof(*th)); 785 th = mtodo(m_head, ehlen + iphlen); 786 787 ip->ip_len = 0; 788 ip->ip_sum = 0; 789 th->th_sum = in_pseudo(ip->ip_src.s_addr, 790 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 791 } 792 #endif 793 #if defined(INET6) && defined(INET) 794 else 795 #endif 796 #ifdef INET6 797 { 798 struct ip6_hdr *ip6; 799 800 PULLUP_HDR(m_head, ehlen + sizeof(*ip6)); 801 ip6 = mtodo(m_head, ehlen); 802 if (ip6->ip6_nxt != IPPROTO_TCP) { 803 m_freem(m_head); 804 return (NULL); 805 } 806 m_head->m_pkthdr.l3hlen = sizeof(*ip6); 807 808 PULLUP_HDR(m_head, ehlen + sizeof(*ip6) + sizeof(*th)); 809 th = mtodo(m_head, ehlen + sizeof(*ip6)); 810 811 ip6->ip6_plen = 0; 812 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 813 } 814 #endif 815 return (m_head); 816 } 817 818 /* 819 * NOTE: If this function failed, the m_head would be freed. 820 */ 821 static __inline struct mbuf * 822 hn_set_hlen(struct mbuf *m_head) 823 { 824 const struct ether_vlan_header *evl; 825 int ehlen; 826 827 PULLUP_HDR(m_head, sizeof(*evl)); 828 evl = mtod(m_head, const struct ether_vlan_header *); 829 if (evl->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) 830 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 831 else 832 ehlen = ETHER_HDR_LEN; 833 m_head->m_pkthdr.l2hlen = ehlen; 834 835 #ifdef INET 836 if (m_head->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP_UDP)) { 837 const struct ip *ip; 838 int iphlen; 839 840 PULLUP_HDR(m_head, ehlen + sizeof(*ip)); 841 ip = mtodo(m_head, ehlen); 842 iphlen = ip->ip_hl << 2; 843 m_head->m_pkthdr.l3hlen = iphlen; 844 845 /* 846 * UDP checksum offload does not work in Azure, if the 847 * following conditions meet: 848 * - sizeof(IP hdr + UDP hdr + payload) > 1420. 849 * - IP_DF is not set in the IP hdr. 850 * 851 * Fallback to software checksum for these UDP datagrams. 852 */ 853 if ((m_head->m_pkthdr.csum_flags & CSUM_IP_UDP) && 854 m_head->m_pkthdr.len > hn_udpcs_fixup_mtu + ehlen && 855 (ntohs(ip->ip_off) & IP_DF) == 0) { 856 uint16_t off = ehlen + iphlen; 857 858 counter_u64_add(hn_udpcs_fixup, 1); 859 PULLUP_HDR(m_head, off + sizeof(struct udphdr)); 860 *(uint16_t *)(m_head->m_data + off + 861 m_head->m_pkthdr.csum_data) = in_cksum_skip( 862 m_head, m_head->m_pkthdr.len, off); 863 m_head->m_pkthdr.csum_flags &= ~CSUM_IP_UDP; 864 } 865 } 866 #endif 867 #if defined(INET6) && defined(INET) 868 else 869 #endif 870 #ifdef INET6 871 { 872 const struct ip6_hdr *ip6; 873 874 PULLUP_HDR(m_head, ehlen + sizeof(*ip6)); 875 ip6 = mtodo(m_head, ehlen); 876 if (ip6->ip6_nxt != IPPROTO_TCP && 877 ip6->ip6_nxt != IPPROTO_UDP) { 878 m_freem(m_head); 879 return (NULL); 880 } 881 m_head->m_pkthdr.l3hlen = sizeof(*ip6); 882 } 883 #endif 884 return (m_head); 885 } 886 887 /* 888 * NOTE: If this function failed, the m_head would be freed. 889 */ 890 static __inline struct mbuf * 891 hn_check_tcpsyn(struct mbuf *m_head, int *tcpsyn) 892 { 893 const struct tcphdr *th; 894 int ehlen, iphlen; 895 896 *tcpsyn = 0; 897 ehlen = m_head->m_pkthdr.l2hlen; 898 iphlen = m_head->m_pkthdr.l3hlen; 899 900 PULLUP_HDR(m_head, ehlen + iphlen + sizeof(*th)); 901 th = mtodo(m_head, ehlen + iphlen); 902 if (tcp_get_flags(th) & TH_SYN) 903 *tcpsyn = 1; 904 return (m_head); 905 } 906 907 #undef PULLUP_HDR 908 909 #endif /* INET6 || INET */ 910 911 static int 912 hn_set_rxfilter(struct hn_softc *sc, uint32_t filter) 913 { 914 int error = 0; 915 916 HN_LOCK_ASSERT(sc); 917 918 if (sc->hn_rx_filter != filter) { 919 error = hn_rndis_set_rxfilter(sc, filter); 920 if (!error) 921 sc->hn_rx_filter = filter; 922 } 923 return (error); 924 } 925 926 static int 927 hn_rxfilter_config(struct hn_softc *sc) 928 { 929 if_t ifp = sc->hn_ifp; 930 uint32_t filter; 931 932 HN_LOCK_ASSERT(sc); 933 934 /* 935 * If the non-transparent mode VF is activated, we don't know how 936 * its RX filter is configured, so stick the synthetic device in 937 * the promiscous mode. 938 */ 939 if ((if_getflags(ifp) & IFF_PROMISC) || (sc->hn_flags & HN_FLAG_RXVF)) { 940 filter = NDIS_PACKET_TYPE_PROMISCUOUS; 941 } else { 942 filter = NDIS_PACKET_TYPE_DIRECTED; 943 if (if_getflags(ifp) & IFF_BROADCAST) 944 filter |= NDIS_PACKET_TYPE_BROADCAST; 945 /* TODO: support multicast list */ 946 if ((if_getflags(ifp) & IFF_ALLMULTI) || 947 !if_maddr_empty(ifp)) 948 filter |= NDIS_PACKET_TYPE_ALL_MULTICAST; 949 } 950 return (hn_set_rxfilter(sc, filter)); 951 } 952 953 static void 954 hn_set_txagg(struct hn_softc *sc) 955 { 956 uint32_t size, pkts; 957 int i; 958 959 /* 960 * Setup aggregation size. 961 */ 962 if (sc->hn_agg_size < 0) 963 size = UINT32_MAX; 964 else 965 size = sc->hn_agg_size; 966 967 if (sc->hn_rndis_agg_size < size) 968 size = sc->hn_rndis_agg_size; 969 970 /* NOTE: We only aggregate packets using chimney sending buffers. */ 971 if (size > (uint32_t)sc->hn_chim_szmax) 972 size = sc->hn_chim_szmax; 973 974 if (size <= 2 * HN_PKTSIZE_MIN(sc->hn_rndis_agg_align)) { 975 /* Disable */ 976 size = 0; 977 pkts = 0; 978 goto done; 979 } 980 981 /* NOTE: Type of the per TX ring setting is 'int'. */ 982 if (size > INT_MAX) 983 size = INT_MAX; 984 985 /* 986 * Setup aggregation packet count. 987 */ 988 if (sc->hn_agg_pkts < 0) 989 pkts = UINT32_MAX; 990 else 991 pkts = sc->hn_agg_pkts; 992 993 if (sc->hn_rndis_agg_pkts < pkts) 994 pkts = sc->hn_rndis_agg_pkts; 995 996 if (pkts <= 1) { 997 /* Disable */ 998 size = 0; 999 pkts = 0; 1000 goto done; 1001 } 1002 1003 /* NOTE: Type of the per TX ring setting is 'short'. */ 1004 if (pkts > SHRT_MAX) 1005 pkts = SHRT_MAX; 1006 1007 done: 1008 /* NOTE: Type of the per TX ring setting is 'short'. */ 1009 if (sc->hn_rndis_agg_align > SHRT_MAX) { 1010 /* Disable */ 1011 size = 0; 1012 pkts = 0; 1013 } 1014 1015 if (bootverbose) { 1016 if_printf(sc->hn_ifp, "TX agg size %u, pkts %u, align %u\n", 1017 size, pkts, sc->hn_rndis_agg_align); 1018 } 1019 1020 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 1021 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 1022 1023 mtx_lock(&txr->hn_tx_lock); 1024 txr->hn_agg_szmax = size; 1025 txr->hn_agg_pktmax = pkts; 1026 txr->hn_agg_align = sc->hn_rndis_agg_align; 1027 mtx_unlock(&txr->hn_tx_lock); 1028 } 1029 } 1030 1031 static int 1032 hn_get_txswq_depth(const struct hn_tx_ring *txr) 1033 { 1034 1035 KASSERT(txr->hn_txdesc_cnt > 0, ("tx ring is not setup yet")); 1036 if (hn_tx_swq_depth < txr->hn_txdesc_cnt) 1037 return txr->hn_txdesc_cnt; 1038 return hn_tx_swq_depth; 1039 } 1040 1041 static int 1042 hn_rss_reconfig(struct hn_softc *sc) 1043 { 1044 int error; 1045 1046 HN_LOCK_ASSERT(sc); 1047 1048 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 1049 return (ENXIO); 1050 1051 /* 1052 * Disable RSS first. 1053 * 1054 * NOTE: 1055 * Direct reconfiguration by setting the UNCHG flags does 1056 * _not_ work properly. 1057 */ 1058 if (bootverbose) 1059 if_printf(sc->hn_ifp, "disable RSS\n"); 1060 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_DISABLE); 1061 if (error) { 1062 if_printf(sc->hn_ifp, "RSS disable failed\n"); 1063 return (error); 1064 } 1065 1066 /* 1067 * Reenable the RSS w/ the updated RSS key or indirect 1068 * table. 1069 */ 1070 if (bootverbose) 1071 if_printf(sc->hn_ifp, "reconfig RSS\n"); 1072 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE); 1073 if (error) { 1074 if_printf(sc->hn_ifp, "RSS reconfig failed\n"); 1075 return (error); 1076 } 1077 return (0); 1078 } 1079 1080 static void 1081 hn_rss_ind_fixup(struct hn_softc *sc) 1082 { 1083 struct ndis_rssprm_toeplitz *rss = &sc->hn_rss; 1084 int i, nchan; 1085 1086 nchan = sc->hn_rx_ring_inuse; 1087 KASSERT(nchan > 1, ("invalid # of channels %d", nchan)); 1088 1089 /* 1090 * Check indirect table to make sure that all channels in it 1091 * can be used. 1092 */ 1093 for (i = 0; i < NDIS_HASH_INDCNT; ++i) { 1094 if (rss->rss_ind[i] >= nchan) { 1095 if_printf(sc->hn_ifp, 1096 "RSS indirect table %d fixup: %u -> %d\n", 1097 i, rss->rss_ind[i], nchan - 1); 1098 rss->rss_ind[i] = nchan - 1; 1099 } 1100 } 1101 } 1102 1103 static int 1104 hn_ifmedia_upd(if_t ifp __unused) 1105 { 1106 1107 /* Ignore since autoselect is the only defined and valid media */ 1108 return (0); 1109 } 1110 1111 static void 1112 hn_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) 1113 { 1114 struct hn_softc *sc = if_getsoftc(ifp); 1115 1116 ifmr->ifm_status = IFM_AVALID; 1117 ifmr->ifm_active = IFM_ETHER; 1118 1119 if ((sc->hn_link_flags & HN_LINK_FLAG_LINKUP) == 0) { 1120 ifmr->ifm_active |= IFM_NONE; 1121 return; 1122 } 1123 ifmr->ifm_status |= IFM_ACTIVE; 1124 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1125 } 1126 1127 static void 1128 hn_rxvf_set_task(void *xarg, int pending __unused) 1129 { 1130 struct hn_rxvf_setarg *arg = xarg; 1131 1132 arg->rxr->hn_rxvf_ifp = arg->vf_ifp; 1133 } 1134 1135 static void 1136 hn_rxvf_set(struct hn_softc *sc, if_t vf_ifp) 1137 { 1138 struct hn_rx_ring *rxr; 1139 struct hn_rxvf_setarg arg; 1140 struct task task; 1141 int i; 1142 1143 HN_LOCK_ASSERT(sc); 1144 1145 TASK_INIT(&task, 0, hn_rxvf_set_task, &arg); 1146 1147 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 1148 rxr = &sc->hn_rx_ring[i]; 1149 1150 if (i < sc->hn_rx_ring_inuse) { 1151 arg.rxr = rxr; 1152 arg.vf_ifp = vf_ifp; 1153 vmbus_chan_run_task(rxr->hn_chan, &task); 1154 } else { 1155 rxr->hn_rxvf_ifp = vf_ifp; 1156 } 1157 } 1158 } 1159 1160 static bool 1161 hn_ismyvf(const struct hn_softc *sc, const if_t ifp) 1162 { 1163 if_t hn_ifp; 1164 1165 hn_ifp = sc->hn_ifp; 1166 1167 if (ifp == hn_ifp) 1168 return (false); 1169 1170 if (if_getalloctype(ifp) != IFT_ETHER) 1171 return (false); 1172 1173 /* Ignore lagg/vlan interfaces */ 1174 if (strcmp(if_getdname(ifp), "lagg") == 0 || 1175 strcmp(if_getdname(ifp), "vlan") == 0) 1176 return (false); 1177 1178 /* 1179 * During detach events if_getifaddr(ifp) might be NULL. 1180 * Make sure the bcmp() below doesn't panic on that: 1181 */ 1182 if (if_getifaddr(ifp) == NULL || if_getifaddr(hn_ifp) == NULL) 1183 return (false); 1184 1185 if (bcmp(if_getlladdr(ifp), if_getlladdr(hn_ifp), ETHER_ADDR_LEN) != 0) 1186 return (false); 1187 1188 return (true); 1189 } 1190 1191 static void 1192 hn_rxvf_change(struct hn_softc *sc, if_t ifp) 1193 { 1194 struct rm_priotracker pt; 1195 1196 /* Address events can run on the channel that delivers completions. */ 1197 rm_rlock(&sc->hn_vf_lock, &pt); 1198 if (sc->hn_vf_ifp == ifp) 1199 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 1200 rm_runlock(&sc->hn_vf_lock, &pt); 1201 } 1202 1203 static void 1204 hn_rxvf_change_locked(struct hn_softc *sc, if_t ifp, bool rxvf) 1205 { 1206 if_t hn_ifp; 1207 u_int assoc, old_flags; 1208 int error; 1209 1210 HN_LOCK_ASSERT(sc); 1211 1212 if (!(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 1213 goto out; 1214 1215 if (!hn_ismyvf(sc, ifp)) 1216 goto out; 1217 hn_ifp = sc->hn_ifp; 1218 assoc = atomic_load_acq_int(&sc->hn_vf_assoc); 1219 if (rxvf && !(assoc & HN_VF_ASSOC_ALLOCATED)) 1220 goto out; 1221 old_flags = sc->hn_flags; 1222 1223 if (rxvf) { 1224 if (sc->hn_flags & HN_FLAG_RXVF) 1225 goto out; 1226 1227 sc->hn_flags |= HN_FLAG_RXVF; 1228 hn_rxfilter_config(sc); 1229 } else { 1230 if (!(sc->hn_flags & HN_FLAG_RXVF)) 1231 goto out; 1232 1233 sc->hn_flags &= ~HN_FLAG_RXVF; 1234 if (if_getdrvflags(hn_ifp) & IFF_DRV_RUNNING) 1235 hn_rxfilter_config(sc); 1236 else 1237 hn_set_rxfilter(sc, NDIS_PACKET_TYPE_NONE); 1238 } 1239 1240 /* Prepare receive routing before the host enables the VF path. */ 1241 if (rxvf) 1242 hn_rxvf_set(sc, ifp); 1243 error = hn_nvs_set_datapath(sc, 1244 rxvf ? HN_NVS_DATAPATH_VF : HN_NVS_DATAPATH_SYNTH); 1245 if (error == 0 && rxvf && 1246 assoc != atomic_load_acq_int(&sc->hn_vf_assoc)) { 1247 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_SYNTH); 1248 error = EAGAIN; 1249 } 1250 if (error) { 1251 sc->hn_flags = old_flags; 1252 hn_rxvf_set(sc, (old_flags & HN_FLAG_RXVF) ? ifp : NULL); 1253 if ((old_flags & HN_FLAG_RXVF) || 1254 (if_getdrvflags(hn_ifp) & IFF_DRV_RUNNING)) 1255 hn_rxfilter_config(sc); 1256 else 1257 hn_set_rxfilter(sc, NDIS_PACKET_TYPE_NONE); 1258 if (error == EAGAIN) 1259 taskqueue_enqueue_timeout(sc->hn_vf_taskq, 1260 &sc->hn_vf_init, hz); 1261 goto out; 1262 } 1263 sc->hn_vf_active_assoc = rxvf ? assoc : 0; 1264 1265 if (!rxvf) 1266 hn_rxvf_set(sc, NULL); 1267 1268 if (rxvf) { 1269 hn_vf_rss_fixup(sc, true); 1270 hn_suspend_mgmt(sc); 1271 sc->hn_link_flags &= 1272 ~(HN_LINK_FLAG_LINKUP | HN_LINK_FLAG_NETCHG); 1273 if_link_state_change(hn_ifp, LINK_STATE_DOWN); 1274 } else { 1275 hn_vf_rss_restore(sc); 1276 hn_resume_mgmt(sc); 1277 } 1278 1279 devctl_notify("HYPERV_NIC_VF", if_name(hn_ifp), 1280 rxvf ? "VF_UP" : "VF_DOWN", NULL); 1281 1282 if (bootverbose) { 1283 if_printf(hn_ifp, "datapath is switched %s %s\n", 1284 rxvf ? "to" : "from", if_name(ifp)); 1285 } 1286 out: 1287 return; 1288 } 1289 1290 static void 1291 hn_ifnet_event(void *arg, if_t ifp, int event) 1292 { 1293 1294 if (event != IFNET_EVENT_UP && event != IFNET_EVENT_DOWN) 1295 return; 1296 hn_rxvf_change(arg, ifp); 1297 } 1298 1299 static void 1300 hn_ifaddr_event(void *arg, if_t ifp) 1301 { 1302 1303 hn_rxvf_change(arg, ifp); 1304 } 1305 1306 static void 1307 hn_xpnt_vf_synccaps(struct hn_softc *sc) 1308 { 1309 if_t ifp, vf_ifp; 1310 int caps; 1311 1312 HN_LOCK_ASSERT(sc); 1313 ifp = sc->hn_ifp; 1314 vf_ifp = sc->hn_vf_ifp; 1315 caps = if_getcapabilities(ifp); 1316 1317 /* Reflect the actual VF state even if an ioctl failed partway through. */ 1318 if_setcapenable(ifp, if_getcapenable(vf_ifp) & caps); 1319 if_sethwassist(ifp, if_gethwassist(vf_ifp) & 1320 (HN_CSUM_IP_MASK | HN_CSUM_IP6_MASK | CSUM_TSO)); 1321 /* The worker refreshes VLAN children without holding hn_lock. */ 1322 if (!sc->hn_detaching) 1323 taskqueue_enqueue(sc->hn_vf_taskq, &sc->hn_vf_vlancap); 1324 } 1325 1326 static void 1327 hn_xpnt_vf_vlancap_taskfunc(void *xsc, int pending __unused) 1328 { 1329 struct hn_softc *sc = xsc; 1330 1331 /* VLAN configuration takes vlan_sx before entering hn ioctls. */ 1332 sx_assert(&sc->hn_lock, SA_UNLOCKED); 1333 if_vlancap(sc->hn_ifp); 1334 } 1335 1336 static int 1337 hn_xpnt_vf_iocsetcaps(struct hn_softc *sc, struct ifreq *ifr) 1338 { 1339 if_t vf_ifp; 1340 u_int assoc; 1341 int caps, error; 1342 1343 HN_LOCK_ASSERT(sc); 1344 if (sc->hn_vf_caps_busy) 1345 return (EBUSY); 1346 if (sc->hn_detaching || sc->hn_vf_detaching || 1347 !hn_xpnt_vf_isready(sc)) 1348 return (ENXIO); 1349 1350 vf_ifp = sc->hn_vf_ifp; 1351 if_ref(vf_ifp); 1352 assoc = sc->hn_vf_active_assoc; 1353 caps = if_getcapabilities(sc->hn_ifp); 1354 /* Leave capabilities which hn does not expose unchanged on the VF. */ 1355 ifr->ifr_reqcap = (ifr->ifr_reqcap & caps) | 1356 (if_getcapenable(vf_ifp) & ~caps); 1357 sc->hn_vf_caps_busy = true; 1358 rm_wlock(&sc->hn_vf_lock); 1359 sc->hn_xvf_flags |= HN_XVFFLAG_SWITCHING; 1360 rm_wunlock(&sc->hn_vf_lock); 1361 1362 /* 1363 * The VF ioctl may acquire vlan_sx, whose callers enter hn ioctls. 1364 * Detach waits for hn_vf_caps_busy with hn_lock released; new VF 1365 * initialization and capability changes cannot overlap this ioctl. 1366 */ 1367 HN_UNLOCK(sc); 1368 error = ifhwioctl(SIOCSIFCAP, vf_ifp, (caddr_t)ifr, curthread); 1369 HN_LOCK(sc); 1370 1371 if (!sc->hn_detaching && !sc->hn_vf_detaching && 1372 sc->hn_vf_ifp == vf_ifp && hn_xpnt_vf_isready(sc) && 1373 sc->hn_vf_active_assoc == assoc) { 1374 hn_xpnt_vf_synccaps(sc); 1375 rm_wlock(&sc->hn_vf_lock); 1376 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1377 /* A link event during the ioctl was suppressed by SWITCHING. */ 1378 if ((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) && 1379 hn_xpnt_vf_isready(sc)) 1380 if_link_state_change(sc->hn_ifp, 1381 if_getlinkstate(vf_ifp)); 1382 rm_wunlock(&sc->hn_vf_lock); 1383 } else if (error == 0) { 1384 error = sc->hn_detaching || sc->hn_vf_detaching ? ENXIO : EAGAIN; 1385 } 1386 if_rele(vf_ifp); 1387 sc->hn_vf_caps_busy = false; 1388 wakeup(&sc->hn_vf_caps_busy); 1389 /* Retry association work deferred while the ioctl was in flight. */ 1390 if (!sc->hn_detaching && !sc->hn_vf_detaching) 1391 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 1392 return (error); 1393 } 1394 1395 static int 1396 hn_xpnt_vf_iocsetflags(struct hn_softc *sc) 1397 { 1398 if_t vf_ifp; 1399 struct ifreq ifr; 1400 1401 HN_LOCK_ASSERT(sc); 1402 vf_ifp = sc->hn_vf_ifp; 1403 1404 memset(&ifr, 0, sizeof(ifr)); 1405 strlcpy(ifr.ifr_name, if_name(vf_ifp), sizeof(ifr.ifr_name)); 1406 ifr.ifr_flags = if_getflags(vf_ifp) & 0xffff; 1407 ifr.ifr_flagshigh = if_getflags(vf_ifp) >> 16; 1408 return (ifhwioctl(SIOCSIFFLAGS, vf_ifp, (caddr_t)&ifr, curthread)); 1409 } 1410 1411 static void 1412 hn_xpnt_vf_saveifflags(struct hn_softc *sc) 1413 { 1414 if_t ifp = sc->hn_ifp; 1415 int allmulti = 0; 1416 1417 HN_LOCK_ASSERT(sc); 1418 1419 /* XXX vlan(4) style mcast addr maintenance */ 1420 if (!if_maddr_empty(ifp)) 1421 allmulti = IFF_ALLMULTI; 1422 1423 /* Always set the VF's if_flags */ 1424 if_setflags(sc->hn_vf_ifp, if_getflags(ifp) | allmulti); 1425 } 1426 1427 static void 1428 hn_xpnt_vf_input(if_t vf_ifp, struct mbuf *m) 1429 { 1430 struct rm_priotracker pt; 1431 if_t hn_ifp = NULL; 1432 struct mbuf *mn; 1433 1434 /* 1435 * XXX racy, if hn(4) ever detached. 1436 */ 1437 rm_rlock(&hn_vfmap_lock, &pt); 1438 if (if_getindex(vf_ifp) < hn_vfmap_size) 1439 hn_ifp = hn_vfmap[if_getindex(vf_ifp)]; 1440 rm_runlock(&hn_vfmap_lock, &pt); 1441 1442 if (hn_ifp != NULL) { 1443 for (mn = m; mn != NULL; mn = mn->m_nextpkt) { 1444 /* 1445 * Allow tapping on the VF. 1446 */ 1447 ETHER_BPF_MTAP(vf_ifp, mn); 1448 1449 /* 1450 * Update VF stats. 1451 */ 1452 if ((if_getcapenable(vf_ifp) & IFCAP_HWSTATS) == 0) { 1453 if_inc_counter(vf_ifp, IFCOUNTER_IBYTES, 1454 mn->m_pkthdr.len); 1455 } 1456 /* 1457 * XXX IFCOUNTER_IMCAST 1458 * This stat updating is kinda invasive, since it 1459 * requires two checks on the mbuf: the length check 1460 * and the ethernet header check. As of this write, 1461 * all multicast packets go directly to hn(4), which 1462 * makes imcast stat updating in the VF a try in vian. 1463 */ 1464 1465 /* 1466 * Fix up rcvif and increase hn(4)'s ipackets. 1467 */ 1468 mn->m_pkthdr.rcvif = hn_ifp; 1469 if_inc_counter(hn_ifp, IFCOUNTER_IPACKETS, 1); 1470 } 1471 /* 1472 * Go through hn(4)'s if_input. 1473 */ 1474 if_input(hn_ifp, m); 1475 } else { 1476 /* 1477 * In the middle of the transition; free this 1478 * mbuf chain. 1479 */ 1480 while (m != NULL) { 1481 mn = m->m_nextpkt; 1482 m->m_nextpkt = NULL; 1483 m_freem(m); 1484 m = mn; 1485 } 1486 } 1487 } 1488 1489 static void 1490 hn_mtu_change_fixup(struct hn_softc *sc) 1491 { 1492 if_t ifp; 1493 1494 HN_LOCK_ASSERT(sc); 1495 ifp = sc->hn_ifp; 1496 1497 hn_set_tso_maxsize(sc, hn_tso_maxlen, if_getmtu(ifp)); 1498 if (sc->hn_rx_ring[0].hn_lro.lro_length_lim < HN_LRO_LENLIM_MIN(ifp)) 1499 hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MIN(ifp)); 1500 } 1501 1502 static uint32_t 1503 hn_rss_type_fromndis(uint32_t rss_hash) 1504 { 1505 uint32_t types = 0; 1506 1507 if (rss_hash & NDIS_HASH_IPV4) 1508 types |= RSS_TYPE_IPV4; 1509 if (rss_hash & NDIS_HASH_TCP_IPV4) 1510 types |= RSS_TYPE_TCP_IPV4; 1511 if (rss_hash & NDIS_HASH_IPV6) 1512 types |= RSS_TYPE_IPV6; 1513 if (rss_hash & NDIS_HASH_IPV6_EX) 1514 types |= RSS_TYPE_IPV6_EX; 1515 if (rss_hash & NDIS_HASH_TCP_IPV6) 1516 types |= RSS_TYPE_TCP_IPV6; 1517 if (rss_hash & NDIS_HASH_TCP_IPV6_EX) 1518 types |= RSS_TYPE_TCP_IPV6_EX; 1519 if (rss_hash & NDIS_HASH_UDP_IPV4_X) 1520 types |= RSS_TYPE_UDP_IPV4; 1521 return (types); 1522 } 1523 1524 static uint32_t 1525 hn_rss_type_tondis(uint32_t types) 1526 { 1527 uint32_t rss_hash = 0; 1528 1529 KASSERT((types & (RSS_TYPE_UDP_IPV6 | RSS_TYPE_UDP_IPV6_EX)) == 0, 1530 ("UDP6 and UDP6EX are not supported")); 1531 1532 if (types & RSS_TYPE_IPV4) 1533 rss_hash |= NDIS_HASH_IPV4; 1534 if (types & RSS_TYPE_TCP_IPV4) 1535 rss_hash |= NDIS_HASH_TCP_IPV4; 1536 if (types & RSS_TYPE_IPV6) 1537 rss_hash |= NDIS_HASH_IPV6; 1538 if (types & RSS_TYPE_IPV6_EX) 1539 rss_hash |= NDIS_HASH_IPV6_EX; 1540 if (types & RSS_TYPE_TCP_IPV6) 1541 rss_hash |= NDIS_HASH_TCP_IPV6; 1542 if (types & RSS_TYPE_TCP_IPV6_EX) 1543 rss_hash |= NDIS_HASH_TCP_IPV6_EX; 1544 if (types & RSS_TYPE_UDP_IPV4) 1545 rss_hash |= NDIS_HASH_UDP_IPV4_X; 1546 return (rss_hash); 1547 } 1548 1549 static void 1550 hn_rss_mbuf_hash(struct hn_softc *sc, uint32_t mbuf_hash) 1551 { 1552 int i; 1553 1554 HN_LOCK_ASSERT(sc); 1555 1556 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1557 sc->hn_rx_ring[i].hn_mbuf_hash = mbuf_hash; 1558 } 1559 1560 static void 1561 hn_vf_rss_fixup(struct hn_softc *sc, bool reconf) 1562 { 1563 if_t ifp, vf_ifp; 1564 struct ifrsshash ifrh; 1565 struct ifrsskey ifrk; 1566 int error; 1567 uint32_t my_types, diff_types, mbuf_types = 0; 1568 1569 HN_LOCK_ASSERT(sc); 1570 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 1571 ("%s: synthetic parts are not attached", if_name(sc->hn_ifp))); 1572 1573 if (sc->hn_rx_ring_inuse == 1) { 1574 /* No RSS on synthetic parts; done. */ 1575 return; 1576 } 1577 if ((sc->hn_rss_hcap & NDIS_HASH_FUNCTION_TOEPLITZ) == 0) { 1578 /* Synthetic parts do not support Toeplitz; done. */ 1579 return; 1580 } 1581 1582 ifp = sc->hn_ifp; 1583 vf_ifp = sc->hn_vf_ifp; 1584 1585 /* 1586 * Extract VF's RSS key. Only 40 bytes key for Toeplitz is 1587 * supported. 1588 */ 1589 memset(&ifrk, 0, sizeof(ifrk)); 1590 strlcpy(ifrk.ifrk_name, if_name(vf_ifp), sizeof(ifrk.ifrk_name)); 1591 error = ifhwioctl(SIOCGIFRSSKEY, vf_ifp, (caddr_t)&ifrk, curthread); 1592 if (error) { 1593 if (error != EOPNOTSUPP) 1594 if_printf(ifp, "%s SIOCGIFRSSKEY failed: %d\n", 1595 if_name(vf_ifp), error); 1596 goto done; 1597 } 1598 if (ifrk.ifrk_func == RSS_FUNC_NONE) 1599 goto done; 1600 if (ifrk.ifrk_func != RSS_FUNC_TOEPLITZ) { 1601 if_printf(ifp, "%s RSS function %u is not Toeplitz\n", 1602 if_name(vf_ifp), ifrk.ifrk_func); 1603 goto done; 1604 } 1605 if (ifrk.ifrk_keylen != NDIS_HASH_KEYSIZE_TOEPLITZ) { 1606 if_printf(ifp, "%s invalid RSS Toeplitz key length %d\n", 1607 if_name(vf_ifp), ifrk.ifrk_keylen); 1608 goto done; 1609 } 1610 1611 /* 1612 * Extract VF's RSS hash. Only Toeplitz is supported. 1613 */ 1614 memset(&ifrh, 0, sizeof(ifrh)); 1615 strlcpy(ifrh.ifrh_name, if_name(vf_ifp), sizeof(ifrh.ifrh_name)); 1616 error = ifhwioctl(SIOCGIFRSSHASH, vf_ifp, (caddr_t)&ifrh, curthread); 1617 if (error) { 1618 if (error != EOPNOTSUPP) 1619 if_printf(ifp, "%s SIOCGIFRSSHASH failed: %d\n", 1620 if_name(vf_ifp), error); 1621 goto done; 1622 } 1623 if (ifrh.ifrh_func == RSS_FUNC_NONE) 1624 goto done; 1625 if (ifrh.ifrh_func != RSS_FUNC_TOEPLITZ) { 1626 if_printf(ifp, "%s RSS function %u is not Toeplitz\n", 1627 if_name(vf_ifp), ifrh.ifrh_func); 1628 goto done; 1629 } 1630 1631 my_types = hn_rss_type_fromndis(sc->hn_rss_hcap); 1632 if ((ifrh.ifrh_types & my_types) == 0) { 1633 /* This disables RSS; ignore it then */ 1634 if_printf(ifp, "%s intersection of RSS types failed. " 1635 "VF %#x, mine %#x\n", if_name(vf_ifp), 1636 ifrh.ifrh_types, my_types); 1637 goto done; 1638 } 1639 1640 diff_types = my_types ^ ifrh.ifrh_types; 1641 my_types &= ifrh.ifrh_types; 1642 mbuf_types = my_types; 1643 1644 /* 1645 * Detect RSS hash value/type confliction. 1646 * 1647 * NOTE: 1648 * We don't disable the hash type, but stop delivery the hash 1649 * value/type through mbufs on RX path. 1650 * 1651 * XXX If HN_CAP_UDPHASH is set in hn_caps, then UDP 4-tuple 1652 * hash is delivered with type of TCP_IPV4. This means if 1653 * UDP_IPV4 is enabled, then TCP_IPV4 should be forced, at 1654 * least to hn_mbuf_hash. However, given that _all_ of the 1655 * NICs implement TCP_IPV4, this will _not_ impose any issues 1656 * here. 1657 */ 1658 if ((my_types & RSS_TYPE_IPV4) && 1659 (diff_types & ifrh.ifrh_types & 1660 (RSS_TYPE_TCP_IPV4 | RSS_TYPE_UDP_IPV4))) { 1661 /* Conflict; disable IPV4 hash type/value delivery. */ 1662 if_printf(ifp, "disable IPV4 mbuf hash delivery\n"); 1663 mbuf_types &= ~RSS_TYPE_IPV4; 1664 } 1665 if ((my_types & RSS_TYPE_IPV6) && 1666 (diff_types & ifrh.ifrh_types & 1667 (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 | 1668 RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX | 1669 RSS_TYPE_IPV6_EX))) { 1670 /* Conflict; disable IPV6 hash type/value delivery. */ 1671 if_printf(ifp, "disable IPV6 mbuf hash delivery\n"); 1672 mbuf_types &= ~RSS_TYPE_IPV6; 1673 } 1674 if ((my_types & RSS_TYPE_IPV6_EX) && 1675 (diff_types & ifrh.ifrh_types & 1676 (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 | 1677 RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX | 1678 RSS_TYPE_IPV6))) { 1679 /* Conflict; disable IPV6_EX hash type/value delivery. */ 1680 if_printf(ifp, "disable IPV6_EX mbuf hash delivery\n"); 1681 mbuf_types &= ~RSS_TYPE_IPV6_EX; 1682 } 1683 if ((my_types & RSS_TYPE_TCP_IPV6) && 1684 (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6_EX)) { 1685 /* Conflict; disable TCP_IPV6 hash type/value delivery. */ 1686 if_printf(ifp, "disable TCP_IPV6 mbuf hash delivery\n"); 1687 mbuf_types &= ~RSS_TYPE_TCP_IPV6; 1688 } 1689 if ((my_types & RSS_TYPE_TCP_IPV6_EX) && 1690 (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6)) { 1691 /* Conflict; disable TCP_IPV6_EX hash type/value delivery. */ 1692 if_printf(ifp, "disable TCP_IPV6_EX mbuf hash delivery\n"); 1693 mbuf_types &= ~RSS_TYPE_TCP_IPV6_EX; 1694 } 1695 if ((my_types & RSS_TYPE_UDP_IPV6) && 1696 (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6_EX)) { 1697 /* Conflict; disable UDP_IPV6 hash type/value delivery. */ 1698 if_printf(ifp, "disable UDP_IPV6 mbuf hash delivery\n"); 1699 mbuf_types &= ~RSS_TYPE_UDP_IPV6; 1700 } 1701 if ((my_types & RSS_TYPE_UDP_IPV6_EX) && 1702 (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6)) { 1703 /* Conflict; disable UDP_IPV6_EX hash type/value delivery. */ 1704 if_printf(ifp, "disable UDP_IPV6_EX mbuf hash delivery\n"); 1705 mbuf_types &= ~RSS_TYPE_UDP_IPV6_EX; 1706 } 1707 1708 /* 1709 * Indirect table does not matter. 1710 */ 1711 1712 sc->hn_rss_hash = (sc->hn_rss_hcap & NDIS_HASH_FUNCTION_MASK) | 1713 hn_rss_type_tondis(my_types); 1714 memcpy(sc->hn_rss.rss_key, ifrk.ifrk_key, sizeof(sc->hn_rss.rss_key)); 1715 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 1716 1717 if (reconf) { 1718 error = hn_rss_reconfig(sc); 1719 if (error) { 1720 /* XXX roll-back? */ 1721 if_printf(ifp, "hn_rss_reconfig failed: %d\n", error); 1722 mbuf_types = 0; 1723 } 1724 } 1725 done: 1726 /* Do not expose hashes unless the VF and synthetic settings agree. */ 1727 hn_rss_mbuf_hash(sc, hn_rss_type_tondis(mbuf_types)); 1728 } 1729 1730 static void 1731 hn_vf_rss_restore(struct hn_softc *sc) 1732 { 1733 1734 HN_LOCK_ASSERT(sc); 1735 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 1736 ("%s: synthetic parts are not attached", if_name(sc->hn_ifp))); 1737 1738 if (sc->hn_rx_ring_inuse == 1) 1739 goto done; 1740 1741 /* 1742 * Restore hash types. Key does _not_ matter. 1743 */ 1744 if (sc->hn_rss_hash != sc->hn_rss_hcap) { 1745 int error; 1746 1747 sc->hn_rss_hash = sc->hn_rss_hcap; 1748 error = hn_rss_reconfig(sc); 1749 if (error) { 1750 if_printf(sc->hn_ifp, "hn_rss_reconfig failed: %d\n", 1751 error); 1752 /* XXX keep going. */ 1753 } 1754 } 1755 done: 1756 /* Hash deliverability for mbufs. */ 1757 hn_rss_mbuf_hash(sc, NDIS_HASH_ALL); 1758 } 1759 1760 static void 1761 hn_xpnt_vf_setready(struct hn_softc *sc) 1762 { 1763 if_t ifp, vf_ifp; 1764 struct ifreq ifr; 1765 1766 HN_LOCK_ASSERT(sc); 1767 ifp = sc->hn_ifp; 1768 vf_ifp = sc->hn_vf_ifp; 1769 1770 /* 1771 * Mark the VF ready. 1772 */ 1773 sc->hn_vf_ready = true; 1774 1775 /* 1776 * Save information for restoration. 1777 */ 1778 sc->hn_saved_caps = if_getcapabilities(ifp); 1779 sc->hn_saved_tsomax = if_gethwtsomax(ifp); 1780 sc->hn_saved_tsosegcnt = if_gethwtsomaxsegcount(ifp); 1781 sc->hn_saved_tsosegsz = if_gethwtsomaxsegsize(ifp); 1782 sc->hn_saved_capenable = if_getcapenable(ifp); 1783 sc->hn_saved_hwassist = if_gethwassist(ifp); 1784 1785 /* 1786 * Expose only capabilities supported by the transparent packet path. 1787 * VF services such as send tags and the extended capability ioctl 1788 * require methods which hn does not implement. 1789 */ 1790 if_setcapabilities(ifp, if_getcapabilities(vf_ifp) & HN_XPNT_VF_CAPS); 1791 1792 /* 1793 * Fix TSO settings. 1794 */ 1795 if (if_gethwtsomax(ifp) > if_gethwtsomax(vf_ifp)) 1796 if_sethwtsomax(ifp, if_gethwtsomax(vf_ifp)); 1797 if (if_gethwtsomaxsegcount(ifp) > if_gethwtsomaxsegcount(vf_ifp)) 1798 if_sethwtsomaxsegcount(ifp, if_gethwtsomaxsegcount(vf_ifp)); 1799 if (if_gethwtsomaxsegsize(ifp) > if_gethwtsomaxsegsize(vf_ifp)) 1800 if_sethwtsomaxsegsize(ifp, if_gethwtsomaxsegsize(vf_ifp)); 1801 1802 /* 1803 * Adopt the VF's enabled capabilities without changing its settings. 1804 */ 1805 hn_xpnt_vf_synccaps(sc); 1806 1807 if (if_getmtu(ifp) != ETHERMTU) { 1808 int error; 1809 1810 /* 1811 * Change VF's MTU. 1812 */ 1813 memset(&ifr, 0, sizeof(ifr)); 1814 strlcpy(ifr.ifr_name, if_name(vf_ifp), sizeof(ifr.ifr_name)); 1815 ifr.ifr_mtu = if_getmtu(ifp); 1816 error = ifhwioctl(SIOCSIFMTU, vf_ifp, (caddr_t)&ifr, curthread); 1817 if (error) { 1818 if_printf(ifp, "%s SIOCSIFMTU %u failed\n", 1819 if_name(vf_ifp), if_getmtu(ifp)); 1820 if (if_getmtu(ifp) > ETHERMTU) { 1821 if_printf(ifp, "change MTU to %d\n", ETHERMTU); 1822 1823 /* 1824 * XXX 1825 * No need to adjust the synthetic parts' MTU; 1826 * failure of the adjustment will cause us 1827 * infinite headache. 1828 */ 1829 if_setmtu(ifp, ETHERMTU); 1830 hn_mtu_change_fixup(sc); 1831 } 1832 } 1833 } 1834 } 1835 1836 static bool 1837 hn_xpnt_vf_isready(struct hn_softc *sc) 1838 { 1839 u_int assoc; 1840 1841 HN_LOCK_ASSERT(sc); 1842 1843 assoc = atomic_load_acq_int(&sc->hn_vf_assoc); 1844 return (hn_xpnt_vf && sc->hn_vf_ifp != NULL && sc->hn_vf_ready && 1845 (assoc & HN_VF_ASSOC_ALLOCATED) && 1846 sc->hn_vf_active_assoc == assoc); 1847 } 1848 1849 static bool 1850 hn_xpnt_vf_caninit(struct hn_softc *sc) 1851 { 1852 1853 HN_LOCK_ASSERT(sc); 1854 return (hn_xpnt_vf && sc->hn_vf_ifp != NULL && 1855 !sc->hn_detaching && !sc->hn_vf_detaching && 1856 !sc->hn_vf_caps_busy && 1857 (atomic_load_acq_int(&sc->hn_vf_assoc) & HN_VF_ASSOC_ALLOCATED) && 1858 (int)(ticks - sc->hn_vf_rdytick) >= 0); 1859 } 1860 1861 static void 1862 hn_xpnt_vf_restore(struct hn_softc *sc) 1863 { 1864 if_t ifp = sc->hn_ifp; 1865 1866 HN_LOCK_ASSERT(sc); 1867 if (!sc->hn_vf_ready) 1868 return; 1869 if_setcapabilities(ifp, sc->hn_saved_caps); 1870 if_sethwtsomax(ifp, sc->hn_saved_tsomax); 1871 if_sethwtsomaxsegcount(ifp, sc->hn_saved_tsosegcnt); 1872 if_sethwtsomaxsegsize(ifp, sc->hn_saved_tsosegsz); 1873 if_setcapenable(ifp, sc->hn_saved_capenable); 1874 if_sethwassist(ifp, sc->hn_saved_hwassist); 1875 if (!sc->hn_detaching) 1876 taskqueue_enqueue(sc->hn_vf_taskq, &sc->hn_vf_vlancap); 1877 sc->hn_vf_ready = false; 1878 } 1879 1880 static void 1881 hn_xpnt_vf_deactivate(struct hn_softc *sc) 1882 { 1883 1884 HN_LOCK_ASSERT(sc); 1885 /* Do not send VF-formatted packets through synthetic during handoff. */ 1886 rm_wlock(&sc->hn_vf_lock); 1887 sc->hn_xvf_flags |= HN_XVFFLAG_SWITCHING; 1888 rm_wunlock(&sc->hn_vf_lock); 1889 hn_xpnt_vf_setdisable(sc, false); 1890 if (sc->hn_vf_active_assoc != 0) { 1891 /* Close local VF transmit even if returning to synthetic fails. */ 1892 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_SYNTH); 1893 sc->hn_vf_active_assoc = 0; 1894 hn_vf_rss_restore(sc); 1895 } 1896 hn_xpnt_vf_restore(sc); 1897 rm_wlock(&sc->hn_vf_lock); 1898 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1899 rm_wunlock(&sc->hn_vf_lock); 1900 hn_resume_mgmt(sc); 1901 } 1902 1903 static bool 1904 hn_xpnt_vf_setenable(struct hn_softc *sc) 1905 { 1906 int i; 1907 1908 HN_LOCK_ASSERT(sc); 1909 1910 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 1911 rm_wlock(&sc->hn_vf_lock); 1912 if (sc->hn_vf_active_assoc != atomic_load_acq_int(&sc->hn_vf_assoc)) { 1913 rm_wunlock(&sc->hn_vf_lock); 1914 return (false); 1915 } 1916 sc->hn_xvf_flags |= HN_XVFFLAG_ENABLED; 1917 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1918 rm_wunlock(&sc->hn_vf_lock); 1919 1920 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1921 sc->hn_rx_ring[i].hn_rx_flags |= HN_RX_FLAG_XPNT_VF; 1922 return (true); 1923 } 1924 1925 static void 1926 hn_xpnt_vf_setdisable(struct hn_softc *sc, bool clear_vf) 1927 { 1928 int i; 1929 1930 HN_LOCK_ASSERT(sc); 1931 1932 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 1933 rm_wlock(&sc->hn_vf_lock); 1934 sc->hn_xvf_flags &= ~HN_XVFFLAG_ENABLED; 1935 if (clear_vf) { 1936 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1937 sc->hn_vf_ifp = NULL; 1938 } 1939 rm_wunlock(&sc->hn_vf_lock); 1940 1941 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1942 sc->hn_rx_ring[i].hn_rx_flags &= ~HN_RX_FLAG_XPNT_VF; 1943 } 1944 1945 /* 1946 * Do not configure the VF from the VLAN event callback. The worker reads 1947 * and applies the current VLAN topology outside the VLAN configuration 1948 * lock, including VLANs configured before the VF arrives. Only 1949 * transparent mode subscribes. 1950 */ 1951 static void 1952 hn_vlan_event(void *xsc, if_t ifp, uint16_t vid __unused) 1953 { 1954 struct hn_softc *sc = xsc; 1955 1956 if (ifp != sc->hn_ifp) 1957 return; 1958 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 1959 } 1960 1961 /* Apply guest VLAN intent, not the host's administrative access VLAN. */ 1962 static void 1963 hn_xpnt_vf_sync_vlans(struct hn_softc *sc, bool remove) 1964 { 1965 struct epoch_tracker et; 1966 u_int desired[HN_VLAN_WORDS] = { 0 }, changed, mask; 1967 unsigned int i, bit; 1968 uint16_t vid; 1969 1970 HN_LOCK_ASSERT(sc); 1971 KASSERT(sc->hn_vf_ifp != NULL, ("VLAN sync without a VF")); 1972 if (!remove) { 1973 NET_EPOCH_ENTER(et); 1974 for (vid = 1; vid < EVL_VLID_MASK; vid++) { 1975 if (VLAN_DEVAT(sc->hn_ifp, vid) != NULL) 1976 desired[vid / 32] |= 1U << (vid % 32); 1977 } 1978 NET_EPOCH_EXIT(et); 1979 } 1980 /* VF callbacks may sleep; never invoke them inside network epoch. */ 1981 for (i = 0; i < HN_VLAN_WORDS; i++) { 1982 changed = desired[i] ^ sc->hn_vf_vlans[i]; 1983 while (changed != 0) { 1984 bit = ffs(changed) - 1; 1985 mask = 1U << bit; 1986 vid = i * 32 + bit; 1987 if ((desired[i] & mask) != 0) 1988 EVENTHANDLER_INVOKE(vlan_config, sc->hn_vf_ifp, 1989 vid); 1990 else 1991 EVENTHANDLER_INVOKE(vlan_unconfig, sc->hn_vf_ifp, 1992 vid); 1993 changed &= ~mask; 1994 } 1995 sc->hn_vf_vlans[i] = desired[i]; 1996 } 1997 } 1998 1999 static void 2000 hn_xpnt_vf_init(struct hn_softc *sc) 2001 { 2002 int error; 2003 u_int assoc; 2004 2005 HN_LOCK_ASSERT(sc); 2006 2007 KASSERT((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) == 0, 2008 ("%s: transparent VF was enabled", if_name(sc->hn_ifp))); 2009 if (!hn_xpnt_vf_caninit(sc)) 2010 return; 2011 hn_xpnt_vf_sync_vlans(sc, false); 2012 assoc = atomic_load_acq_int(&sc->hn_vf_assoc); 2013 rm_wlock(&sc->hn_vf_lock); 2014 sc->hn_xvf_flags |= HN_XVFFLAG_SWITCHING; 2015 rm_wunlock(&sc->hn_vf_lock); 2016 if (!sc->hn_vf_ready) 2017 hn_xpnt_vf_setready(sc); 2018 2019 if (bootverbose) { 2020 if_printf(sc->hn_ifp, "try bringing up %s\n", 2021 if_name(sc->hn_vf_ifp)); 2022 } 2023 2024 /* 2025 * Bring the VF up. 2026 */ 2027 hn_xpnt_vf_saveifflags(sc); 2028 if_setflagbits(sc->hn_ifp, IFF_UP, 0); 2029 error = hn_xpnt_vf_iocsetflags(sc); 2030 if (error) { 2031 if_printf(sc->hn_ifp, "bringing up %s failed: %d\n", 2032 if_name(sc->hn_vf_ifp), error); 2033 goto failed; 2034 } 2035 2036 /* 2037 * Some VF drivers initialize hwassist only when brought up. Refresh 2038 * the offload state before allowing transmit through the VF. 2039 */ 2040 hn_xpnt_vf_synccaps(sc); 2041 2042 /* 2043 * NOTE: 2044 * Datapath setting must happen _after_ bringing the VF up. 2045 */ 2046 error = hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_VF); 2047 if (error) 2048 goto failed; 2049 sc->hn_vf_active_assoc = assoc; 2050 hn_suspend_mgmt(sc); 2051 2052 /* 2053 * NOTE: 2054 * Fixup RSS related bits _after_ the VF is brought up, since 2055 * many VFs generate RSS key during it's initialization. 2056 */ 2057 hn_vf_rss_fixup(sc, true); 2058 2059 /* Mark transparent mode VF as enabled. */ 2060 if (!hn_xpnt_vf_setenable(sc)) { 2061 error = EAGAIN; 2062 goto failed; 2063 } 2064 if_link_state_change(sc->hn_ifp, if_getlinkstate(sc->hn_vf_ifp)); 2065 return; 2066 2067 failed: 2068 hn_xpnt_vf_deactivate(sc); 2069 if (error == EAGAIN) 2070 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, hz); 2071 } 2072 2073 static void 2074 hn_xpnt_vf_init_taskfunc(void *xsc, int pending __unused) 2075 { 2076 struct hn_softc *sc = xsc; 2077 2078 HN_LOCK(sc); 2079 if (sc->hn_detaching) 2080 goto done; 2081 2082 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 2083 goto done; 2084 if (sc->hn_vf_ifp == NULL || sc->hn_vf_detaching || 2085 sc->hn_vf_caps_busy) 2086 goto done; 2087 if (!hn_xpnt_vf) { 2088 if ((sc->hn_flags & HN_FLAG_RXVF) && sc->hn_vf_active_assoc != 2089 atomic_load_acq_int(&sc->hn_vf_assoc)) 2090 hn_rxvf_change_locked(sc, sc->hn_vf_ifp, false); 2091 hn_rxvf_change_locked(sc, sc->hn_vf_ifp, 2092 (if_getflags(sc->hn_vf_ifp) & IFF_UP) != 0); 2093 goto rss; 2094 } 2095 if (sc->hn_vf_active_assoc != 0 && sc->hn_vf_active_assoc != 2096 atomic_load_acq_int(&sc->hn_vf_assoc)) 2097 hn_xpnt_vf_deactivate(sc); 2098 if ((int)(ticks - sc->hn_vf_rdytick) < 0) { 2099 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 2100 sc->hn_vf_rdytick - ticks); 2101 goto done; 2102 } 2103 hn_xpnt_vf_sync_vlans(sc, false); 2104 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 2105 goto rss; 2106 2107 if (if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) { 2108 /* 2109 * Delayed VF initialization. 2110 */ 2111 if (bootverbose) { 2112 if_printf(sc->hn_ifp, "delayed initialize %s\n", 2113 if_name(sc->hn_vf_ifp)); 2114 } 2115 hn_xpnt_vf_init(sc); 2116 } 2117 rss: 2118 /* 2119 * A link-up event can follow recovery from a failed handoff RSS query. 2120 * An inactive path can consume this request: its next handoff queries 2121 * RSS again. This is a one-shot refresh, not a readiness poll. 2122 */ 2123 if (atomic_readandclear_int(&sc->hn_vf_rss_refresh) != 0 && 2124 ((sc->hn_flags & HN_FLAG_RXVF) || 2125 (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) && 2126 (sc->hn_vf_active_assoc & HN_VF_ASSOC_ALLOCATED) != 0 && 2127 sc->hn_vf_active_assoc == atomic_load_acq_int(&sc->hn_vf_assoc) && 2128 (if_getflags(sc->hn_vf_ifp) & IFF_UP) != 0 && 2129 if_getlinkstate(sc->hn_vf_ifp) == LINK_STATE_UP) 2130 hn_vf_rss_fixup(sc, true); 2131 done: 2132 HN_UNLOCK(sc); 2133 } 2134 2135 static void 2136 hn_ifnet_attevent(void *xsc, if_t ifp) 2137 { 2138 struct hn_softc *sc = xsc; 2139 2140 HN_LOCK(sc); 2141 2142 if (!(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 2143 goto done; 2144 2145 if (!hn_ismyvf(sc, ifp)) 2146 goto done; 2147 2148 if (sc->hn_vf_ifp != NULL) { 2149 if_printf(sc->hn_ifp, "%s was attached as VF\n", 2150 if_name(sc->hn_vf_ifp)); 2151 goto done; 2152 } 2153 2154 if (hn_xpnt_vf && if_getstartfn(ifp) != NULL) { 2155 /* 2156 * ifnet.if_start is _not_ supported by transparent 2157 * mode VF; mainly due to the IFF_DRV_OACTIVE flag. 2158 */ 2159 if_printf(sc->hn_ifp, "%s uses if_start, which is unsupported " 2160 "in transparent VF mode.\n", if_name(sc->hn_vf_ifp)); 2161 2162 goto done; 2163 } 2164 2165 rm_wlock(&hn_vfmap_lock); 2166 2167 if (if_getindex(ifp) >= hn_vfmap_size) { 2168 if_t *newmap; 2169 int newsize; 2170 2171 newsize = if_getindex(ifp) + HN_VFMAP_SIZE_DEF; 2172 newmap = malloc(sizeof(if_t) * newsize, M_DEVBUF, 2173 M_WAITOK | M_ZERO); 2174 2175 memcpy(newmap, hn_vfmap, 2176 sizeof(if_t) * hn_vfmap_size); 2177 free(hn_vfmap, M_DEVBUF); 2178 hn_vfmap = newmap; 2179 hn_vfmap_size = newsize; 2180 } 2181 KASSERT(hn_vfmap[if_getindex(ifp)] == NULL, 2182 ("%s: ifindex %d was mapped to %s", 2183 if_name(ifp), if_getindex(ifp), if_name(hn_vfmap[if_getindex(ifp)]))); 2184 hn_vfmap[if_getindex(ifp)] = sc->hn_ifp; 2185 2186 rm_wunlock(&hn_vfmap_lock); 2187 2188 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 2189 rm_wlock(&sc->hn_vf_lock); 2190 KASSERT((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) == 0, 2191 ("%s: transparent VF was enabled", if_name(sc->hn_ifp))); 2192 sc->hn_vf_ifp = ifp; 2193 rm_wunlock(&sc->hn_vf_lock); 2194 sc->hn_vf_detaching = false; 2195 2196 if (hn_xpnt_vf) { 2197 int wait_ticks; 2198 2199 /* 2200 * Install if_input for vf_ifp, which does vf_ifp -> hn_ifp. 2201 * Save vf_ifp's current if_input for later restoration. 2202 */ 2203 sc->hn_vf_input = if_getinputfn(ifp); 2204 if_setinputfn(ifp, hn_xpnt_vf_input); 2205 2206 /* 2207 * Give VF sometime to complete its attach routing. 2208 */ 2209 wait_ticks = hn_xpnt_vf_attwait * hz; 2210 sc->hn_vf_rdytick = ticks + wait_ticks; 2211 2212 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 2213 wait_ticks); 2214 } else { 2215 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 2216 } 2217 done: 2218 HN_UNLOCK(sc); 2219 } 2220 2221 static void 2222 hn_ifnet_detevent(void *xsc, if_t ifp) 2223 { 2224 struct hn_softc *sc = xsc; 2225 2226 HN_LOCK(sc); 2227 2228 if (sc->hn_vf_ifp == NULL) 2229 goto done; 2230 2231 if (!hn_ismyvf(sc, ifp)) 2232 goto done; 2233 2234 if (hn_xpnt_vf) { 2235 sc->hn_vf_detaching = true; 2236 while (sc->hn_vf_caps_busy) 2237 sx_sleep(&sc->hn_vf_caps_busy, &sc->hn_lock, 0, 2238 "hnvfcap", 0); 2239 /* 2240 * Make sure that the delayed initialization is not running. 2241 * 2242 * NOTE: 2243 * - This lock _must_ be released, since the hn_vf_init task 2244 * will try holding this lock. 2245 * - It is safe to release this lock here, since the 2246 * hn_ifnet_attevent() is interlocked by the hn_vf_ifp. 2247 * 2248 * XXX racy, if hn(4) ever detached. 2249 */ 2250 HN_UNLOCK(sc); 2251 taskqueue_drain_timeout(sc->hn_vf_taskq, &sc->hn_vf_init); 2252 HN_LOCK(sc); 2253 2254 KASSERT(sc->hn_vf_input != NULL, ("%s VF input is not saved", 2255 if_name(sc->hn_ifp))); 2256 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) 2257 hn_xpnt_vf_deactivate(sc); 2258 else 2259 hn_xpnt_vf_restore(sc); 2260 /* A departing VF discards its registration state itself. */ 2261 bzero(sc->hn_vf_vlans, sizeof(sc->hn_vf_vlans)); 2262 if_setinputfn(ifp, sc->hn_vf_input); 2263 sc->hn_vf_input = NULL; 2264 } else if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 2265 hn_rxvf_change_locked(sc, ifp, false); 2266 /* Never leave receive routing pointing at a departing VF. */ 2267 if (sc->hn_flags & HN_FLAG_RXVF) { 2268 sc->hn_flags &= ~HN_FLAG_RXVF; 2269 sc->hn_vf_active_assoc = 0; 2270 hn_rxvf_set(sc, NULL); 2271 hn_vf_rss_restore(sc); 2272 hn_resume_mgmt(sc); 2273 } 2274 } 2275 2276 /* Mark transparent mode VF as disabled. */ 2277 hn_xpnt_vf_setdisable(sc, true /* clear hn_vf_ifp */); 2278 2279 rm_wlock(&hn_vfmap_lock); 2280 2281 KASSERT(if_getindex(ifp) < hn_vfmap_size, 2282 ("ifindex %d, vfmapsize %d", if_getindex(ifp), hn_vfmap_size)); 2283 if (hn_vfmap[if_getindex(ifp)] != NULL) { 2284 KASSERT(hn_vfmap[if_getindex(ifp)] == sc->hn_ifp, 2285 ("%s: ifindex %d was mapped to %s", 2286 if_name(ifp), if_getindex(ifp), 2287 if_name(hn_vfmap[if_getindex(ifp)]))); 2288 hn_vfmap[if_getindex(ifp)] = NULL; 2289 } 2290 2291 rm_wunlock(&hn_vfmap_lock); 2292 done: 2293 HN_UNLOCK(sc); 2294 } 2295 2296 static void 2297 hn_ifnet_lnkevent(void *xsc, if_t ifp, int link_state) 2298 { 2299 struct hn_softc *sc = xsc; 2300 struct rm_priotracker pt; 2301 2302 /* Publish before a concurrent handoff can restore synthetic carrier. */ 2303 rm_rlock(&sc->hn_vf_lock, &pt); 2304 if (sc->hn_vf_ifp != ifp) 2305 goto out; 2306 if (link_state == LINK_STATE_UP) { 2307 /* RSS queries and host reconfiguration require sleepable context. */ 2308 atomic_store_rel_int(&sc->hn_vf_rss_refresh, 1); 2309 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 2310 } 2311 if ((sc->hn_xvf_flags & (HN_XVFFLAG_ENABLED | HN_XVFFLAG_SWITCHING)) == 2312 HN_XVFFLAG_ENABLED && sc->hn_vf_active_assoc == 2313 atomic_load_acq_int(&sc->hn_vf_assoc)) 2314 if_link_state_change(sc->hn_ifp, link_state); 2315 out: 2316 rm_runlock(&sc->hn_vf_lock, &pt); 2317 } 2318 2319 static int 2320 hn_tsomax_sysctl(SYSCTL_HANDLER_ARGS) 2321 { 2322 struct hn_softc *sc = arg1; 2323 unsigned int tsomax; 2324 int error; 2325 2326 tsomax = if_gethwtsomax(sc->hn_ifp); 2327 error = sysctl_handle_int(oidp, &tsomax, 0, req); 2328 return error; 2329 } 2330 2331 static int 2332 hn_tsomaxsegcnt_sysctl(SYSCTL_HANDLER_ARGS) 2333 { 2334 struct hn_softc *sc = arg1; 2335 unsigned int tsomaxsegcnt; 2336 int error; 2337 2338 tsomaxsegcnt = if_gethwtsomaxsegcount(sc->hn_ifp); 2339 error = sysctl_handle_int(oidp, &tsomaxsegcnt, 0, req); 2340 return error; 2341 } 2342 2343 static int 2344 hn_tsomaxsegsz_sysctl(SYSCTL_HANDLER_ARGS) 2345 { 2346 struct hn_softc *sc = arg1; 2347 unsigned int tsomaxsegsz; 2348 int error; 2349 2350 tsomaxsegsz = if_gethwtsomaxsegsize(sc->hn_ifp); 2351 error = sysctl_handle_int(oidp, &tsomaxsegsz, 0, req); 2352 return error; 2353 } 2354 2355 static int 2356 hn_probe(device_t dev) 2357 { 2358 2359 if (VMBUS_PROBE_GUID(device_get_parent(dev), dev, &hn_guid) == 0) { 2360 device_set_desc(dev, "Hyper-V Network Interface"); 2361 return BUS_PROBE_DEFAULT; 2362 } 2363 return ENXIO; 2364 } 2365 2366 static int 2367 hn_attach(device_t dev) 2368 { 2369 struct hn_softc *sc = device_get_softc(dev); 2370 struct sysctl_oid_list *child; 2371 struct sysctl_ctx_list *ctx; 2372 uint8_t eaddr[ETHER_ADDR_LEN]; 2373 if_t ifp = NULL; 2374 int error, ring_cnt, tx_ring_cnt; 2375 uint32_t mtu; 2376 2377 sc->hn_dev = dev; 2378 sc->hn_prichan = vmbus_get_channel(dev); 2379 HN_LOCK_INIT(sc); 2380 rm_init(&sc->hn_vf_lock, "hnvf"); 2381 if (hn_xpnt_vf && hn_xpnt_vf_accbpf) 2382 sc->hn_xvf_flags |= HN_XVFFLAG_ACCBPF; 2383 2384 /* 2385 * Initialize these tunables once. 2386 */ 2387 sc->hn_agg_size = hn_tx_agg_size; 2388 sc->hn_agg_pkts = hn_tx_agg_pkts; 2389 2390 /* 2391 * Setup taskqueue for transmission. 2392 */ 2393 if (hn_tx_taskq_mode == HN_TX_TASKQ_M_INDEP) { 2394 int i; 2395 2396 sc->hn_tx_taskqs = 2397 malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), 2398 M_DEVBUF, M_WAITOK); 2399 for (i = 0; i < hn_tx_taskq_cnt; ++i) { 2400 sc->hn_tx_taskqs[i] = taskqueue_create("hn_tx", 2401 M_WAITOK, taskqueue_thread_enqueue, 2402 &sc->hn_tx_taskqs[i]); 2403 taskqueue_start_threads(&sc->hn_tx_taskqs[i], 1, PI_NET, 2404 "%s tx%d", device_get_nameunit(dev), i); 2405 } 2406 } else if (hn_tx_taskq_mode == HN_TX_TASKQ_M_GLOBAL) { 2407 sc->hn_tx_taskqs = hn_tx_taskque; 2408 } 2409 2410 /* 2411 * Setup taskqueue for mangement tasks, e.g. link status. 2412 */ 2413 sc->hn_mgmt_taskq0 = taskqueue_create("hn_mgmt", M_WAITOK, 2414 taskqueue_thread_enqueue, &sc->hn_mgmt_taskq0); 2415 taskqueue_start_threads(&sc->hn_mgmt_taskq0, 1, PI_NET, "%s mgmt", 2416 device_get_nameunit(dev)); 2417 TASK_INIT(&sc->hn_link_task, 0, hn_link_taskfunc, sc); 2418 TASK_INIT(&sc->hn_netchg_init, 0, hn_netchg_init_taskfunc, sc); 2419 TIMEOUT_TASK_INIT(sc->hn_mgmt_taskq0, &sc->hn_netchg_status, 0, 2420 hn_netchg_status_taskfunc, sc); 2421 2422 /* Association work must not block the channel delivering completions. */ 2423 sc->hn_vf_taskq = taskqueue_create("hn_vf", M_WAITOK, 2424 taskqueue_thread_enqueue, &sc->hn_vf_taskq); 2425 taskqueue_start_threads(&sc->hn_vf_taskq, 1, PI_NET, "%s vf", 2426 device_get_nameunit(dev)); 2427 TIMEOUT_TASK_INIT(sc->hn_vf_taskq, &sc->hn_vf_init, 0, 2428 hn_xpnt_vf_init_taskfunc, sc); 2429 TASK_INIT(&sc->hn_vf_vlancap, 0, hn_xpnt_vf_vlancap_taskfunc, sc); 2430 2431 /* 2432 * Allocate ifnet and setup its name earlier, so that if_printf 2433 * can be used by functions, which will be called after 2434 * ether_ifattach(). 2435 */ 2436 ifp = sc->hn_ifp = if_alloc(IFT_ETHER); 2437 if_setsoftc(ifp, sc); 2438 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2439 2440 /* 2441 * Initialize ifmedia earlier so that it can be unconditionally 2442 * destroyed, if error happened later on. 2443 */ 2444 ifmedia_init(&sc->hn_media, 0, hn_ifmedia_upd, hn_ifmedia_sts); 2445 2446 /* 2447 * Figure out the # of RX rings (ring_cnt) and the # of TX rings 2448 * to use (tx_ring_cnt). 2449 * 2450 * NOTE: 2451 * The # of RX rings to use is same as the # of channels to use. 2452 */ 2453 ring_cnt = hn_chan_cnt; 2454 if (ring_cnt <= 0) { 2455 /* Default */ 2456 ring_cnt = mp_ncpus; 2457 if (ring_cnt > HN_RING_CNT_DEF_MAX) 2458 ring_cnt = HN_RING_CNT_DEF_MAX; 2459 } else if (ring_cnt > mp_ncpus) { 2460 ring_cnt = mp_ncpus; 2461 } 2462 #ifdef RSS 2463 if (ring_cnt > rss_getnumbuckets()) 2464 ring_cnt = rss_getnumbuckets(); 2465 #endif 2466 2467 tx_ring_cnt = hn_tx_ring_cnt; 2468 if (tx_ring_cnt <= 0 || tx_ring_cnt > ring_cnt) 2469 tx_ring_cnt = ring_cnt; 2470 #ifdef HN_IFSTART_SUPPORT 2471 if (hn_use_if_start) { 2472 /* ifnet.if_start only needs one TX ring. */ 2473 tx_ring_cnt = 1; 2474 } 2475 #endif 2476 2477 /* 2478 * Set the leader CPU for channels. 2479 */ 2480 sc->hn_cpu = atomic_fetchadd_int(&hn_cpu_index, ring_cnt) % mp_ncpus; 2481 2482 /* 2483 * Create enough TX/RX rings, even if only limited number of 2484 * channels can be allocated. 2485 */ 2486 error = hn_create_tx_data(sc, tx_ring_cnt); 2487 if (error) 2488 goto failed; 2489 error = hn_create_rx_data(sc, ring_cnt); 2490 if (error) 2491 goto failed; 2492 2493 /* 2494 * Create transaction context for NVS and RNDIS transactions. 2495 */ 2496 sc->hn_xact = vmbus_xact_ctx_create(bus_get_dma_tag(dev), 2497 HN_XACT_REQ_SIZE, HN_XACT_RESP_SIZE, 0); 2498 if (sc->hn_xact == NULL) { 2499 error = ENXIO; 2500 goto failed; 2501 } 2502 2503 /* 2504 * Install orphan handler for the revocation of this device's 2505 * primary channel. 2506 * 2507 * NOTE: 2508 * The processing order is critical here: 2509 * Install the orphan handler, _before_ testing whether this 2510 * device's primary channel has been revoked or not. 2511 */ 2512 vmbus_chan_set_orphan(sc->hn_prichan, sc->hn_xact); 2513 if (vmbus_chan_is_revoked(sc->hn_prichan)) { 2514 error = ENXIO; 2515 goto failed; 2516 } 2517 2518 /* 2519 * Attach the synthetic parts, i.e. NVS and RNDIS. 2520 */ 2521 error = hn_synth_attach(sc, ETHERMTU); 2522 if (error) 2523 goto failed; 2524 2525 error = hn_rndis_get_eaddr(sc, eaddr); 2526 if (error) 2527 goto failed; 2528 2529 error = hn_rndis_get_mtu(sc, &mtu); 2530 if (error) 2531 mtu = ETHERMTU; 2532 else if (bootverbose) 2533 device_printf(dev, "RNDIS mtu %u\n", mtu); 2534 2535 if (sc->hn_rx_ring_inuse > 1) { 2536 /* 2537 * Reduce TCP segment aggregation limit for multiple 2538 * RX rings to increase ACK timeliness. 2539 */ 2540 hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MULTIRX_DEF); 2541 } 2542 2543 /* 2544 * Fixup TX/RX stuffs after synthetic parts are attached. 2545 */ 2546 hn_fixup_tx_data(sc); 2547 hn_fixup_rx_data(sc); 2548 2549 ctx = device_get_sysctl_ctx(dev); 2550 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 2551 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "nvs_version", CTLFLAG_RD, 2552 &sc->hn_nvs_ver, 0, "NVS version"); 2553 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "ndis_version", 2554 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2555 hn_ndis_version_sysctl, "A", "NDIS version"); 2556 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "caps", 2557 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2558 hn_caps_sysctl, "A", "capabilities"); 2559 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "hwassist", 2560 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2561 hn_hwassist_sysctl, "A", "hwassist"); 2562 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_max", 2563 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomax_sysctl, 2564 "IU", "max TSO size"); 2565 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_maxsegcnt", 2566 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomaxsegcnt_sysctl, 2567 "IU", "max # of TSO segments"); 2568 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_maxsegsz", 2569 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomaxsegsz_sysctl, 2570 "IU", "max size of TSO segment"); 2571 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxfilter", 2572 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2573 hn_rxfilter_sysctl, "A", "rxfilter"); 2574 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hash", 2575 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2576 hn_rss_hash_sysctl, "A", "RSS hash"); 2577 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hashcap", 2578 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2579 hn_rss_hcap_sysctl, "A", "RSS hash capabilities"); 2580 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "mbuf_hash", 2581 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2582 hn_rss_mbuf_sysctl, "A", "RSS hash for mbufs"); 2583 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rss_ind_size", 2584 CTLFLAG_RD, &sc->hn_rss_ind_size, 0, "RSS indirect entry count"); 2585 #ifndef RSS 2586 /* 2587 * Don't allow RSS key/indirect table changes, if RSS is defined. 2588 */ 2589 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_key", 2590 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2591 hn_rss_key_sysctl, "IU", "RSS key"); 2592 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_ind", 2593 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2594 hn_rss_ind_sysctl, "IU", "RSS indirect table"); 2595 #endif 2596 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_size", 2597 CTLFLAG_RD, &sc->hn_rndis_agg_size, 0, 2598 "RNDIS offered packet transmission aggregation size limit"); 2599 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_pkts", 2600 CTLFLAG_RD, &sc->hn_rndis_agg_pkts, 0, 2601 "RNDIS offered packet transmission aggregation count limit"); 2602 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_align", 2603 CTLFLAG_RD, &sc->hn_rndis_agg_align, 0, 2604 "RNDIS packet transmission aggregation alignment"); 2605 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_size", 2606 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2607 hn_txagg_size_sysctl, "I", 2608 "Packet transmission aggregation size, 0 -- disable, -1 -- auto"); 2609 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_pkts", 2610 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2611 hn_txagg_pkts_sysctl, "I", 2612 "Packet transmission aggregation packets, " 2613 "0 -- disable, -1 -- auto"); 2614 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "polling", 2615 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2616 hn_polling_sysctl, "I", 2617 "Polling frequency: [100,1000000], 0 disable polling"); 2618 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf", 2619 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2620 hn_vf_sysctl, "A", "Virtual Function's name"); 2621 if (!hn_xpnt_vf) { 2622 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxvf", 2623 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2624 hn_rxvf_sysctl, "A", "activated Virtual Function's name"); 2625 } else { 2626 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf_xpnt_enabled", 2627 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2628 hn_xpnt_vf_enabled_sysctl, "I", 2629 "Transparent VF enabled"); 2630 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf_xpnt_accbpf", 2631 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2632 hn_xpnt_vf_accbpf_sysctl, "I", 2633 "Accurate BPF for transparent VF"); 2634 } 2635 2636 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rsc_switch", 2637 CTLTYPE_UINT | CTLFLAG_RW, sc, 0, hn_rsc_sysctl, "I", 2638 "switch to rsc"); 2639 2640 /* 2641 * Setup the ifmedia, which has been initialized earlier. 2642 */ 2643 ifmedia_add(&sc->hn_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2644 ifmedia_set(&sc->hn_media, IFM_ETHER | IFM_AUTO); 2645 /* XXX ifmedia_set really should do this for us */ 2646 sc->hn_media.ifm_media = sc->hn_media.ifm_cur->ifm_media; 2647 2648 /* 2649 * Setup the ifnet for this interface. 2650 */ 2651 2652 if_setbaudrate(ifp, IF_Gbps(10)); 2653 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2654 if_setioctlfn(ifp, hn_ioctl); 2655 if_setinitfn(ifp, hn_init); 2656 #ifdef HN_IFSTART_SUPPORT 2657 if (hn_use_if_start) { 2658 int qdepth = hn_get_txswq_depth(&sc->hn_tx_ring[0]); 2659 2660 if_setstartfn(ifp, hn_start); 2661 if_setsendqlen(ifp, qdepth); 2662 if_setsendqready(ifp); 2663 } else 2664 #endif 2665 { 2666 if_settransmitfn(ifp, hn_transmit); 2667 if_setqflushfn(ifp, hn_xmit_qflush); 2668 } 2669 2670 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM | IFCAP_LRO | IFCAP_LINKSTATE, 0); 2671 #ifdef foo 2672 /* We can't diff IPv6 packets from IPv4 packets on RX path. */ 2673 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM_IPV6, 0); 2674 #endif 2675 if (sc->hn_caps & HN_CAP_VLAN) { 2676 /* XXX not sure about VLAN_MTU. */ 2677 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); 2678 } 2679 2680 if_sethwassist(ifp, sc->hn_tx_ring[0].hn_csum_assist); 2681 if (if_gethwassist(ifp) & HN_CSUM_IP_MASK) 2682 if_setcapabilitiesbit(ifp, IFCAP_TXCSUM, 0); 2683 if (if_gethwassist(ifp) & HN_CSUM_IP6_MASK) 2684 if_setcapabilitiesbit(ifp, IFCAP_TXCSUM_IPV6, 0); 2685 if (sc->hn_caps & HN_CAP_TSO4) { 2686 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 2687 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 2688 } 2689 if (sc->hn_caps & HN_CAP_TSO6) { 2690 if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0); 2691 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 2692 } 2693 2694 /* Enable all available capabilities by default. */ 2695 if_setcapenable(ifp, if_getcapabilities(ifp)); 2696 2697 /* 2698 * Disable IPv6 TSO and TXCSUM by default, they still can 2699 * be enabled through SIOCSIFCAP. 2700 */ 2701 if_setcapenablebit(ifp, 0, (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6)); 2702 if_sethwassistbits(ifp, 0, (HN_CSUM_IP6_MASK | CSUM_IP6_TSO)); 2703 2704 if (if_getcapabilities(ifp) & (IFCAP_TSO6 | IFCAP_TSO4)) { 2705 /* 2706 * Lock hn_set_tso_maxsize() to simplify its 2707 * internal logic. 2708 */ 2709 HN_LOCK(sc); 2710 hn_set_tso_maxsize(sc, hn_tso_maxlen, ETHERMTU); 2711 HN_UNLOCK(sc); 2712 if_sethwtsomaxsegcount(ifp, HN_TX_DATA_SEGCNT_MAX); 2713 if_sethwtsomaxsegsize(ifp, PAGE_SIZE); 2714 } 2715 2716 /* Schedule reconciliation when the synthetic interface's VLANs change. */ 2717 if (hn_xpnt_vf) { 2718 sc->hn_vlan_atthand = EVENTHANDLER_REGISTER(vlan_config, 2719 hn_vlan_event, sc, EVENTHANDLER_PRI_ANY); 2720 sc->hn_vlan_dethand = EVENTHANDLER_REGISTER(vlan_unconfig, 2721 hn_vlan_event, sc, EVENTHANDLER_PRI_ANY); 2722 } 2723 ether_ifattach(ifp, eaddr); 2724 2725 if ((if_getcapabilities(ifp) & (IFCAP_TSO6 | IFCAP_TSO4)) && bootverbose) { 2726 if_printf(ifp, "TSO segcnt %u segsz %u\n", 2727 if_gethwtsomaxsegcount(ifp), if_gethwtsomaxsegsize(ifp)); 2728 } 2729 if (mtu < ETHERMTU) { 2730 2731 if_setmtu(ifp, mtu); 2732 } 2733 2734 /* Inform the upper layer about the long frame support. */ 2735 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2736 2737 /* 2738 * Kick off link status check. 2739 */ 2740 sc->hn_mgmt_taskq = sc->hn_mgmt_taskq0; 2741 hn_update_link_status(sc); 2742 2743 if (!hn_xpnt_vf) { 2744 sc->hn_ifnet_evthand = EVENTHANDLER_REGISTER(ifnet_event, 2745 hn_ifnet_event, sc, EVENTHANDLER_PRI_ANY); 2746 sc->hn_ifaddr_evthand = EVENTHANDLER_REGISTER(ifaddr_event, 2747 hn_ifaddr_event, sc, EVENTHANDLER_PRI_ANY); 2748 } 2749 sc->hn_ifnet_lnkhand = EVENTHANDLER_REGISTER(ifnet_link_event, 2750 hn_ifnet_lnkevent, sc, EVENTHANDLER_PRI_ANY); 2751 2752 /* 2753 * NOTE: 2754 * Subscribe ether_ifattach event, instead of ifnet_arrival event, 2755 * since interface's LLADDR is needed; interface LLADDR is not 2756 * available when ifnet_arrival event is triggered. 2757 */ 2758 sc->hn_ifnet_atthand = EVENTHANDLER_REGISTER(ether_ifattach_event, 2759 hn_ifnet_attevent, sc, EVENTHANDLER_PRI_ANY); 2760 sc->hn_ifnet_dethand = EVENTHANDLER_REGISTER(ifnet_departure_event, 2761 hn_ifnet_detevent, sc, EVENTHANDLER_PRI_ANY); 2762 2763 return (0); 2764 failed: 2765 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) 2766 hn_synth_detach(sc); 2767 hn_detach(dev); 2768 return (error); 2769 } 2770 2771 static int 2772 hn_detach(device_t dev) 2773 { 2774 struct hn_softc *sc = device_get_softc(dev); 2775 if_t ifp = sc->hn_ifp, vf_ifp; 2776 2777 if (sc->hn_xact != NULL && vmbus_chan_is_revoked(sc->hn_prichan)) { 2778 /* 2779 * In case that the vmbus missed the orphan handler 2780 * installation. 2781 */ 2782 vmbus_xact_ctx_orphan(sc->hn_xact); 2783 } 2784 2785 HN_LOCK(sc); 2786 sc->hn_detaching = true; 2787 while (sc->hn_vf_caps_busy) 2788 sx_sleep(&sc->hn_vf_caps_busy, &sc->hn_lock, 0, "hnvfcap", 0); 2789 HN_UNLOCK(sc); 2790 if (sc->hn_vlan_atthand != NULL) 2791 EVENTHANDLER_DEREGISTER(vlan_config, sc->hn_vlan_atthand); 2792 if (sc->hn_vlan_dethand != NULL) 2793 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->hn_vlan_dethand); 2794 taskqueue_drain_timeout(sc->hn_vf_taskq, &sc->hn_vf_init); 2795 taskqueue_drain(sc->hn_vf_taskq, &sc->hn_vf_vlancap); 2796 2797 if (sc->hn_ifaddr_evthand != NULL) 2798 EVENTHANDLER_DEREGISTER(ifaddr_event, sc->hn_ifaddr_evthand); 2799 if (sc->hn_ifnet_evthand != NULL) 2800 EVENTHANDLER_DEREGISTER(ifnet_event, sc->hn_ifnet_evthand); 2801 if (sc->hn_ifnet_atthand != NULL) { 2802 EVENTHANDLER_DEREGISTER(ether_ifattach_event, 2803 sc->hn_ifnet_atthand); 2804 } 2805 if (sc->hn_ifnet_dethand != NULL) { 2806 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 2807 sc->hn_ifnet_dethand); 2808 } 2809 if (sc->hn_ifnet_lnkhand != NULL) 2810 EVENTHANDLER_DEREGISTER(ifnet_link_event, sc->hn_ifnet_lnkhand); 2811 2812 HN_LOCK(sc); 2813 vf_ifp = sc->hn_vf_ifp; 2814 /* hn is leaving; remove its registrations from the live VF. */ 2815 if (vf_ifp != NULL && hn_xpnt_vf) 2816 hn_xpnt_vf_sync_vlans(sc, true); 2817 HN_UNLOCK(sc); 2818 if (vf_ifp != NULL) 2819 hn_ifnet_detevent(sc, vf_ifp); 2820 2821 if (device_is_attached(dev)) { 2822 HN_LOCK(sc); 2823 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 2824 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 2825 hn_stop(sc, true); 2826 /* 2827 * NOTE: 2828 * hn_stop() only suspends data, so management 2829 * stuffs have to be suspended manually here. 2830 */ 2831 hn_suspend_mgmt(sc); 2832 hn_synth_detach(sc); 2833 } 2834 HN_UNLOCK(sc); 2835 ether_ifdetach(ifp); 2836 } 2837 2838 ifmedia_removeall(&sc->hn_media); 2839 hn_destroy_rx_data(sc); 2840 hn_destroy_tx_data(sc); 2841 2842 if (sc->hn_tx_taskqs != NULL && sc->hn_tx_taskqs != hn_tx_taskque) { 2843 int i; 2844 2845 for (i = 0; i < hn_tx_taskq_cnt; ++i) 2846 taskqueue_free(sc->hn_tx_taskqs[i]); 2847 free(sc->hn_tx_taskqs, M_DEVBUF); 2848 } 2849 taskqueue_free(sc->hn_mgmt_taskq0); 2850 if (sc->hn_vf_taskq != NULL) 2851 taskqueue_free(sc->hn_vf_taskq); 2852 2853 if (sc->hn_xact != NULL) { 2854 /* 2855 * Uninstall the orphan handler _before_ the xact is 2856 * destructed. 2857 */ 2858 vmbus_chan_unset_orphan(sc->hn_prichan); 2859 vmbus_xact_ctx_destroy(sc->hn_xact); 2860 } 2861 2862 if_free(ifp); 2863 2864 HN_LOCK_DESTROY(sc); 2865 rm_destroy(&sc->hn_vf_lock); 2866 return (0); 2867 } 2868 2869 static int 2870 hn_shutdown(device_t dev) 2871 { 2872 2873 return (0); 2874 } 2875 2876 static void 2877 hn_link_status(struct hn_softc *sc) 2878 { 2879 uint32_t link_status; 2880 int error; 2881 2882 error = hn_rndis_get_linkstatus(sc, &link_status); 2883 if (error) { 2884 /* XXX what to do? */ 2885 return; 2886 } 2887 2888 if (link_status == NDIS_MEDIA_STATE_CONNECTED) 2889 sc->hn_link_flags |= HN_LINK_FLAG_LINKUP; 2890 else 2891 sc->hn_link_flags &= ~HN_LINK_FLAG_LINKUP; 2892 if_link_state_change(sc->hn_ifp, 2893 (sc->hn_link_flags & HN_LINK_FLAG_LINKUP) ? 2894 LINK_STATE_UP : LINK_STATE_DOWN); 2895 } 2896 2897 static void 2898 hn_link_taskfunc(void *xsc, int pending __unused) 2899 { 2900 struct hn_softc *sc = xsc; 2901 2902 if (sc->hn_link_flags & HN_LINK_FLAG_NETCHG) 2903 return; 2904 hn_link_status(sc); 2905 } 2906 2907 static void 2908 hn_netchg_init_taskfunc(void *xsc, int pending __unused) 2909 { 2910 struct hn_softc *sc = xsc; 2911 2912 /* Prevent any link status checks from running. */ 2913 sc->hn_link_flags |= HN_LINK_FLAG_NETCHG; 2914 2915 /* 2916 * Fake up a [link down --> link up] state change; 5 seconds 2917 * delay is used, which closely simulates miibus reaction 2918 * upon link down event. 2919 */ 2920 sc->hn_link_flags &= ~HN_LINK_FLAG_LINKUP; 2921 if_link_state_change(sc->hn_ifp, LINK_STATE_DOWN); 2922 taskqueue_enqueue_timeout(sc->hn_mgmt_taskq0, 2923 &sc->hn_netchg_status, 5 * hz); 2924 } 2925 2926 static void 2927 hn_netchg_status_taskfunc(void *xsc, int pending __unused) 2928 { 2929 struct hn_softc *sc = xsc; 2930 2931 /* Re-allow link status checks. */ 2932 sc->hn_link_flags &= ~HN_LINK_FLAG_NETCHG; 2933 hn_link_status(sc); 2934 } 2935 2936 static void 2937 hn_update_link_status(struct hn_softc *sc) 2938 { 2939 2940 if (sc->hn_mgmt_taskq != NULL) 2941 taskqueue_enqueue(sc->hn_mgmt_taskq, &sc->hn_link_task); 2942 } 2943 2944 static void 2945 hn_change_network(struct hn_softc *sc) 2946 { 2947 2948 if (sc->hn_mgmt_taskq != NULL) 2949 taskqueue_enqueue(sc->hn_mgmt_taskq, &sc->hn_netchg_init); 2950 } 2951 2952 static __inline int 2953 hn_txdesc_dmamap_load(struct hn_tx_ring *txr, struct hn_txdesc *txd, 2954 struct mbuf **m_head, bus_dma_segment_t *segs, int *nsegs) 2955 { 2956 struct mbuf *m = *m_head; 2957 int error; 2958 2959 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID, ("txd uses chim")); 2960 2961 error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, txd->data_dmap, 2962 m, segs, nsegs, BUS_DMA_NOWAIT); 2963 if (error == EFBIG) { 2964 struct mbuf *m_new; 2965 2966 m_new = m_collapse(m, M_NOWAIT, HN_TX_DATA_SEGCNT_MAX); 2967 if (m_new == NULL) 2968 return ENOBUFS; 2969 else 2970 *m_head = m = m_new; 2971 txr->hn_tx_collapsed++; 2972 2973 error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, 2974 txd->data_dmap, m, segs, nsegs, BUS_DMA_NOWAIT); 2975 } 2976 if (!error) { 2977 bus_dmamap_sync(txr->hn_tx_data_dtag, txd->data_dmap, 2978 BUS_DMASYNC_PREWRITE); 2979 txd->flags |= HN_TXD_FLAG_DMAMAP; 2980 } 2981 return error; 2982 } 2983 2984 static __inline int 2985 hn_txdesc_put(struct hn_tx_ring *txr, struct hn_txdesc *txd) 2986 { 2987 2988 KASSERT((txd->flags & HN_TXD_FLAG_ONLIST) == 0, 2989 ("put an onlist txd %#x", txd->flags)); 2990 KASSERT((txd->flags & HN_TXD_FLAG_ONAGG) == 0, 2991 ("put an onagg txd %#x", txd->flags)); 2992 2993 KASSERT(txd->refs > 0, ("invalid txd refs %d", txd->refs)); 2994 if (atomic_fetchadd_int(&txd->refs, -1) != 1) 2995 return 0; 2996 2997 if (!STAILQ_EMPTY(&txd->agg_list)) { 2998 struct hn_txdesc *tmp_txd; 2999 3000 while ((tmp_txd = STAILQ_FIRST(&txd->agg_list)) != NULL) { 3001 int freed __diagused; 3002 3003 KASSERT(STAILQ_EMPTY(&tmp_txd->agg_list), 3004 ("resursive aggregation on aggregated txdesc")); 3005 KASSERT((tmp_txd->flags & HN_TXD_FLAG_ONAGG), 3006 ("not aggregated txdesc")); 3007 KASSERT((tmp_txd->flags & HN_TXD_FLAG_DMAMAP) == 0, 3008 ("aggregated txdesc uses dmamap")); 3009 KASSERT(tmp_txd->chim_index == HN_NVS_CHIM_IDX_INVALID, 3010 ("aggregated txdesc consumes " 3011 "chimney sending buffer")); 3012 KASSERT(tmp_txd->chim_size == 0, 3013 ("aggregated txdesc has non-zero " 3014 "chimney sending size")); 3015 3016 STAILQ_REMOVE_HEAD(&txd->agg_list, agg_link); 3017 tmp_txd->flags &= ~HN_TXD_FLAG_ONAGG; 3018 freed = hn_txdesc_put(txr, tmp_txd); 3019 KASSERT(freed, ("failed to free aggregated txdesc")); 3020 } 3021 } 3022 3023 if (txd->chim_index != HN_NVS_CHIM_IDX_INVALID) { 3024 KASSERT((txd->flags & HN_TXD_FLAG_DMAMAP) == 0, 3025 ("chim txd uses dmamap")); 3026 hn_chim_free(txr->hn_sc, txd->chim_index); 3027 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 3028 txd->chim_size = 0; 3029 } else if (txd->flags & HN_TXD_FLAG_DMAMAP) { 3030 bus_dmamap_sync(txr->hn_tx_data_dtag, 3031 txd->data_dmap, BUS_DMASYNC_POSTWRITE); 3032 bus_dmamap_unload(txr->hn_tx_data_dtag, 3033 txd->data_dmap); 3034 txd->flags &= ~HN_TXD_FLAG_DMAMAP; 3035 } 3036 3037 if (txd->m != NULL) { 3038 m_freem(txd->m); 3039 txd->m = NULL; 3040 } 3041 3042 txd->flags |= HN_TXD_FLAG_ONLIST; 3043 #ifndef HN_USE_TXDESC_BUFRING 3044 mtx_lock_spin(&txr->hn_txlist_spin); 3045 KASSERT(txr->hn_txdesc_avail >= 0 && 3046 txr->hn_txdesc_avail < txr->hn_txdesc_cnt, 3047 ("txdesc_put: invalid txd avail %d", txr->hn_txdesc_avail)); 3048 txr->hn_txdesc_avail++; 3049 SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); 3050 mtx_unlock_spin(&txr->hn_txlist_spin); 3051 #else /* HN_USE_TXDESC_BUFRING */ 3052 #ifdef HN_DEBUG 3053 atomic_add_int(&txr->hn_txdesc_avail, 1); 3054 #endif 3055 buf_ring_enqueue(txr->hn_txdesc_br, txd); 3056 #endif /* !HN_USE_TXDESC_BUFRING */ 3057 3058 return 1; 3059 } 3060 3061 static __inline struct hn_txdesc * 3062 hn_txdesc_get(struct hn_tx_ring *txr) 3063 { 3064 struct hn_txdesc *txd; 3065 3066 #ifndef HN_USE_TXDESC_BUFRING 3067 mtx_lock_spin(&txr->hn_txlist_spin); 3068 txd = SLIST_FIRST(&txr->hn_txlist); 3069 if (txd != NULL) { 3070 KASSERT(txr->hn_txdesc_avail > 0, 3071 ("txdesc_get: invalid txd avail %d", txr->hn_txdesc_avail)); 3072 txr->hn_txdesc_avail--; 3073 SLIST_REMOVE_HEAD(&txr->hn_txlist, link); 3074 } 3075 mtx_unlock_spin(&txr->hn_txlist_spin); 3076 #else 3077 txd = buf_ring_dequeue_sc(txr->hn_txdesc_br); 3078 #endif 3079 3080 if (txd != NULL) { 3081 #ifdef HN_USE_TXDESC_BUFRING 3082 #ifdef HN_DEBUG 3083 atomic_subtract_int(&txr->hn_txdesc_avail, 1); 3084 #endif 3085 #endif /* HN_USE_TXDESC_BUFRING */ 3086 KASSERT(txd->m == NULL && txd->refs == 0 && 3087 STAILQ_EMPTY(&txd->agg_list) && 3088 txd->chim_index == HN_NVS_CHIM_IDX_INVALID && 3089 txd->chim_size == 0 && 3090 (txd->flags & HN_TXD_FLAG_ONLIST) && 3091 (txd->flags & HN_TXD_FLAG_ONAGG) == 0 && 3092 (txd->flags & HN_TXD_FLAG_DMAMAP) == 0, ("invalid txd")); 3093 txd->flags &= ~HN_TXD_FLAG_ONLIST; 3094 txd->refs = 1; 3095 } 3096 return txd; 3097 } 3098 3099 static __inline void 3100 hn_txdesc_hold(struct hn_txdesc *txd) 3101 { 3102 3103 /* 0->1 transition will never work */ 3104 KASSERT(txd->refs > 0, ("invalid txd refs %d", txd->refs)); 3105 atomic_add_int(&txd->refs, 1); 3106 } 3107 3108 static __inline void 3109 hn_txdesc_agg(struct hn_txdesc *agg_txd, struct hn_txdesc *txd) 3110 { 3111 3112 KASSERT((agg_txd->flags & HN_TXD_FLAG_ONAGG) == 0, 3113 ("recursive aggregation on aggregating txdesc")); 3114 3115 KASSERT((txd->flags & HN_TXD_FLAG_ONAGG) == 0, 3116 ("already aggregated")); 3117 KASSERT(STAILQ_EMPTY(&txd->agg_list), 3118 ("recursive aggregation on to-be-aggregated txdesc")); 3119 3120 txd->flags |= HN_TXD_FLAG_ONAGG; 3121 STAILQ_INSERT_TAIL(&agg_txd->agg_list, txd, agg_link); 3122 } 3123 3124 static bool 3125 hn_tx_ring_pending(struct hn_tx_ring *txr) 3126 { 3127 bool pending = false; 3128 3129 #ifndef HN_USE_TXDESC_BUFRING 3130 mtx_lock_spin(&txr->hn_txlist_spin); 3131 if (txr->hn_txdesc_avail != txr->hn_txdesc_cnt) 3132 pending = true; 3133 mtx_unlock_spin(&txr->hn_txlist_spin); 3134 #else 3135 if (!buf_ring_full(txr->hn_txdesc_br)) 3136 pending = true; 3137 #endif 3138 return (pending); 3139 } 3140 3141 static __inline void 3142 hn_txeof(struct hn_tx_ring *txr) 3143 { 3144 txr->hn_has_txeof = 0; 3145 txr->hn_txeof(txr); 3146 } 3147 3148 static void 3149 hn_txpkt_done(struct hn_nvs_sendctx *sndc, struct hn_softc *sc, 3150 struct vmbus_channel *chan, const void *data __unused, int dlen __unused) 3151 { 3152 struct hn_txdesc *txd = sndc->hn_cbarg; 3153 struct hn_tx_ring *txr; 3154 3155 txr = txd->txr; 3156 KASSERT(txr->hn_chan == chan, 3157 ("channel mismatch, on chan%u, should be chan%u", 3158 vmbus_chan_id(chan), vmbus_chan_id(txr->hn_chan))); 3159 3160 txr->hn_has_txeof = 1; 3161 hn_txdesc_put(txr, txd); 3162 3163 ++txr->hn_txdone_cnt; 3164 if (txr->hn_txdone_cnt >= HN_EARLY_TXEOF_THRESH) { 3165 txr->hn_txdone_cnt = 0; 3166 if (txr->hn_oactive) 3167 hn_txeof(txr); 3168 } 3169 } 3170 3171 static void 3172 hn_chan_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr) 3173 { 3174 #if defined(INET) || defined(INET6) 3175 struct epoch_tracker et; 3176 3177 NET_EPOCH_ENTER(et); 3178 tcp_lro_flush_all(&rxr->hn_lro); 3179 NET_EPOCH_EXIT(et); 3180 #endif 3181 3182 /* 3183 * NOTE: 3184 * 'txr' could be NULL, if multiple channels and 3185 * ifnet.if_start method are enabled. 3186 */ 3187 if (txr == NULL || !txr->hn_has_txeof) 3188 return; 3189 3190 txr->hn_txdone_cnt = 0; 3191 hn_txeof(txr); 3192 } 3193 3194 static __inline uint32_t 3195 hn_rndis_pktmsg_offset(uint32_t ofs) 3196 { 3197 3198 KASSERT(ofs >= sizeof(struct rndis_packet_msg), 3199 ("invalid RNDIS packet msg offset %u", ofs)); 3200 return (ofs - __offsetof(struct rndis_packet_msg, rm_dataoffset)); 3201 } 3202 3203 static __inline void * 3204 hn_rndis_pktinfo_append(struct rndis_packet_msg *pkt, size_t pktsize, 3205 size_t pi_dlen, uint32_t pi_type) 3206 { 3207 const size_t pi_size = HN_RNDIS_PKTINFO_SIZE(pi_dlen); 3208 struct rndis_pktinfo *pi; 3209 3210 KASSERT((pi_size & RNDIS_PACKET_MSG_OFFSET_ALIGNMASK) == 0, 3211 ("unaligned pktinfo size %zu, pktinfo dlen %zu", pi_size, pi_dlen)); 3212 3213 /* 3214 * Per-packet-info does not move; it only grows. 3215 * 3216 * NOTE: 3217 * rm_pktinfooffset in this phase counts from the beginning 3218 * of rndis_packet_msg. 3219 */ 3220 KASSERT(pkt->rm_pktinfooffset + pkt->rm_pktinfolen + pi_size <= pktsize, 3221 ("%u pktinfo overflows RNDIS packet msg", pi_type)); 3222 pi = (struct rndis_pktinfo *)((uint8_t *)pkt + pkt->rm_pktinfooffset + 3223 pkt->rm_pktinfolen); 3224 pkt->rm_pktinfolen += pi_size; 3225 3226 pi->rm_size = pi_size; 3227 pi->rm_type = pi_type; 3228 pi->rm_internal = 0; 3229 pi->rm_pktinfooffset = RNDIS_PKTINFO_OFFSET; 3230 3231 return (pi->rm_data); 3232 } 3233 3234 static __inline int 3235 hn_flush_txagg(if_t ifp, struct hn_tx_ring *txr) 3236 { 3237 struct hn_txdesc *txd; 3238 struct mbuf *m; 3239 int error, pkts; 3240 3241 txd = txr->hn_agg_txd; 3242 KASSERT(txd != NULL, ("no aggregate txdesc")); 3243 3244 /* 3245 * Since hn_txpkt() will reset this temporary stat, save 3246 * it now, so that oerrors can be updated properly, if 3247 * hn_txpkt() ever fails. 3248 */ 3249 pkts = txr->hn_stat_pkts; 3250 3251 /* 3252 * Since txd's mbuf will _not_ be freed upon hn_txpkt() 3253 * failure, save it for later freeing, if hn_txpkt() ever 3254 * fails. 3255 */ 3256 m = txd->m; 3257 error = hn_txpkt(ifp, txr, txd); 3258 if (__predict_false(error)) { 3259 /* txd is freed, but m is not. */ 3260 m_freem(m); 3261 3262 txr->hn_flush_failed++; 3263 if_inc_counter(ifp, IFCOUNTER_OERRORS, pkts); 3264 } 3265 3266 /* Reset all aggregation states. */ 3267 txr->hn_agg_txd = NULL; 3268 txr->hn_agg_szleft = 0; 3269 txr->hn_agg_pktleft = 0; 3270 txr->hn_agg_prevpkt = NULL; 3271 3272 return (error); 3273 } 3274 3275 static void * 3276 hn_try_txagg(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd, 3277 int pktsize) 3278 { 3279 void *chim; 3280 3281 if (txr->hn_agg_txd != NULL) { 3282 if (txr->hn_agg_pktleft >= 1 && txr->hn_agg_szleft > pktsize) { 3283 struct hn_txdesc *agg_txd = txr->hn_agg_txd; 3284 struct rndis_packet_msg *pkt = txr->hn_agg_prevpkt; 3285 int olen; 3286 3287 /* 3288 * Update the previous RNDIS packet's total length, 3289 * it can be increased due to the mandatory alignment 3290 * padding for this RNDIS packet. And update the 3291 * aggregating txdesc's chimney sending buffer size 3292 * accordingly. 3293 * 3294 * XXX 3295 * Zero-out the padding, as required by the RNDIS spec. 3296 */ 3297 olen = pkt->rm_len; 3298 pkt->rm_len = roundup2(olen, txr->hn_agg_align); 3299 agg_txd->chim_size += pkt->rm_len - olen; 3300 3301 /* Link this txdesc to the parent. */ 3302 hn_txdesc_agg(agg_txd, txd); 3303 3304 chim = (uint8_t *)pkt + pkt->rm_len; 3305 /* Save the current packet for later fixup. */ 3306 txr->hn_agg_prevpkt = chim; 3307 3308 txr->hn_agg_pktleft--; 3309 txr->hn_agg_szleft -= pktsize; 3310 if (txr->hn_agg_szleft <= 3311 HN_PKTSIZE_MIN(txr->hn_agg_align)) { 3312 /* 3313 * Probably can't aggregate more packets, 3314 * flush this aggregating txdesc proactively. 3315 */ 3316 txr->hn_agg_pktleft = 0; 3317 } 3318 /* Done! */ 3319 return (chim); 3320 } 3321 hn_flush_txagg(ifp, txr); 3322 } 3323 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 3324 3325 txr->hn_tx_chimney_tried++; 3326 txd->chim_index = hn_chim_alloc(txr->hn_sc); 3327 if (txd->chim_index == HN_NVS_CHIM_IDX_INVALID) 3328 return (NULL); 3329 txr->hn_tx_chimney++; 3330 3331 chim = txr->hn_sc->hn_chim + 3332 (txd->chim_index * txr->hn_sc->hn_chim_szmax); 3333 3334 if (txr->hn_agg_pktmax > 1 && 3335 txr->hn_agg_szmax > pktsize + HN_PKTSIZE_MIN(txr->hn_agg_align)) { 3336 txr->hn_agg_txd = txd; 3337 txr->hn_agg_pktleft = txr->hn_agg_pktmax - 1; 3338 txr->hn_agg_szleft = txr->hn_agg_szmax - pktsize; 3339 txr->hn_agg_prevpkt = chim; 3340 } 3341 return (chim); 3342 } 3343 3344 /* 3345 * NOTE: 3346 * If this function fails, then both txd and m_head0 will be freed. 3347 */ 3348 static int 3349 hn_encap(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd, 3350 struct mbuf **m_head0) 3351 { 3352 bus_dma_segment_t segs[HN_TX_DATA_SEGCNT_MAX]; 3353 int error, nsegs, i; 3354 struct mbuf *m_head = *m_head0; 3355 struct rndis_packet_msg *pkt; 3356 uint32_t *pi_data; 3357 void *chim = NULL; 3358 int pkt_hlen, pkt_size; 3359 3360 pkt = txd->rndis_pkt; 3361 pkt_size = HN_PKTSIZE(m_head, txr->hn_agg_align); 3362 if (pkt_size < txr->hn_chim_size) { 3363 chim = hn_try_txagg(ifp, txr, txd, pkt_size); 3364 if (chim != NULL) 3365 pkt = chim; 3366 } else { 3367 if (txr->hn_agg_txd != NULL) 3368 hn_flush_txagg(ifp, txr); 3369 } 3370 3371 pkt->rm_type = REMOTE_NDIS_PACKET_MSG; 3372 pkt->rm_len = m_head->m_pkthdr.len; 3373 pkt->rm_dataoffset = 0; 3374 pkt->rm_datalen = m_head->m_pkthdr.len; 3375 pkt->rm_oobdataoffset = 0; 3376 pkt->rm_oobdatalen = 0; 3377 pkt->rm_oobdataelements = 0; 3378 pkt->rm_pktinfooffset = sizeof(*pkt); 3379 pkt->rm_pktinfolen = 0; 3380 pkt->rm_vchandle = 0; 3381 pkt->rm_reserved = 0; 3382 3383 if (txr->hn_tx_flags & HN_TX_FLAG_HASHVAL) { 3384 /* 3385 * Set the hash value for this packet. 3386 */ 3387 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3388 HN_NDIS_HASH_VALUE_SIZE, HN_NDIS_PKTINFO_TYPE_HASHVAL); 3389 3390 if (M_HASHTYPE_ISHASH(m_head)) 3391 /* 3392 * The flowid field contains the hash value host 3393 * set in the rx queue if it is a ip forwarding pkt. 3394 * Set the same hash value so host can send on the 3395 * cpu it was received. 3396 */ 3397 *pi_data = m_head->m_pkthdr.flowid; 3398 else 3399 /* 3400 * Otherwise just put the tx queue index. 3401 */ 3402 *pi_data = txr->hn_tx_idx; 3403 } 3404 3405 if (m_head->m_flags & M_VLANTAG) { 3406 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3407 NDIS_VLAN_INFO_SIZE, NDIS_PKTINFO_TYPE_VLAN); 3408 *pi_data = NDIS_VLAN_INFO_MAKE( 3409 EVL_VLANOFTAG(m_head->m_pkthdr.ether_vtag), 3410 EVL_PRIOFTAG(m_head->m_pkthdr.ether_vtag), 3411 EVL_CFIOFTAG(m_head->m_pkthdr.ether_vtag)); 3412 } 3413 3414 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 3415 #if defined(INET6) || defined(INET) 3416 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3417 NDIS_LSO2_INFO_SIZE, NDIS_PKTINFO_TYPE_LSO); 3418 #ifdef INET 3419 if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { 3420 *pi_data = NDIS_LSO2_INFO_MAKEIPV4( 3421 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen, 3422 m_head->m_pkthdr.tso_segsz); 3423 } 3424 #endif 3425 #if defined(INET6) && defined(INET) 3426 else 3427 #endif 3428 #ifdef INET6 3429 { 3430 *pi_data = NDIS_LSO2_INFO_MAKEIPV6( 3431 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen, 3432 m_head->m_pkthdr.tso_segsz); 3433 } 3434 #endif 3435 #endif /* INET6 || INET */ 3436 } else if (m_head->m_pkthdr.csum_flags & txr->hn_csum_assist) { 3437 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3438 NDIS_TXCSUM_INFO_SIZE, NDIS_PKTINFO_TYPE_CSUM); 3439 if (m_head->m_pkthdr.csum_flags & 3440 (CSUM_IP6_TCP | CSUM_IP6_UDP)) { 3441 *pi_data = NDIS_TXCSUM_INFO_IPV6; 3442 } else { 3443 *pi_data = NDIS_TXCSUM_INFO_IPV4; 3444 if (m_head->m_pkthdr.csum_flags & CSUM_IP) 3445 *pi_data |= NDIS_TXCSUM_INFO_IPCS; 3446 } 3447 3448 if (m_head->m_pkthdr.csum_flags & 3449 (CSUM_IP_TCP | CSUM_IP6_TCP)) { 3450 *pi_data |= NDIS_TXCSUM_INFO_MKTCPCS( 3451 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen); 3452 } else if (m_head->m_pkthdr.csum_flags & 3453 (CSUM_IP_UDP | CSUM_IP6_UDP)) { 3454 *pi_data |= NDIS_TXCSUM_INFO_MKUDPCS( 3455 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen); 3456 } 3457 } 3458 3459 pkt_hlen = pkt->rm_pktinfooffset + pkt->rm_pktinfolen; 3460 /* Fixup RNDIS packet message total length */ 3461 pkt->rm_len += pkt_hlen; 3462 /* Convert RNDIS packet message offsets */ 3463 pkt->rm_dataoffset = hn_rndis_pktmsg_offset(pkt_hlen); 3464 pkt->rm_pktinfooffset = hn_rndis_pktmsg_offset(pkt->rm_pktinfooffset); 3465 3466 /* 3467 * Fast path: Chimney sending. 3468 */ 3469 if (chim != NULL) { 3470 struct hn_txdesc *tgt_txd = txd; 3471 3472 if (txr->hn_agg_txd != NULL) { 3473 tgt_txd = txr->hn_agg_txd; 3474 #ifdef INVARIANTS 3475 *m_head0 = NULL; 3476 #endif 3477 } 3478 3479 KASSERT(pkt == chim, 3480 ("RNDIS pkt not in chimney sending buffer")); 3481 KASSERT(tgt_txd->chim_index != HN_NVS_CHIM_IDX_INVALID, 3482 ("chimney sending buffer is not used")); 3483 tgt_txd->chim_size += pkt->rm_len; 3484 3485 m_copydata(m_head, 0, m_head->m_pkthdr.len, 3486 ((uint8_t *)chim) + pkt_hlen); 3487 3488 txr->hn_gpa_cnt = 0; 3489 txr->hn_sendpkt = hn_txpkt_chim; 3490 goto done; 3491 } 3492 3493 KASSERT(txr->hn_agg_txd == NULL, ("aggregating sglist txdesc")); 3494 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID, 3495 ("chimney buffer is used")); 3496 KASSERT(pkt == txd->rndis_pkt, ("RNDIS pkt not in txdesc")); 3497 3498 error = hn_txdesc_dmamap_load(txr, txd, &m_head, segs, &nsegs); 3499 if (__predict_false(error)) { 3500 int freed __diagused; 3501 3502 /* 3503 * This mbuf is not linked w/ the txd yet, so free it now. 3504 */ 3505 m_freem(m_head); 3506 *m_head0 = NULL; 3507 3508 freed = hn_txdesc_put(txr, txd); 3509 KASSERT(freed != 0, 3510 ("fail to free txd upon txdma error")); 3511 3512 txr->hn_txdma_failed++; 3513 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 3514 return error; 3515 } 3516 *m_head0 = m_head; 3517 3518 /* +1 RNDIS packet message */ 3519 txr->hn_gpa_cnt = nsegs + 1; 3520 3521 /* send packet with page buffer */ 3522 txr->hn_gpa[0].gpa_page = atop(txd->rndis_pkt_paddr); 3523 txr->hn_gpa[0].gpa_ofs = txd->rndis_pkt_paddr & PAGE_MASK; 3524 txr->hn_gpa[0].gpa_len = pkt_hlen; 3525 3526 /* 3527 * Fill the page buffers with mbuf info after the page 3528 * buffer for RNDIS packet message. 3529 */ 3530 for (i = 0; i < nsegs; ++i) { 3531 struct vmbus_gpa *gpa = &txr->hn_gpa[i + 1]; 3532 3533 gpa->gpa_page = atop(segs[i].ds_addr); 3534 gpa->gpa_ofs = segs[i].ds_addr & PAGE_MASK; 3535 gpa->gpa_len = segs[i].ds_len; 3536 } 3537 3538 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 3539 txd->chim_size = 0; 3540 txr->hn_sendpkt = hn_txpkt_sglist; 3541 done: 3542 txd->m = m_head; 3543 3544 /* Set the completion routine */ 3545 hn_nvs_sendctx_init(&txd->send_ctx, hn_txpkt_done, txd); 3546 3547 /* Update temporary stats for later use. */ 3548 txr->hn_stat_pkts++; 3549 txr->hn_stat_size += m_head->m_pkthdr.len; 3550 if (m_head->m_flags & M_MCAST) 3551 txr->hn_stat_mcasts++; 3552 3553 return 0; 3554 } 3555 3556 /* 3557 * NOTE: 3558 * If this function fails, then txd will be freed, but the mbuf 3559 * associated w/ the txd will _not_ be freed. 3560 */ 3561 static int 3562 hn_txpkt(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd) 3563 { 3564 int error, send_failed = 0, has_bpf; 3565 3566 again: 3567 has_bpf = bpf_peers_present_if(ifp); 3568 if (has_bpf) { 3569 /* 3570 * Make sure that this txd and any aggregated txds are not 3571 * freed before ETHER_BPF_MTAP. 3572 */ 3573 hn_txdesc_hold(txd); 3574 } 3575 error = txr->hn_sendpkt(txr, txd); 3576 if (!error) { 3577 if (has_bpf) { 3578 const struct hn_txdesc *tmp_txd; 3579 3580 ETHER_BPF_MTAP(ifp, txd->m); 3581 STAILQ_FOREACH(tmp_txd, &txd->agg_list, agg_link) 3582 ETHER_BPF_MTAP(ifp, tmp_txd->m); 3583 } 3584 3585 if_inc_counter(ifp, IFCOUNTER_OPACKETS, txr->hn_stat_pkts); 3586 #ifdef HN_IFSTART_SUPPORT 3587 if (!hn_use_if_start) 3588 #endif 3589 { 3590 if_inc_counter(ifp, IFCOUNTER_OBYTES, 3591 txr->hn_stat_size); 3592 if (txr->hn_stat_mcasts != 0) { 3593 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 3594 txr->hn_stat_mcasts); 3595 } 3596 } 3597 txr->hn_pkts += txr->hn_stat_pkts; 3598 txr->hn_sends++; 3599 } 3600 if (has_bpf) 3601 hn_txdesc_put(txr, txd); 3602 3603 if (__predict_false(error)) { 3604 int freed __diagused; 3605 3606 /* 3607 * This should "really rarely" happen. 3608 * 3609 * XXX Too many RX to be acked or too many sideband 3610 * commands to run? Ask netvsc_channel_rollup() 3611 * to kick start later. 3612 */ 3613 txr->hn_has_txeof = 1; 3614 if (!send_failed) { 3615 txr->hn_send_failed++; 3616 send_failed = 1; 3617 /* 3618 * Try sending again after set hn_has_txeof; 3619 * in case that we missed the last 3620 * netvsc_channel_rollup(). 3621 */ 3622 goto again; 3623 } 3624 if_printf(ifp, "send failed\n"); 3625 3626 /* 3627 * Caller will perform further processing on the 3628 * associated mbuf, so don't free it in hn_txdesc_put(); 3629 * only unload it from the DMA map in hn_txdesc_put(), 3630 * if it was loaded. 3631 */ 3632 txd->m = NULL; 3633 freed = hn_txdesc_put(txr, txd); 3634 KASSERT(freed != 0, 3635 ("fail to free txd upon send error")); 3636 3637 txr->hn_send_failed++; 3638 } 3639 3640 /* Reset temporary stats, after this sending is done. */ 3641 txr->hn_stat_size = 0; 3642 txr->hn_stat_pkts = 0; 3643 txr->hn_stat_mcasts = 0; 3644 3645 return (error); 3646 } 3647 3648 /* 3649 * Append the specified data to the indicated mbuf chain, 3650 * Extend the mbuf chain if the new data does not fit in 3651 * existing space. 3652 * 3653 * This is a minor rewrite of m_append() from sys/kern/uipc_mbuf.c. 3654 * There should be an equivalent in the kernel mbuf code, 3655 * but there does not appear to be one yet. 3656 * 3657 * Differs from m_append() in that additional mbufs are 3658 * allocated with cluster size MJUMPAGESIZE, and filled 3659 * accordingly. 3660 * 3661 * Return the last mbuf in the chain or NULL if failed to 3662 * allocate new mbuf. 3663 */ 3664 static struct mbuf * 3665 hv_m_append(struct mbuf *m0, int len, c_caddr_t cp) 3666 { 3667 struct mbuf *m, *n; 3668 int remainder, space; 3669 3670 for (m = m0; m->m_next != NULL; m = m->m_next) 3671 ; 3672 remainder = len; 3673 space = M_TRAILINGSPACE(m); 3674 if (space > 0) { 3675 /* 3676 * Copy into available space. 3677 */ 3678 if (space > remainder) 3679 space = remainder; 3680 bcopy(cp, mtod(m, caddr_t) + m->m_len, space); 3681 m->m_len += space; 3682 cp += space; 3683 remainder -= space; 3684 } 3685 while (remainder > 0) { 3686 /* 3687 * Allocate a new mbuf; could check space 3688 * and allocate a cluster instead. 3689 */ 3690 n = m_getjcl(M_NOWAIT, m->m_type, 0, MJUMPAGESIZE); 3691 if (n == NULL) 3692 return NULL; 3693 n->m_len = min(MJUMPAGESIZE, remainder); 3694 bcopy(cp, mtod(n, caddr_t), n->m_len); 3695 cp += n->m_len; 3696 remainder -= n->m_len; 3697 m->m_next = n; 3698 m = n; 3699 } 3700 3701 return m; 3702 } 3703 3704 #if defined(INET) || defined(INET6) 3705 static __inline int 3706 hn_lro_rx(struct lro_ctrl *lc, struct mbuf *m) 3707 { 3708 if (hn_lro_mbufq_depth) { 3709 tcp_lro_queue_mbuf(lc, m); 3710 return 0; 3711 } 3712 return tcp_lro_rx(lc, m, 0); 3713 } 3714 #endif 3715 3716 static int 3717 hn_rxpkt(struct hn_rx_ring *rxr) 3718 { 3719 if_t ifp, hn_ifp = rxr->hn_ifp; 3720 struct mbuf *m_new, *n; 3721 int size, do_lro = 0, do_csum = 1, is_vf = 0; 3722 int hash_type = M_HASHTYPE_NONE; 3723 int l3proto = ETHERTYPE_MAX, l4proto = IPPROTO_DONE; 3724 int i; 3725 3726 ifp = hn_ifp; 3727 if (rxr->hn_rxvf_ifp != NULL) { 3728 /* 3729 * Non-transparent mode VF; pretend this packet is from 3730 * the VF. 3731 */ 3732 ifp = rxr->hn_rxvf_ifp; 3733 is_vf = 1; 3734 } else if (rxr->hn_rx_flags & HN_RX_FLAG_XPNT_VF) { 3735 /* Transparent mode VF. */ 3736 is_vf = 1; 3737 } 3738 3739 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 3740 /* 3741 * NOTE: 3742 * See the NOTE of hn_rndis_init_fixat(). This 3743 * function can be reached, immediately after the 3744 * RNDIS is initialized but before the ifnet is 3745 * setup on the hn_attach() path; drop the unexpected 3746 * packets. 3747 */ 3748 return (0); 3749 } 3750 3751 if (__predict_false(rxr->rsc.pktlen < ETHER_HDR_LEN)) { 3752 if_inc_counter(hn_ifp, IFCOUNTER_IERRORS, 1); 3753 return (0); 3754 } 3755 3756 if (rxr->rsc.cnt == 1 && rxr->rsc.pktlen <= MHLEN) { 3757 m_new = m_gethdr(M_NOWAIT, MT_DATA); 3758 if (m_new == NULL) { 3759 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3760 return (0); 3761 } 3762 memcpy(mtod(m_new, void *), rxr->rsc.frag_data[0], 3763 rxr->rsc.frag_len[0]); 3764 m_new->m_pkthdr.len = m_new->m_len = rxr->rsc.frag_len[0]; 3765 } else { 3766 /* 3767 * Get an mbuf with a cluster. For packets 2K or less, 3768 * get a standard 2K cluster. For anything larger, get a 3769 * 4K cluster. Any buffers larger than 4K can cause problems 3770 * if looped around to the Hyper-V TX channel, so avoid them. 3771 */ 3772 size = MCLBYTES; 3773 if (rxr->rsc.pktlen > MCLBYTES) { 3774 /* 4096 */ 3775 size = MJUMPAGESIZE; 3776 } 3777 3778 m_new = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size); 3779 if (m_new == NULL) { 3780 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3781 return (0); 3782 } 3783 3784 n = m_new; 3785 for (i = 0; i < rxr->rsc.cnt; i++) { 3786 n = hv_m_append(n, rxr->rsc.frag_len[i], 3787 rxr->rsc.frag_data[i]); 3788 if (n == NULL) { 3789 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3790 return (0); 3791 } else { 3792 m_new->m_pkthdr.len += rxr->rsc.frag_len[i]; 3793 } 3794 } 3795 } 3796 if (rxr->rsc.pktlen <= MHLEN) 3797 rxr->hn_small_pkts++; 3798 3799 m_new->m_pkthdr.rcvif = ifp; 3800 3801 if (__predict_false((if_getcapenable(hn_ifp) & IFCAP_RXCSUM) == 0)) 3802 do_csum = 0; 3803 3804 /* receive side checksum offload */ 3805 if (rxr->rsc.csum_info != NULL) { 3806 /* IP csum offload */ 3807 if ((*(rxr->rsc.csum_info) & NDIS_RXCSUM_INFO_IPCS_OK) && do_csum) { 3808 m_new->m_pkthdr.csum_flags |= 3809 (CSUM_IP_CHECKED | CSUM_IP_VALID); 3810 rxr->hn_csum_ip++; 3811 } 3812 3813 /* TCP/UDP csum offload */ 3814 if ((*(rxr->rsc.csum_info) & (NDIS_RXCSUM_INFO_UDPCS_OK | 3815 NDIS_RXCSUM_INFO_TCPCS_OK)) && do_csum) { 3816 m_new->m_pkthdr.csum_flags |= 3817 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3818 m_new->m_pkthdr.csum_data = 0xffff; 3819 if (*(rxr->rsc.csum_info) & NDIS_RXCSUM_INFO_TCPCS_OK) 3820 rxr->hn_csum_tcp++; 3821 else 3822 rxr->hn_csum_udp++; 3823 } 3824 3825 /* 3826 * XXX 3827 * As of this write (Oct 28th, 2016), host side will turn 3828 * on only TCPCS_OK and IPCS_OK even for UDP datagrams, so 3829 * the do_lro setting here is actually _not_ accurate. We 3830 * depend on the RSS hash type check to reset do_lro. 3831 */ 3832 if ((*(rxr->rsc.csum_info) & 3833 (NDIS_RXCSUM_INFO_TCPCS_OK | NDIS_RXCSUM_INFO_IPCS_OK)) == 3834 (NDIS_RXCSUM_INFO_TCPCS_OK | NDIS_RXCSUM_INFO_IPCS_OK)) 3835 do_lro = 1; 3836 } else { 3837 hn_rxpkt_proto(m_new, &l3proto, &l4proto); 3838 if (l3proto == ETHERTYPE_IP) { 3839 if (l4proto == IPPROTO_TCP) { 3840 if (do_csum && 3841 (rxr->hn_trust_hcsum & 3842 HN_TRUST_HCSUM_TCP)) { 3843 rxr->hn_csum_trusted++; 3844 m_new->m_pkthdr.csum_flags |= 3845 (CSUM_IP_CHECKED | CSUM_IP_VALID | 3846 CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3847 m_new->m_pkthdr.csum_data = 0xffff; 3848 } 3849 do_lro = 1; 3850 } else if (l4proto == IPPROTO_UDP) { 3851 if (do_csum && 3852 (rxr->hn_trust_hcsum & 3853 HN_TRUST_HCSUM_UDP)) { 3854 rxr->hn_csum_trusted++; 3855 m_new->m_pkthdr.csum_flags |= 3856 (CSUM_IP_CHECKED | CSUM_IP_VALID | 3857 CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3858 m_new->m_pkthdr.csum_data = 0xffff; 3859 } 3860 } else if (l4proto != IPPROTO_DONE && do_csum && 3861 (rxr->hn_trust_hcsum & HN_TRUST_HCSUM_IP)) { 3862 rxr->hn_csum_trusted++; 3863 m_new->m_pkthdr.csum_flags |= 3864 (CSUM_IP_CHECKED | CSUM_IP_VALID); 3865 } 3866 } 3867 } 3868 3869 if (rxr->rsc.vlan_info != NULL) { 3870 m_new->m_pkthdr.ether_vtag = EVL_MAKETAG( 3871 NDIS_VLAN_INFO_ID(*(rxr->rsc.vlan_info)), 3872 NDIS_VLAN_INFO_PRI(*(rxr->rsc.vlan_info)), 3873 NDIS_VLAN_INFO_CFI(*(rxr->rsc.vlan_info))); 3874 m_new->m_flags |= M_VLANTAG; 3875 } 3876 3877 /* 3878 * If VF is activated (transparent/non-transparent mode does not 3879 * matter here). 3880 * 3881 * - Disable LRO 3882 * 3883 * hn(4) will only receive broadcast packets, multicast packets, 3884 * TCP SYN and SYN|ACK (in Azure), LRO is useless for these 3885 * packet types. 3886 * 3887 * For non-transparent, we definitely _cannot_ enable LRO at 3888 * all, since the LRO flush will use hn(4) as the receiving 3889 * interface; i.e. hn_ifp->if_input(hn_ifp, m). 3890 */ 3891 if (is_vf) 3892 do_lro = 0; 3893 3894 /* 3895 * If VF is activated (transparent/non-transparent mode does not 3896 * matter here), do _not_ mess with unsupported hash types or 3897 * functions. 3898 */ 3899 if (rxr->rsc.hash_info != NULL) { 3900 rxr->hn_rss_pkts++; 3901 m_new->m_pkthdr.flowid = *(rxr->rsc.hash_value); 3902 if (!is_vf) 3903 hash_type = M_HASHTYPE_OPAQUE_HASH; 3904 if ((*(rxr->rsc.hash_info) & NDIS_HASH_FUNCTION_MASK) == 3905 NDIS_HASH_FUNCTION_TOEPLITZ) { 3906 uint32_t type = (*(rxr->rsc.hash_info) & NDIS_HASH_TYPE_MASK & 3907 rxr->hn_mbuf_hash); 3908 3909 /* 3910 * NOTE: 3911 * do_lro is resetted, if the hash types are not TCP 3912 * related. See the comment in the above csum_flags 3913 * setup section. 3914 */ 3915 switch (type) { 3916 case NDIS_HASH_IPV4: 3917 hash_type = M_HASHTYPE_RSS_IPV4; 3918 do_lro = 0; 3919 break; 3920 3921 case NDIS_HASH_TCP_IPV4: 3922 hash_type = M_HASHTYPE_RSS_TCP_IPV4; 3923 if (rxr->hn_rx_flags & HN_RX_FLAG_UDP_HASH) { 3924 int def_htype = M_HASHTYPE_OPAQUE_HASH; 3925 3926 if (is_vf) 3927 def_htype = M_HASHTYPE_NONE; 3928 3929 /* 3930 * UDP 4-tuple hash is delivered as 3931 * TCP 4-tuple hash. 3932 */ 3933 if (l3proto == ETHERTYPE_MAX) { 3934 hn_rxpkt_proto(m_new, 3935 &l3proto, &l4proto); 3936 } 3937 if (l3proto == ETHERTYPE_IP) { 3938 if (l4proto == IPPROTO_UDP && 3939 (rxr->hn_mbuf_hash & 3940 NDIS_HASH_UDP_IPV4_X)) { 3941 hash_type = 3942 M_HASHTYPE_RSS_UDP_IPV4; 3943 do_lro = 0; 3944 } else if (l4proto != 3945 IPPROTO_TCP) { 3946 hash_type = def_htype; 3947 do_lro = 0; 3948 } 3949 } else { 3950 hash_type = def_htype; 3951 do_lro = 0; 3952 } 3953 } 3954 break; 3955 3956 case NDIS_HASH_IPV6: 3957 hash_type = M_HASHTYPE_RSS_IPV6; 3958 do_lro = 0; 3959 break; 3960 3961 case NDIS_HASH_IPV6_EX: 3962 hash_type = M_HASHTYPE_RSS_IPV6_EX; 3963 do_lro = 0; 3964 break; 3965 3966 case NDIS_HASH_TCP_IPV6: 3967 hash_type = M_HASHTYPE_RSS_TCP_IPV6; 3968 break; 3969 3970 case NDIS_HASH_TCP_IPV6_EX: 3971 hash_type = M_HASHTYPE_RSS_TCP_IPV6_EX; 3972 break; 3973 } 3974 } 3975 } else if (!is_vf) { 3976 m_new->m_pkthdr.flowid = rxr->hn_rx_idx; 3977 hash_type = M_HASHTYPE_OPAQUE; 3978 } 3979 M_HASHTYPE_SET(m_new, hash_type); 3980 3981 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 3982 if (hn_ifp != ifp) { 3983 const struct ether_header *eh; 3984 3985 /* 3986 * Non-transparent mode VF is activated. 3987 */ 3988 3989 /* 3990 * Allow tapping on hn(4). 3991 */ 3992 ETHER_BPF_MTAP(hn_ifp, m_new); 3993 3994 /* 3995 * Update hn(4)'s stats. 3996 */ 3997 if_inc_counter(hn_ifp, IFCOUNTER_IPACKETS, 1); 3998 if_inc_counter(hn_ifp, IFCOUNTER_IBYTES, m_new->m_pkthdr.len); 3999 /* Checked at the beginning of this function. */ 4000 KASSERT(m_new->m_len >= ETHER_HDR_LEN, ("not ethernet frame")); 4001 eh = mtod(m_new, struct ether_header *); 4002 if (ETHER_IS_MULTICAST(eh->ether_dhost)) 4003 if_inc_counter(hn_ifp, IFCOUNTER_IMCASTS, 1); 4004 } 4005 rxr->hn_pkts++; 4006 4007 if ((if_getcapenable(hn_ifp) & IFCAP_LRO) && do_lro) { 4008 #if defined(INET) || defined(INET6) 4009 struct lro_ctrl *lro = &rxr->hn_lro; 4010 4011 if (lro->lro_cnt) { 4012 rxr->hn_lro_tried++; 4013 if (hn_lro_rx(lro, m_new) == 0) { 4014 /* DONE! */ 4015 return 0; 4016 } 4017 } 4018 #endif 4019 } 4020 if_input(ifp, m_new); 4021 4022 return (0); 4023 } 4024 4025 static int 4026 hn_ioctl(if_t ifp, u_long cmd, caddr_t data) 4027 { 4028 struct hn_softc *sc = if_getsoftc(ifp); 4029 struct ifreq *ifr = (struct ifreq *)data, ifr_vf; 4030 if_t vf_ifp; 4031 int mask, error = 0; 4032 struct ifrsskey *ifrk; 4033 struct ifrsshash *ifrh; 4034 uint32_t mtu; 4035 4036 switch (cmd) { 4037 case SIOCSIFMTU: 4038 if (ifr->ifr_mtu > HN_MTU_MAX) { 4039 error = EINVAL; 4040 break; 4041 } 4042 4043 HN_LOCK(sc); 4044 4045 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 4046 HN_UNLOCK(sc); 4047 break; 4048 } 4049 4050 if ((sc->hn_caps & HN_CAP_MTU) == 0) { 4051 /* Can't change MTU */ 4052 HN_UNLOCK(sc); 4053 error = EOPNOTSUPP; 4054 break; 4055 } 4056 4057 if (if_getmtu(ifp) == ifr->ifr_mtu) { 4058 HN_UNLOCK(sc); 4059 break; 4060 } 4061 4062 if (hn_xpnt_vf_isready(sc)) { 4063 vf_ifp = sc->hn_vf_ifp; 4064 ifr_vf = *ifr; 4065 strlcpy(ifr_vf.ifr_name, if_name(vf_ifp), 4066 sizeof(ifr_vf.ifr_name)); 4067 error = ifhwioctl(SIOCSIFMTU, vf_ifp, 4068 (caddr_t)&ifr_vf, curthread); 4069 HN_UNLOCK(sc); 4070 if (error) { 4071 if_printf(ifp, "%s SIOCSIFMTU %d failed: %d\n", 4072 if_name(vf_ifp), ifr->ifr_mtu, error); 4073 } else { 4074 if_setmtu(ifp, ifr->ifr_mtu); 4075 } 4076 break; 4077 } 4078 4079 /* 4080 * Suspend this interface before the synthetic parts 4081 * are ripped. 4082 */ 4083 if (sc->hn_vf_ready) 4084 hn_xpnt_vf_deactivate(sc); 4085 if (sc->hn_flags & HN_FLAG_RXVF) { 4086 hn_rxvf_change_locked(sc, sc->hn_vf_ifp, false); 4087 if (sc->hn_flags & HN_FLAG_RXVF) { 4088 error = EIO; 4089 HN_UNLOCK(sc); 4090 break; 4091 } 4092 } 4093 hn_suspend(sc); 4094 4095 /* 4096 * Detach the synthetics parts, i.e. NVS and RNDIS. 4097 */ 4098 hn_synth_detach(sc); 4099 4100 /* 4101 * Reattach the synthetic parts, i.e. NVS and RNDIS, 4102 * with the new MTU setting. 4103 */ 4104 error = hn_synth_attach(sc, ifr->ifr_mtu); 4105 if (error) { 4106 HN_UNLOCK(sc); 4107 break; 4108 } 4109 4110 error = hn_rndis_get_mtu(sc, &mtu); 4111 if (error) 4112 mtu = ifr->ifr_mtu; 4113 else if (bootverbose) 4114 if_printf(ifp, "RNDIS mtu %u\n", mtu); 4115 4116 /* 4117 * Commit the requested MTU, after the synthetic parts 4118 * have been successfully attached. 4119 */ 4120 if (mtu >= ifr->ifr_mtu) { 4121 mtu = ifr->ifr_mtu; 4122 } else { 4123 if_printf(ifp, "fixup mtu %d -> %u\n", 4124 ifr->ifr_mtu, mtu); 4125 } 4126 if_setmtu(ifp, mtu); 4127 4128 /* 4129 * Synthetic parts' reattach may change the chimney 4130 * sending size; update it. 4131 */ 4132 if (sc->hn_tx_ring[0].hn_chim_size > sc->hn_chim_szmax) 4133 hn_set_chim_size(sc, sc->hn_chim_szmax); 4134 4135 /* 4136 * Make sure that various parameters based on MTU are 4137 * still valid, after the MTU change. 4138 */ 4139 hn_mtu_change_fixup(sc); 4140 4141 /* 4142 * All done! Resume the interface now. 4143 */ 4144 hn_resume(sc); 4145 4146 /* Reattach requires a fresh association and acknowledged switch. */ 4147 if (sc->hn_vf_ifp != NULL) 4148 taskqueue_enqueue_timeout(sc->hn_vf_taskq, 4149 &sc->hn_vf_init, 0); 4150 4151 HN_UNLOCK(sc); 4152 break; 4153 4154 case SIOCSIFFLAGS: 4155 HN_LOCK(sc); 4156 4157 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 4158 HN_UNLOCK(sc); 4159 break; 4160 } 4161 4162 if (hn_xpnt_vf_isready(sc)) 4163 hn_xpnt_vf_saveifflags(sc); 4164 4165 if (if_getflags(ifp) & IFF_UP) { 4166 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 4167 /* 4168 * Caller meight hold mutex, e.g. 4169 * bpf; use busy-wait for the RNDIS 4170 * reply. 4171 */ 4172 HN_NO_SLEEPING(sc); 4173 hn_rxfilter_config(sc); 4174 HN_SLEEPING_OK(sc); 4175 4176 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 4177 error = hn_xpnt_vf_iocsetflags(sc); 4178 } else { 4179 hn_init_locked(sc); 4180 } 4181 } else { 4182 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 4183 hn_stop(sc, false); 4184 } 4185 sc->hn_if_flags = if_getflags(ifp); 4186 4187 HN_UNLOCK(sc); 4188 break; 4189 4190 case SIOCSIFCAP: 4191 HN_LOCK(sc); 4192 if (sc->hn_detaching || sc->hn_vf_caps_busy) { 4193 error = sc->hn_vf_caps_busy ? EBUSY : ENXIO; 4194 HN_UNLOCK(sc); 4195 break; 4196 } 4197 4198 if (hn_xpnt_vf_isready(sc)) { 4199 ifr_vf = *ifr; 4200 strlcpy(ifr_vf.ifr_name, if_name(sc->hn_vf_ifp), 4201 sizeof(ifr_vf.ifr_name)); 4202 error = hn_xpnt_vf_iocsetcaps(sc, &ifr_vf); 4203 HN_UNLOCK(sc); 4204 break; 4205 } 4206 4207 /* 4208 * Fix up requested capabilities w/ supported capabilities, 4209 * since the supported capabilities could have been changed. 4210 */ 4211 mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ 4212 if_getcapenable(ifp); 4213 4214 if (mask & IFCAP_TXCSUM) { 4215 if_togglecapenable(ifp, IFCAP_TXCSUM); 4216 if (if_getcapenable(ifp) & IFCAP_TXCSUM) 4217 if_sethwassistbits(ifp, HN_CSUM_IP_HWASSIST(sc), 0); 4218 else 4219 if_sethwassistbits(ifp, 0, HN_CSUM_IP_HWASSIST(sc)); 4220 } 4221 if (mask & IFCAP_TXCSUM_IPV6) { 4222 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 4223 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) 4224 if_sethwassistbits(ifp, HN_CSUM_IP6_HWASSIST(sc), 0); 4225 else 4226 if_sethwassistbits(ifp, 0, HN_CSUM_IP6_HWASSIST(sc)); 4227 } 4228 4229 /* TODO: flip RNDIS offload parameters for RXCSUM. */ 4230 if (mask & IFCAP_RXCSUM) 4231 if_togglecapenable(ifp, IFCAP_RXCSUM); 4232 #ifdef foo 4233 /* We can't diff IPv6 packets from IPv4 packets on RX path. */ 4234 if (mask & IFCAP_RXCSUM_IPV6) 4235 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 4236 #endif 4237 4238 if (mask & IFCAP_LRO) 4239 if_togglecapenable(ifp, IFCAP_LRO); 4240 4241 if (mask & IFCAP_TSO4) { 4242 if_togglecapenable(ifp, IFCAP_TSO4); 4243 if (if_getcapenable(ifp) & IFCAP_TSO4) 4244 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 4245 else 4246 if_sethwassistbits(ifp, 0, CSUM_IP_TSO); 4247 } 4248 if (mask & IFCAP_TSO6) { 4249 if_togglecapenable(ifp, IFCAP_TSO6); 4250 if (if_getcapenable(ifp) & IFCAP_TSO6) 4251 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 4252 else 4253 if_sethwassistbits(ifp, 0, CSUM_IP6_TSO); 4254 } 4255 4256 HN_UNLOCK(sc); 4257 break; 4258 4259 case SIOCADDMULTI: 4260 case SIOCDELMULTI: 4261 HN_LOCK(sc); 4262 4263 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 4264 HN_UNLOCK(sc); 4265 break; 4266 } 4267 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 4268 /* 4269 * Multicast uses mutex; use busy-wait for 4270 * the RNDIS reply. 4271 */ 4272 HN_NO_SLEEPING(sc); 4273 hn_rxfilter_config(sc); 4274 HN_SLEEPING_OK(sc); 4275 } 4276 4277 /* XXX vlan(4) style mcast addr maintenance */ 4278 if (hn_xpnt_vf_isready(sc)) { 4279 int old_if_flags; 4280 4281 old_if_flags = if_getflags(sc->hn_vf_ifp); 4282 hn_xpnt_vf_saveifflags(sc); 4283 4284 if ((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) && 4285 ((old_if_flags ^ if_getflags(sc->hn_vf_ifp)) & 4286 IFF_ALLMULTI)) 4287 error = hn_xpnt_vf_iocsetflags(sc); 4288 } 4289 4290 HN_UNLOCK(sc); 4291 break; 4292 4293 case SIOCSIFMEDIA: 4294 case SIOCGIFMEDIA: 4295 HN_LOCK(sc); 4296 if (hn_xpnt_vf_isready(sc)) { 4297 /* 4298 * SIOCGIFMEDIA expects ifmediareq, so don't 4299 * create and pass ifr_vf to the VF here; just 4300 * replace the ifr_name. 4301 */ 4302 vf_ifp = sc->hn_vf_ifp; 4303 strlcpy(ifr->ifr_name, if_name(vf_ifp), 4304 sizeof(ifr->ifr_name)); 4305 error = ifhwioctl(cmd, vf_ifp, data, curthread); 4306 /* Restore the ifr_name. */ 4307 strlcpy(ifr->ifr_name, if_name(ifp), 4308 sizeof(ifr->ifr_name)); 4309 HN_UNLOCK(sc); 4310 break; 4311 } 4312 HN_UNLOCK(sc); 4313 error = ifmedia_ioctl(ifp, ifr, &sc->hn_media, cmd); 4314 break; 4315 4316 case SIOCGIFRSSHASH: 4317 ifrh = (struct ifrsshash *)data; 4318 HN_LOCK(sc); 4319 if (sc->hn_rx_ring_inuse == 1) { 4320 HN_UNLOCK(sc); 4321 ifrh->ifrh_func = RSS_FUNC_NONE; 4322 ifrh->ifrh_types = 0; 4323 break; 4324 } 4325 4326 if (sc->hn_rss_hash & NDIS_HASH_FUNCTION_TOEPLITZ) 4327 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ; 4328 else 4329 ifrh->ifrh_func = RSS_FUNC_PRIVATE; 4330 ifrh->ifrh_types = hn_rss_type_fromndis(sc->hn_rss_hash); 4331 HN_UNLOCK(sc); 4332 break; 4333 4334 case SIOCGIFRSSKEY: 4335 ifrk = (struct ifrsskey *)data; 4336 HN_LOCK(sc); 4337 if (sc->hn_rx_ring_inuse == 1) { 4338 HN_UNLOCK(sc); 4339 ifrk->ifrk_func = RSS_FUNC_NONE; 4340 ifrk->ifrk_keylen = 0; 4341 break; 4342 } 4343 if (sc->hn_rss_hash & NDIS_HASH_FUNCTION_TOEPLITZ) 4344 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ; 4345 else 4346 ifrk->ifrk_func = RSS_FUNC_PRIVATE; 4347 ifrk->ifrk_keylen = NDIS_HASH_KEYSIZE_TOEPLITZ; 4348 memcpy(ifrk->ifrk_key, sc->hn_rss.rss_key, 4349 NDIS_HASH_KEYSIZE_TOEPLITZ); 4350 HN_UNLOCK(sc); 4351 break; 4352 4353 default: 4354 error = ether_ioctl(ifp, cmd, data); 4355 break; 4356 } 4357 return (error); 4358 } 4359 4360 static void 4361 hn_stop(struct hn_softc *sc, bool detaching) 4362 { 4363 if_t ifp = sc->hn_ifp; 4364 int i; 4365 4366 HN_LOCK_ASSERT(sc); 4367 4368 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 4369 ("synthetic parts were not attached")); 4370 4371 /* Clear RUNNING bit ASAP. */ 4372 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 4373 4374 /* Disable polling. */ 4375 hn_polling(sc, 0); 4376 4377 if (sc->hn_vf_ready || (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 4378 KASSERT(sc->hn_vf_ifp != NULL, 4379 ("%s: VF is not attached", if_name(ifp))); 4380 4381 /* 4382 * NOTE: 4383 * Datapath setting must happen _before_ bringing 4384 * the VF down. 4385 */ 4386 hn_xpnt_vf_deactivate(sc); 4387 4388 /* 4389 * Bring the VF down. 4390 */ 4391 hn_xpnt_vf_saveifflags(sc); 4392 if_setflagbits(ifp, 0, IFF_UP); 4393 hn_xpnt_vf_iocsetflags(sc); 4394 } 4395 4396 /* Suspend data transfers. */ 4397 hn_suspend_data(sc); 4398 4399 /* Clear OACTIVE bit. */ 4400 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 4401 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 4402 sc->hn_tx_ring[i].hn_oactive = 0; 4403 4404 /* 4405 * If the non-transparent mode VF is active, make sure 4406 * that the RX filter still allows packet reception. 4407 */ 4408 if (!detaching && (sc->hn_flags & HN_FLAG_RXVF)) 4409 hn_rxfilter_config(sc); 4410 } 4411 4412 static void 4413 hn_init_locked(struct hn_softc *sc) 4414 { 4415 if_t ifp = sc->hn_ifp; 4416 int i; 4417 4418 HN_LOCK_ASSERT(sc); 4419 4420 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 4421 return; 4422 4423 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 4424 return; 4425 4426 /* Configure RX filter */ 4427 hn_rxfilter_config(sc); 4428 4429 /* Clear OACTIVE bit. */ 4430 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 4431 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 4432 sc->hn_tx_ring[i].hn_oactive = 0; 4433 4434 /* Clear TX 'suspended' bit. */ 4435 hn_resume_tx(sc, sc->hn_tx_ring_inuse); 4436 4437 if (hn_xpnt_vf_caninit(sc)) { 4438 /* Initialize transparent VF. */ 4439 hn_xpnt_vf_init(sc); 4440 } 4441 4442 /* Everything is ready; unleash! */ 4443 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 4444 4445 /* Re-enable polling if requested. */ 4446 if (sc->hn_pollhz > 0) 4447 hn_polling(sc, sc->hn_pollhz); 4448 } 4449 4450 static void 4451 hn_init(void *xsc) 4452 { 4453 struct hn_softc *sc = xsc; 4454 4455 HN_LOCK(sc); 4456 hn_init_locked(sc); 4457 HN_UNLOCK(sc); 4458 } 4459 4460 static int 4461 hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS) 4462 { 4463 struct hn_softc *sc = arg1; 4464 unsigned int lenlim; 4465 int error; 4466 4467 lenlim = sc->hn_rx_ring[0].hn_lro.lro_length_lim; 4468 error = sysctl_handle_int(oidp, &lenlim, 0, req); 4469 if (error || req->newptr == NULL) 4470 return error; 4471 4472 HN_LOCK(sc); 4473 if (lenlim < HN_LRO_LENLIM_MIN(sc->hn_ifp) || 4474 lenlim > TCP_LRO_LENGTH_MAX) { 4475 HN_UNLOCK(sc); 4476 return EINVAL; 4477 } 4478 hn_set_lro_lenlim(sc, lenlim); 4479 HN_UNLOCK(sc); 4480 4481 return 0; 4482 } 4483 4484 static int 4485 hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS) 4486 { 4487 struct hn_softc *sc = arg1; 4488 int ackcnt, error, i; 4489 4490 /* 4491 * lro_ackcnt_lim is append count limit, 4492 * +1 to turn it into aggregation limit. 4493 */ 4494 ackcnt = sc->hn_rx_ring[0].hn_lro.lro_ackcnt_lim + 1; 4495 error = sysctl_handle_int(oidp, &ackcnt, 0, req); 4496 if (error || req->newptr == NULL) 4497 return error; 4498 4499 if (ackcnt < 2 || ackcnt > (TCP_LRO_ACKCNT_MAX + 1)) 4500 return EINVAL; 4501 4502 /* 4503 * Convert aggregation limit back to append 4504 * count limit. 4505 */ 4506 --ackcnt; 4507 HN_LOCK(sc); 4508 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 4509 sc->hn_rx_ring[i].hn_lro.lro_ackcnt_lim = ackcnt; 4510 HN_UNLOCK(sc); 4511 return 0; 4512 } 4513 4514 static int 4515 hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS) 4516 { 4517 struct hn_softc *sc = arg1; 4518 int hcsum = arg2; 4519 int on, error, i; 4520 4521 on = 0; 4522 if (sc->hn_rx_ring[0].hn_trust_hcsum & hcsum) 4523 on = 1; 4524 4525 error = sysctl_handle_int(oidp, &on, 0, req); 4526 if (error || req->newptr == NULL) 4527 return error; 4528 4529 HN_LOCK(sc); 4530 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4531 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 4532 4533 if (on) 4534 rxr->hn_trust_hcsum |= hcsum; 4535 else 4536 rxr->hn_trust_hcsum &= ~hcsum; 4537 } 4538 HN_UNLOCK(sc); 4539 return 0; 4540 } 4541 4542 static int 4543 hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS) 4544 { 4545 struct hn_softc *sc = arg1; 4546 int chim_size, error; 4547 4548 chim_size = sc->hn_tx_ring[0].hn_chim_size; 4549 error = sysctl_handle_int(oidp, &chim_size, 0, req); 4550 if (error || req->newptr == NULL) 4551 return error; 4552 4553 if (chim_size > sc->hn_chim_szmax || chim_size <= 0) 4554 return EINVAL; 4555 4556 HN_LOCK(sc); 4557 hn_set_chim_size(sc, chim_size); 4558 HN_UNLOCK(sc); 4559 return 0; 4560 } 4561 4562 static int 4563 hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS) 4564 { 4565 struct hn_softc *sc = arg1; 4566 int ofs = arg2, i, error; 4567 struct hn_rx_ring *rxr; 4568 uint64_t stat; 4569 4570 stat = 0; 4571 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4572 rxr = &sc->hn_rx_ring[i]; 4573 stat += *((uint64_t *)((uint8_t *)rxr + ofs)); 4574 } 4575 4576 error = sysctl_handle_64(oidp, &stat, 0, req); 4577 if (error || req->newptr == NULL) 4578 return error; 4579 4580 /* Zero out this stat. */ 4581 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4582 rxr = &sc->hn_rx_ring[i]; 4583 *((uint64_t *)((uint8_t *)rxr + ofs)) = 0; 4584 } 4585 return 0; 4586 } 4587 4588 static int 4589 hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) 4590 { 4591 struct hn_softc *sc = arg1; 4592 int ofs = arg2, i, error; 4593 struct hn_rx_ring *rxr; 4594 u_long stat; 4595 4596 stat = 0; 4597 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4598 rxr = &sc->hn_rx_ring[i]; 4599 stat += *((u_long *)((uint8_t *)rxr + ofs)); 4600 } 4601 4602 error = sysctl_handle_long(oidp, &stat, 0, req); 4603 if (error || req->newptr == NULL) 4604 return error; 4605 4606 /* Zero out this stat. */ 4607 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4608 rxr = &sc->hn_rx_ring[i]; 4609 *((u_long *)((uint8_t *)rxr + ofs)) = 0; 4610 } 4611 return 0; 4612 } 4613 4614 static int 4615 hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) 4616 { 4617 struct hn_softc *sc = arg1; 4618 int ofs = arg2, i, error; 4619 struct hn_tx_ring *txr; 4620 u_long stat; 4621 4622 stat = 0; 4623 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4624 txr = &sc->hn_tx_ring[i]; 4625 stat += *((u_long *)((uint8_t *)txr + ofs)); 4626 } 4627 4628 error = sysctl_handle_long(oidp, &stat, 0, req); 4629 if (error || req->newptr == NULL) 4630 return error; 4631 4632 /* Zero out this stat. */ 4633 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4634 txr = &sc->hn_tx_ring[i]; 4635 *((u_long *)((uint8_t *)txr + ofs)) = 0; 4636 } 4637 return 0; 4638 } 4639 4640 static int 4641 hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS) 4642 { 4643 struct hn_softc *sc = arg1; 4644 int ofs = arg2, i, error, conf; 4645 struct hn_tx_ring *txr; 4646 4647 txr = &sc->hn_tx_ring[0]; 4648 conf = *((int *)((uint8_t *)txr + ofs)); 4649 4650 error = sysctl_handle_int(oidp, &conf, 0, req); 4651 if (error || req->newptr == NULL) 4652 return error; 4653 4654 HN_LOCK(sc); 4655 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4656 txr = &sc->hn_tx_ring[i]; 4657 *((int *)((uint8_t *)txr + ofs)) = conf; 4658 } 4659 HN_UNLOCK(sc); 4660 4661 return 0; 4662 } 4663 4664 static int 4665 hn_txagg_size_sysctl(SYSCTL_HANDLER_ARGS) 4666 { 4667 struct hn_softc *sc = arg1; 4668 int error, size; 4669 4670 size = sc->hn_agg_size; 4671 error = sysctl_handle_int(oidp, &size, 0, req); 4672 if (error || req->newptr == NULL) 4673 return (error); 4674 4675 HN_LOCK(sc); 4676 sc->hn_agg_size = size; 4677 hn_set_txagg(sc); 4678 HN_UNLOCK(sc); 4679 4680 return (0); 4681 } 4682 4683 static int 4684 hn_txagg_pkts_sysctl(SYSCTL_HANDLER_ARGS) 4685 { 4686 struct hn_softc *sc = arg1; 4687 int error, pkts; 4688 4689 pkts = sc->hn_agg_pkts; 4690 error = sysctl_handle_int(oidp, &pkts, 0, req); 4691 if (error || req->newptr == NULL) 4692 return (error); 4693 4694 HN_LOCK(sc); 4695 sc->hn_agg_pkts = pkts; 4696 hn_set_txagg(sc); 4697 HN_UNLOCK(sc); 4698 4699 return (0); 4700 } 4701 4702 static int 4703 hn_txagg_pktmax_sysctl(SYSCTL_HANDLER_ARGS) 4704 { 4705 struct hn_softc *sc = arg1; 4706 int pkts; 4707 4708 pkts = sc->hn_tx_ring[0].hn_agg_pktmax; 4709 return (sysctl_handle_int(oidp, &pkts, 0, req)); 4710 } 4711 4712 static int 4713 hn_txagg_align_sysctl(SYSCTL_HANDLER_ARGS) 4714 { 4715 struct hn_softc *sc = arg1; 4716 int align; 4717 4718 align = sc->hn_tx_ring[0].hn_agg_align; 4719 return (sysctl_handle_int(oidp, &align, 0, req)); 4720 } 4721 4722 static void 4723 hn_chan_polling(struct vmbus_channel *chan, u_int pollhz) 4724 { 4725 if (pollhz == 0) 4726 vmbus_chan_poll_disable(chan); 4727 else 4728 vmbus_chan_poll_enable(chan, pollhz); 4729 } 4730 4731 static void 4732 hn_polling(struct hn_softc *sc, u_int pollhz) 4733 { 4734 int nsubch = sc->hn_rx_ring_inuse - 1; 4735 4736 HN_LOCK_ASSERT(sc); 4737 4738 if (nsubch > 0) { 4739 struct vmbus_channel **subch; 4740 int i; 4741 4742 subch = vmbus_subchan_get(sc->hn_prichan, nsubch); 4743 for (i = 0; i < nsubch; ++i) 4744 hn_chan_polling(subch[i], pollhz); 4745 vmbus_subchan_rel(subch, nsubch); 4746 } 4747 hn_chan_polling(sc->hn_prichan, pollhz); 4748 } 4749 4750 static int 4751 hn_polling_sysctl(SYSCTL_HANDLER_ARGS) 4752 { 4753 struct hn_softc *sc = arg1; 4754 int pollhz, error; 4755 4756 pollhz = sc->hn_pollhz; 4757 error = sysctl_handle_int(oidp, &pollhz, 0, req); 4758 if (error || req->newptr == NULL) 4759 return (error); 4760 4761 if (pollhz != 0 && 4762 (pollhz < VMBUS_CHAN_POLLHZ_MIN || pollhz > VMBUS_CHAN_POLLHZ_MAX)) 4763 return (EINVAL); 4764 4765 HN_LOCK(sc); 4766 if (sc->hn_pollhz != pollhz) { 4767 sc->hn_pollhz = pollhz; 4768 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) && 4769 (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 4770 hn_polling(sc, sc->hn_pollhz); 4771 } 4772 HN_UNLOCK(sc); 4773 4774 return (0); 4775 } 4776 4777 static int 4778 hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS) 4779 { 4780 struct hn_softc *sc = arg1; 4781 char verstr[16]; 4782 4783 snprintf(verstr, sizeof(verstr), "%u.%u", 4784 HN_NDIS_VERSION_MAJOR(sc->hn_ndis_ver), 4785 HN_NDIS_VERSION_MINOR(sc->hn_ndis_ver)); 4786 return sysctl_handle_string(oidp, verstr, sizeof(verstr), req); 4787 } 4788 4789 static int 4790 hn_caps_sysctl(SYSCTL_HANDLER_ARGS) 4791 { 4792 struct hn_softc *sc = arg1; 4793 char caps_str[128]; 4794 uint32_t caps; 4795 4796 HN_LOCK(sc); 4797 caps = sc->hn_caps; 4798 HN_UNLOCK(sc); 4799 snprintf(caps_str, sizeof(caps_str), "%b", caps, HN_CAP_BITS); 4800 return sysctl_handle_string(oidp, caps_str, sizeof(caps_str), req); 4801 } 4802 4803 static int 4804 hn_hwassist_sysctl(SYSCTL_HANDLER_ARGS) 4805 { 4806 struct hn_softc *sc = arg1; 4807 char assist_str[128]; 4808 uint32_t hwassist; 4809 4810 HN_LOCK(sc); 4811 hwassist = if_gethwassist(sc->hn_ifp); 4812 HN_UNLOCK(sc); 4813 snprintf(assist_str, sizeof(assist_str), "%b", hwassist, CSUM_BITS); 4814 return sysctl_handle_string(oidp, assist_str, sizeof(assist_str), req); 4815 } 4816 4817 static int 4818 hn_rxfilter_sysctl(SYSCTL_HANDLER_ARGS) 4819 { 4820 struct hn_softc *sc = arg1; 4821 char filter_str[128]; 4822 uint32_t filter; 4823 4824 HN_LOCK(sc); 4825 filter = sc->hn_rx_filter; 4826 HN_UNLOCK(sc); 4827 snprintf(filter_str, sizeof(filter_str), "%b", filter, 4828 NDIS_PACKET_TYPES); 4829 return sysctl_handle_string(oidp, filter_str, sizeof(filter_str), req); 4830 } 4831 4832 static int 4833 hn_rsc_sysctl(SYSCTL_HANDLER_ARGS) 4834 { 4835 struct hn_softc *sc = arg1; 4836 int rsc_ctrl, mtu; 4837 int error; 4838 4839 rsc_ctrl = sc->hn_rsc_ctrl; 4840 error = sysctl_handle_int(oidp, &rsc_ctrl, 0, req); 4841 if (error || req->newptr == NULL) 4842 return (error); 4843 4844 if (sc->hn_rsc_ctrl != rsc_ctrl) { 4845 HN_LOCK(sc); 4846 sc->hn_rsc_ctrl = rsc_ctrl; 4847 mtu = if_getmtu(sc->hn_ifp); 4848 error = hn_rndis_reconf_offload(sc, mtu); 4849 HN_UNLOCK(sc); 4850 } 4851 4852 return (error); 4853 } 4854 #ifndef RSS 4855 4856 static int 4857 hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS) 4858 { 4859 struct hn_softc *sc = arg1; 4860 int error; 4861 4862 HN_LOCK(sc); 4863 4864 error = SYSCTL_OUT(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key)); 4865 if (error || req->newptr == NULL) 4866 goto back; 4867 4868 if ((sc->hn_flags & HN_FLAG_RXVF) || 4869 (hn_xpnt_vf && sc->hn_vf_ifp != NULL)) { 4870 /* 4871 * RSS key is synchronized w/ VF's, don't allow users 4872 * to change it. 4873 */ 4874 error = EBUSY; 4875 goto back; 4876 } 4877 4878 error = SYSCTL_IN(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key)); 4879 if (error) 4880 goto back; 4881 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 4882 4883 if (sc->hn_rx_ring_inuse > 1) { 4884 error = hn_rss_reconfig(sc); 4885 } else { 4886 /* Not RSS capable, at least for now; just save the RSS key. */ 4887 error = 0; 4888 } 4889 back: 4890 HN_UNLOCK(sc); 4891 return (error); 4892 } 4893 4894 static int 4895 hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS) 4896 { 4897 struct hn_softc *sc = arg1; 4898 int error; 4899 4900 HN_LOCK(sc); 4901 4902 error = SYSCTL_OUT(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind)); 4903 if (error || req->newptr == NULL) 4904 goto back; 4905 4906 /* 4907 * Don't allow RSS indirect table change, if this interface is not 4908 * RSS capable currently. 4909 */ 4910 if (sc->hn_rx_ring_inuse == 1) { 4911 error = EOPNOTSUPP; 4912 goto back; 4913 } 4914 4915 error = SYSCTL_IN(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind)); 4916 if (error) 4917 goto back; 4918 sc->hn_flags |= HN_FLAG_HAS_RSSIND; 4919 4920 hn_rss_ind_fixup(sc); 4921 error = hn_rss_reconfig(sc); 4922 back: 4923 HN_UNLOCK(sc); 4924 return (error); 4925 } 4926 4927 #endif /* !RSS */ 4928 4929 static int 4930 hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS) 4931 { 4932 struct hn_softc *sc = arg1; 4933 char hash_str[128]; 4934 uint32_t hash; 4935 4936 HN_LOCK(sc); 4937 hash = sc->hn_rss_hash; 4938 HN_UNLOCK(sc); 4939 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4940 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4941 } 4942 4943 static int 4944 hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS) 4945 { 4946 struct hn_softc *sc = arg1; 4947 char hash_str[128]; 4948 uint32_t hash; 4949 4950 HN_LOCK(sc); 4951 hash = sc->hn_rss_hcap; 4952 HN_UNLOCK(sc); 4953 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4954 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4955 } 4956 4957 static int 4958 hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS) 4959 { 4960 struct hn_softc *sc = arg1; 4961 char hash_str[128]; 4962 uint32_t hash; 4963 4964 HN_LOCK(sc); 4965 hash = sc->hn_rx_ring[0].hn_mbuf_hash; 4966 HN_UNLOCK(sc); 4967 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4968 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4969 } 4970 4971 static int 4972 hn_vf_sysctl(SYSCTL_HANDLER_ARGS) 4973 { 4974 struct hn_softc *sc = arg1; 4975 char vf_name[IFNAMSIZ + 1]; 4976 if_t vf_ifp; 4977 4978 HN_LOCK(sc); 4979 vf_name[0] = '\0'; 4980 vf_ifp = sc->hn_vf_ifp; 4981 if (vf_ifp != NULL) 4982 snprintf(vf_name, sizeof(vf_name), "%s", if_name(vf_ifp)); 4983 HN_UNLOCK(sc); 4984 return sysctl_handle_string(oidp, vf_name, sizeof(vf_name), req); 4985 } 4986 4987 static int 4988 hn_rxvf_sysctl(SYSCTL_HANDLER_ARGS) 4989 { 4990 struct hn_softc *sc = arg1; 4991 char vf_name[IFNAMSIZ + 1]; 4992 if_t vf_ifp; 4993 4994 HN_LOCK(sc); 4995 vf_name[0] = '\0'; 4996 vf_ifp = sc->hn_rx_ring[0].hn_rxvf_ifp; 4997 if (vf_ifp != NULL) 4998 snprintf(vf_name, sizeof(vf_name), "%s", if_name(vf_ifp)); 4999 HN_UNLOCK(sc); 5000 return sysctl_handle_string(oidp, vf_name, sizeof(vf_name), req); 5001 } 5002 5003 static int 5004 hn_vflist_sysctl(SYSCTL_HANDLER_ARGS) 5005 { 5006 struct rm_priotracker pt; 5007 struct sbuf *sb; 5008 int error, i; 5009 bool first; 5010 5011 error = sysctl_wire_old_buffer(req, 0); 5012 if (error != 0) 5013 return (error); 5014 5015 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5016 if (sb == NULL) 5017 return (ENOMEM); 5018 5019 rm_rlock(&hn_vfmap_lock, &pt); 5020 5021 first = true; 5022 for (i = 0; i < hn_vfmap_size; ++i) { 5023 struct epoch_tracker et; 5024 if_t ifp; 5025 5026 if (hn_vfmap[i] == NULL) 5027 continue; 5028 5029 NET_EPOCH_ENTER(et); 5030 ifp = ifnet_byindex(i); 5031 if (ifp != NULL) { 5032 if (first) 5033 sbuf_printf(sb, "%s", if_name(ifp)); 5034 else 5035 sbuf_printf(sb, " %s", if_name(ifp)); 5036 first = false; 5037 } 5038 NET_EPOCH_EXIT(et); 5039 } 5040 5041 rm_runlock(&hn_vfmap_lock, &pt); 5042 5043 error = sbuf_finish(sb); 5044 sbuf_delete(sb); 5045 return (error); 5046 } 5047 5048 static int 5049 hn_vfmap_sysctl(SYSCTL_HANDLER_ARGS) 5050 { 5051 struct rm_priotracker pt; 5052 struct sbuf *sb; 5053 int error, i; 5054 bool first; 5055 5056 error = sysctl_wire_old_buffer(req, 0); 5057 if (error != 0) 5058 return (error); 5059 5060 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5061 if (sb == NULL) 5062 return (ENOMEM); 5063 5064 rm_rlock(&hn_vfmap_lock, &pt); 5065 5066 first = true; 5067 for (i = 0; i < hn_vfmap_size; ++i) { 5068 struct epoch_tracker et; 5069 if_t ifp, hn_ifp; 5070 5071 hn_ifp = hn_vfmap[i]; 5072 if (hn_ifp == NULL) 5073 continue; 5074 5075 NET_EPOCH_ENTER(et); 5076 ifp = ifnet_byindex(i); 5077 if (ifp != NULL) { 5078 if (first) { 5079 sbuf_printf(sb, "%s:%s", if_name(ifp), 5080 if_name(hn_ifp)); 5081 } else { 5082 sbuf_printf(sb, " %s:%s", if_name(ifp), 5083 if_name(hn_ifp)); 5084 } 5085 first = false; 5086 } 5087 NET_EPOCH_EXIT(et); 5088 } 5089 5090 rm_runlock(&hn_vfmap_lock, &pt); 5091 5092 error = sbuf_finish(sb); 5093 sbuf_delete(sb); 5094 return (error); 5095 } 5096 5097 static int 5098 hn_xpnt_vf_accbpf_sysctl(SYSCTL_HANDLER_ARGS) 5099 { 5100 struct hn_softc *sc = arg1; 5101 int error, onoff = 0; 5102 5103 if (sc->hn_xvf_flags & HN_XVFFLAG_ACCBPF) 5104 onoff = 1; 5105 error = sysctl_handle_int(oidp, &onoff, 0, req); 5106 if (error || req->newptr == NULL) 5107 return (error); 5108 5109 HN_LOCK(sc); 5110 /* NOTE: hn_vf_lock for hn_transmit() */ 5111 rm_wlock(&sc->hn_vf_lock); 5112 if (onoff) 5113 sc->hn_xvf_flags |= HN_XVFFLAG_ACCBPF; 5114 else 5115 sc->hn_xvf_flags &= ~HN_XVFFLAG_ACCBPF; 5116 rm_wunlock(&sc->hn_vf_lock); 5117 HN_UNLOCK(sc); 5118 5119 return (0); 5120 } 5121 5122 static int 5123 hn_xpnt_vf_enabled_sysctl(SYSCTL_HANDLER_ARGS) 5124 { 5125 struct hn_softc *sc = arg1; 5126 int enabled = 0; 5127 5128 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 5129 enabled = 1; 5130 return (sysctl_handle_int(oidp, &enabled, 0, req)); 5131 } 5132 5133 static int 5134 hn_check_iplen(const struct mbuf *m, int hoff) 5135 { 5136 const struct ip *ip; 5137 int len, iphlen, iplen; 5138 const struct tcphdr *th; 5139 int thoff; /* TCP data offset */ 5140 5141 len = hoff + sizeof(struct ip); 5142 5143 /* The packet must be at least the size of an IP header. */ 5144 if (m->m_pkthdr.len < len) 5145 return IPPROTO_DONE; 5146 5147 /* The fixed IP header must reside completely in the first mbuf. */ 5148 if (m->m_len < len) 5149 return IPPROTO_DONE; 5150 5151 ip = mtodo(m, hoff); 5152 5153 /* Bound check the packet's stated IP header length. */ 5154 iphlen = ip->ip_hl << 2; 5155 if (iphlen < sizeof(struct ip)) /* minimum header length */ 5156 return IPPROTO_DONE; 5157 5158 /* The full IP header must reside completely in the one mbuf. */ 5159 if (m->m_len < hoff + iphlen) 5160 return IPPROTO_DONE; 5161 5162 iplen = ntohs(ip->ip_len); 5163 5164 /* 5165 * Check that the amount of data in the buffers is as 5166 * at least much as the IP header would have us expect. 5167 */ 5168 if (m->m_pkthdr.len < hoff + iplen) 5169 return IPPROTO_DONE; 5170 5171 /* 5172 * Ignore IP fragments. 5173 */ 5174 if (ntohs(ip->ip_off) & (IP_OFFMASK | IP_MF)) 5175 return IPPROTO_DONE; 5176 5177 /* 5178 * The TCP/IP or UDP/IP header must be entirely contained within 5179 * the first fragment of a packet. 5180 */ 5181 switch (ip->ip_p) { 5182 case IPPROTO_TCP: 5183 if (iplen < iphlen + sizeof(struct tcphdr)) 5184 return IPPROTO_DONE; 5185 if (m->m_len < hoff + iphlen + sizeof(struct tcphdr)) 5186 return IPPROTO_DONE; 5187 th = (const struct tcphdr *)((const uint8_t *)ip + iphlen); 5188 thoff = th->th_off << 2; 5189 if (thoff < sizeof(struct tcphdr) || thoff + iphlen > iplen) 5190 return IPPROTO_DONE; 5191 if (m->m_len < hoff + iphlen + thoff) 5192 return IPPROTO_DONE; 5193 break; 5194 case IPPROTO_UDP: 5195 if (iplen < iphlen + sizeof(struct udphdr)) 5196 return IPPROTO_DONE; 5197 if (m->m_len < hoff + iphlen + sizeof(struct udphdr)) 5198 return IPPROTO_DONE; 5199 break; 5200 default: 5201 if (iplen < iphlen) 5202 return IPPROTO_DONE; 5203 break; 5204 } 5205 return ip->ip_p; 5206 } 5207 5208 static void 5209 hn_rxpkt_proto(const struct mbuf *m_new, int *l3proto, int *l4proto) 5210 { 5211 const struct ether_header *eh; 5212 uint16_t etype; 5213 int hoff; 5214 5215 hoff = sizeof(*eh); 5216 /* Checked at the beginning of this function. */ 5217 KASSERT(m_new->m_len >= hoff, ("not ethernet frame")); 5218 5219 eh = mtod(m_new, const struct ether_header *); 5220 etype = ntohs(eh->ether_type); 5221 if (etype == ETHERTYPE_VLAN) { 5222 const struct ether_vlan_header *evl; 5223 5224 hoff = sizeof(*evl); 5225 if (m_new->m_len < hoff) 5226 return; 5227 evl = mtod(m_new, const struct ether_vlan_header *); 5228 etype = ntohs(evl->evl_proto); 5229 } 5230 *l3proto = etype; 5231 5232 if (etype == ETHERTYPE_IP) 5233 *l4proto = hn_check_iplen(m_new, hoff); 5234 else 5235 *l4proto = IPPROTO_DONE; 5236 } 5237 5238 static int 5239 hn_create_rx_data(struct hn_softc *sc, int ring_cnt) 5240 { 5241 struct sysctl_oid_list *child; 5242 struct sysctl_ctx_list *ctx; 5243 device_t dev = sc->hn_dev; 5244 #if defined(INET) || defined(INET6) 5245 int lroent_cnt; 5246 #endif 5247 int i; 5248 5249 /* 5250 * Create RXBUF for reception. 5251 * 5252 * NOTE: 5253 * - It is shared by all channels. 5254 * - A large enough buffer is allocated, certain version of NVSes 5255 * may further limit the usable space. 5256 */ 5257 sc->hn_rxbuf = contigmalloc(HN_RXBUF_SIZE, M_DEVBUF, M_WAITOK | M_ZERO, 5258 0ul, ~0ul, PAGE_SIZE, 0); 5259 if (sc->hn_rxbuf == NULL) { 5260 device_printf(sc->hn_dev, "allocate rxbuf failed\n"); 5261 return (ENOMEM); 5262 } 5263 5264 sc->hn_rx_ring_cnt = ring_cnt; 5265 sc->hn_rx_ring_inuse = sc->hn_rx_ring_cnt; 5266 5267 sc->hn_rx_ring = malloc(sizeof(struct hn_rx_ring) * sc->hn_rx_ring_cnt, 5268 M_DEVBUF, M_WAITOK | M_ZERO); 5269 5270 #if defined(INET) || defined(INET6) 5271 lroent_cnt = hn_lro_entry_count; 5272 if (lroent_cnt < TCP_LRO_ENTRIES) 5273 lroent_cnt = TCP_LRO_ENTRIES; 5274 if (bootverbose) 5275 device_printf(dev, "LRO: entry count %d\n", lroent_cnt); 5276 #endif /* INET || INET6 */ 5277 5278 ctx = device_get_sysctl_ctx(dev); 5279 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 5280 5281 /* Create dev.hn.UNIT.rx sysctl tree */ 5282 sc->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "rx", 5283 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5284 5285 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 5286 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 5287 5288 rxr->hn_br = contigmalloc(HN_TXBR_SIZE + HN_RXBR_SIZE, M_DEVBUF, 5289 M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0); 5290 if (rxr->hn_br == NULL) { 5291 device_printf(dev, "allocate bufring failed\n"); 5292 return (ENOMEM); 5293 } 5294 5295 if (hn_trust_hosttcp) 5296 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_TCP; 5297 if (hn_trust_hostudp) 5298 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_UDP; 5299 if (hn_trust_hostip) 5300 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_IP; 5301 rxr->hn_mbuf_hash = NDIS_HASH_ALL; 5302 rxr->hn_ifp = sc->hn_ifp; 5303 if (i < sc->hn_tx_ring_cnt) 5304 rxr->hn_txr = &sc->hn_tx_ring[i]; 5305 rxr->hn_pktbuf_len = HN_PKTBUF_LEN_DEF; 5306 rxr->hn_pktbuf = malloc(rxr->hn_pktbuf_len, M_DEVBUF, M_WAITOK); 5307 rxr->hn_rx_idx = i; 5308 rxr->hn_rxbuf = sc->hn_rxbuf; 5309 5310 /* 5311 * Initialize LRO. 5312 */ 5313 #if defined(INET) || defined(INET6) 5314 tcp_lro_init_args(&rxr->hn_lro, sc->hn_ifp, lroent_cnt, 5315 hn_lro_mbufq_depth); 5316 rxr->hn_lro.lro_length_lim = HN_LRO_LENLIM_DEF; 5317 rxr->hn_lro.lro_ackcnt_lim = HN_LRO_ACKCNT_DEF; 5318 #endif /* INET || INET6 */ 5319 5320 if (sc->hn_rx_sysctl_tree != NULL) { 5321 char name[16]; 5322 5323 /* 5324 * Create per RX ring sysctl tree: 5325 * dev.hn.UNIT.rx.RINGID 5326 */ 5327 snprintf(name, sizeof(name), "%d", i); 5328 rxr->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, 5329 SYSCTL_CHILDREN(sc->hn_rx_sysctl_tree), 5330 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5331 5332 if (rxr->hn_rx_sysctl_tree != NULL) { 5333 SYSCTL_ADD_ULONG(ctx, 5334 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5335 OID_AUTO, "packets", 5336 CTLFLAG_RW | CTLFLAG_STATS, &rxr->hn_pkts, 5337 "# of packets received"); 5338 SYSCTL_ADD_ULONG(ctx, 5339 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5340 OID_AUTO, "rss_pkts", 5341 CTLFLAG_RW | CTLFLAG_STATS, 5342 &rxr->hn_rss_pkts, 5343 "# of packets w/ RSS info received"); 5344 SYSCTL_ADD_ULONG(ctx, 5345 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5346 OID_AUTO, "rsc_pkts", 5347 CTLFLAG_RW | CTLFLAG_STATS, 5348 &rxr->hn_rsc_pkts, 5349 "# of RSC packets received"); 5350 SYSCTL_ADD_ULONG(ctx, 5351 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5352 OID_AUTO, "rsc_drop", 5353 CTLFLAG_RW | CTLFLAG_STATS, 5354 &rxr->hn_rsc_drop, 5355 "# of RSC fragments dropped"); 5356 SYSCTL_ADD_INT(ctx, 5357 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5358 OID_AUTO, "pktbuf_len", CTLFLAG_RD, 5359 &rxr->hn_pktbuf_len, 0, 5360 "Temporary channel packet buffer length"); 5361 } 5362 } 5363 } 5364 5365 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_queued", 5366 CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5367 __offsetof(struct hn_rx_ring, hn_lro.lro_queued), 5368 hn_rx_stat_u64_sysctl, 5369 "LU", "LRO queued"); 5370 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_flushed", 5371 CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5372 __offsetof(struct hn_rx_ring, hn_lro.lro_flushed), 5373 hn_rx_stat_u64_sysctl, 5374 "LU", "LRO flushed"); 5375 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_tried", 5376 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5377 __offsetof(struct hn_rx_ring, hn_lro_tried), 5378 hn_rx_stat_ulong_sysctl, "LU", "# of LRO tries"); 5379 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_length_lim", 5380 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5381 hn_lro_lenlim_sysctl, "IU", 5382 "Max # of data bytes to be aggregated by LRO"); 5383 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_ackcnt_lim", 5384 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5385 hn_lro_ackcnt_sysctl, "I", 5386 "Max # of ACKs to be aggregated by LRO"); 5387 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hosttcp", 5388 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_TCP, 5389 hn_trust_hcsum_sysctl, "I", 5390 "Trust tcp segment verification on host side, " 5391 "when csum info is missing"); 5392 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostudp", 5393 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_UDP, 5394 hn_trust_hcsum_sysctl, "I", 5395 "Trust udp datagram verification on host side, " 5396 "when csum info is missing"); 5397 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostip", 5398 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_IP, 5399 hn_trust_hcsum_sysctl, "I", 5400 "Trust ip packet verification on host side, " 5401 "when csum info is missing"); 5402 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_ip", 5403 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5404 __offsetof(struct hn_rx_ring, hn_csum_ip), 5405 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM IP"); 5406 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_tcp", 5407 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5408 __offsetof(struct hn_rx_ring, hn_csum_tcp), 5409 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM TCP"); 5410 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_udp", 5411 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5412 __offsetof(struct hn_rx_ring, hn_csum_udp), 5413 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM UDP"); 5414 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_trusted", 5415 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5416 __offsetof(struct hn_rx_ring, hn_csum_trusted), 5417 hn_rx_stat_ulong_sysctl, "LU", 5418 "# of packets that we trust host's csum verification"); 5419 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "small_pkts", 5420 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5421 __offsetof(struct hn_rx_ring, hn_small_pkts), 5422 hn_rx_stat_ulong_sysctl, "LU", "# of small packets received"); 5423 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_ack_failed", 5424 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5425 __offsetof(struct hn_rx_ring, hn_ack_failed), 5426 hn_rx_stat_ulong_sysctl, "LU", "# of RXBUF ack failures"); 5427 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_cnt", 5428 CTLFLAG_RD, &sc->hn_rx_ring_cnt, 0, "# created RX rings"); 5429 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_inuse", 5430 CTLFLAG_RD, &sc->hn_rx_ring_inuse, 0, "# used RX rings"); 5431 5432 return (0); 5433 } 5434 5435 static void 5436 hn_destroy_rx_data(struct hn_softc *sc) 5437 { 5438 int i; 5439 5440 if (sc->hn_rxbuf != NULL) { 5441 if ((sc->hn_flags & HN_FLAG_RXBUF_REF) == 0) 5442 free(sc->hn_rxbuf, M_DEVBUF); 5443 else 5444 device_printf(sc->hn_dev, "RXBUF is referenced\n"); 5445 sc->hn_rxbuf = NULL; 5446 } 5447 5448 if (sc->hn_rx_ring_cnt == 0) 5449 return; 5450 5451 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 5452 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 5453 5454 if (rxr->hn_br == NULL) 5455 continue; 5456 if ((rxr->hn_rx_flags & HN_RX_FLAG_BR_REF) == 0) { 5457 free(rxr->hn_br, M_DEVBUF); 5458 } else { 5459 device_printf(sc->hn_dev, 5460 "%dth channel bufring is referenced", i); 5461 } 5462 rxr->hn_br = NULL; 5463 5464 #if defined(INET) || defined(INET6) 5465 tcp_lro_free(&rxr->hn_lro); 5466 #endif 5467 free(rxr->hn_pktbuf, M_DEVBUF); 5468 } 5469 free(sc->hn_rx_ring, M_DEVBUF); 5470 sc->hn_rx_ring = NULL; 5471 5472 sc->hn_rx_ring_cnt = 0; 5473 sc->hn_rx_ring_inuse = 0; 5474 } 5475 5476 static int 5477 hn_tx_ring_create(struct hn_softc *sc, int id) 5478 { 5479 struct hn_tx_ring *txr = &sc->hn_tx_ring[id]; 5480 device_t dev = sc->hn_dev; 5481 bus_dma_tag_t parent_dtag; 5482 int error, i; 5483 5484 txr->hn_sc = sc; 5485 txr->hn_tx_idx = id; 5486 5487 #ifndef HN_USE_TXDESC_BUFRING 5488 mtx_init(&txr->hn_txlist_spin, "hn txlist", NULL, MTX_SPIN); 5489 #endif 5490 mtx_init(&txr->hn_tx_lock, "hn tx", NULL, MTX_DEF); 5491 5492 txr->hn_txdesc_cnt = HN_TX_DESC_CNT; 5493 txr->hn_txdesc = malloc(sizeof(struct hn_txdesc) * txr->hn_txdesc_cnt, 5494 M_DEVBUF, M_WAITOK | M_ZERO); 5495 #ifndef HN_USE_TXDESC_BUFRING 5496 SLIST_INIT(&txr->hn_txlist); 5497 #else 5498 txr->hn_txdesc_br = buf_ring_alloc(txr->hn_txdesc_cnt, M_DEVBUF, 5499 M_WAITOK, &txr->hn_tx_lock); 5500 #endif 5501 5502 if (hn_tx_taskq_mode == HN_TX_TASKQ_M_EVTTQ) { 5503 txr->hn_tx_taskq = VMBUS_GET_EVENT_TASKQ( 5504 device_get_parent(dev), dev, HN_RING_IDX2CPU(sc, id)); 5505 } else { 5506 txr->hn_tx_taskq = sc->hn_tx_taskqs[id % hn_tx_taskq_cnt]; 5507 } 5508 5509 #ifdef HN_IFSTART_SUPPORT 5510 if (hn_use_if_start) { 5511 txr->hn_txeof = hn_start_txeof; 5512 TASK_INIT(&txr->hn_tx_task, 0, hn_start_taskfunc, txr); 5513 TASK_INIT(&txr->hn_txeof_task, 0, hn_start_txeof_taskfunc, txr); 5514 } else 5515 #endif 5516 { 5517 int br_depth; 5518 5519 txr->hn_txeof = hn_xmit_txeof; 5520 TASK_INIT(&txr->hn_tx_task, 0, hn_xmit_taskfunc, txr); 5521 TASK_INIT(&txr->hn_txeof_task, 0, hn_xmit_txeof_taskfunc, txr); 5522 5523 br_depth = hn_get_txswq_depth(txr); 5524 txr->hn_mbuf_br = buf_ring_alloc(br_depth, M_DEVBUF, 5525 M_WAITOK, &txr->hn_tx_lock); 5526 } 5527 5528 txr->hn_direct_tx_size = hn_direct_tx_size; 5529 5530 /* 5531 * Always schedule transmission instead of trying to do direct 5532 * transmission. This one gives the best performance so far. 5533 */ 5534 txr->hn_sched_tx = 1; 5535 5536 parent_dtag = bus_get_dma_tag(dev); 5537 5538 /* DMA tag for RNDIS packet messages. */ 5539 error = bus_dma_tag_create(parent_dtag, /* parent */ 5540 HN_RNDIS_PKT_ALIGN, /* alignment */ 5541 HN_RNDIS_PKT_BOUNDARY, /* boundary */ 5542 BUS_SPACE_MAXADDR, /* lowaddr */ 5543 BUS_SPACE_MAXADDR, /* highaddr */ 5544 NULL, NULL, /* filter, filterarg */ 5545 HN_RNDIS_PKT_LEN, /* maxsize */ 5546 1, /* nsegments */ 5547 HN_RNDIS_PKT_LEN, /* maxsegsize */ 5548 0, /* flags */ 5549 NULL, /* lockfunc */ 5550 NULL, /* lockfuncarg */ 5551 &txr->hn_tx_rndis_dtag); 5552 if (error) { 5553 device_printf(dev, "failed to create rndis dmatag\n"); 5554 return error; 5555 } 5556 5557 /* DMA tag for data. */ 5558 error = bus_dma_tag_create(parent_dtag, /* parent */ 5559 1, /* alignment */ 5560 HN_TX_DATA_BOUNDARY, /* boundary */ 5561 BUS_SPACE_MAXADDR, /* lowaddr */ 5562 BUS_SPACE_MAXADDR, /* highaddr */ 5563 NULL, NULL, /* filter, filterarg */ 5564 HN_TX_DATA_MAXSIZE, /* maxsize */ 5565 HN_TX_DATA_SEGCNT_MAX, /* nsegments */ 5566 HN_TX_DATA_SEGSIZE, /* maxsegsize */ 5567 0, /* flags */ 5568 NULL, /* lockfunc */ 5569 NULL, /* lockfuncarg */ 5570 &txr->hn_tx_data_dtag); 5571 if (error) { 5572 device_printf(dev, "failed to create data dmatag\n"); 5573 return error; 5574 } 5575 5576 for (i = 0; i < txr->hn_txdesc_cnt; ++i) { 5577 struct hn_txdesc *txd = &txr->hn_txdesc[i]; 5578 5579 txd->txr = txr; 5580 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 5581 STAILQ_INIT(&txd->agg_list); 5582 5583 /* 5584 * Allocate and load RNDIS packet message. 5585 */ 5586 error = bus_dmamem_alloc(txr->hn_tx_rndis_dtag, 5587 (void **)&txd->rndis_pkt, 5588 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, 5589 &txd->rndis_pkt_dmap); 5590 if (error) { 5591 device_printf(dev, 5592 "failed to allocate rndis_packet_msg, %d\n", i); 5593 return error; 5594 } 5595 5596 error = bus_dmamap_load(txr->hn_tx_rndis_dtag, 5597 txd->rndis_pkt_dmap, 5598 txd->rndis_pkt, HN_RNDIS_PKT_LEN, 5599 hyperv_dma_map_paddr, &txd->rndis_pkt_paddr, 5600 BUS_DMA_NOWAIT); 5601 if (error) { 5602 device_printf(dev, 5603 "failed to load rndis_packet_msg, %d\n", i); 5604 bus_dmamem_free(txr->hn_tx_rndis_dtag, 5605 txd->rndis_pkt, txd->rndis_pkt_dmap); 5606 return error; 5607 } 5608 5609 /* DMA map for TX data. */ 5610 error = bus_dmamap_create(txr->hn_tx_data_dtag, 0, 5611 &txd->data_dmap); 5612 if (error) { 5613 device_printf(dev, 5614 "failed to allocate tx data dmamap\n"); 5615 bus_dmamap_unload(txr->hn_tx_rndis_dtag, 5616 txd->rndis_pkt_dmap); 5617 bus_dmamem_free(txr->hn_tx_rndis_dtag, 5618 txd->rndis_pkt, txd->rndis_pkt_dmap); 5619 return error; 5620 } 5621 5622 /* All set, put it to list */ 5623 txd->flags |= HN_TXD_FLAG_ONLIST; 5624 #ifndef HN_USE_TXDESC_BUFRING 5625 SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); 5626 #else 5627 buf_ring_enqueue(txr->hn_txdesc_br, txd); 5628 #endif 5629 } 5630 txr->hn_txdesc_avail = txr->hn_txdesc_cnt; 5631 5632 if (sc->hn_tx_sysctl_tree != NULL) { 5633 struct sysctl_oid_list *child; 5634 struct sysctl_ctx_list *ctx; 5635 char name[16]; 5636 5637 /* 5638 * Create per TX ring sysctl tree: 5639 * dev.hn.UNIT.tx.RINGID 5640 */ 5641 ctx = device_get_sysctl_ctx(dev); 5642 child = SYSCTL_CHILDREN(sc->hn_tx_sysctl_tree); 5643 5644 snprintf(name, sizeof(name), "%d", id); 5645 txr->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, 5646 name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5647 5648 if (txr->hn_tx_sysctl_tree != NULL) { 5649 child = SYSCTL_CHILDREN(txr->hn_tx_sysctl_tree); 5650 5651 #ifdef HN_DEBUG 5652 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_avail", 5653 CTLFLAG_RD, &txr->hn_txdesc_avail, 0, 5654 "# of available TX descs"); 5655 #endif 5656 #ifdef HN_IFSTART_SUPPORT 5657 if (!hn_use_if_start) 5658 #endif 5659 { 5660 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "oactive", 5661 CTLFLAG_RD, &txr->hn_oactive, 0, 5662 "over active"); 5663 } 5664 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "packets", 5665 CTLFLAG_RW | CTLFLAG_STATS, &txr->hn_pkts, 5666 "# of packets transmitted"); 5667 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "sends", 5668 CTLFLAG_RW | CTLFLAG_STATS, &txr->hn_sends, 5669 "# of sends"); 5670 } 5671 } 5672 5673 return 0; 5674 } 5675 5676 static void 5677 hn_txdesc_dmamap_destroy(struct hn_txdesc *txd) 5678 { 5679 struct hn_tx_ring *txr = txd->txr; 5680 5681 KASSERT(txd->m == NULL, ("still has mbuf installed")); 5682 KASSERT((txd->flags & HN_TXD_FLAG_DMAMAP) == 0, ("still dma mapped")); 5683 5684 bus_dmamap_unload(txr->hn_tx_rndis_dtag, txd->rndis_pkt_dmap); 5685 bus_dmamem_free(txr->hn_tx_rndis_dtag, txd->rndis_pkt, 5686 txd->rndis_pkt_dmap); 5687 bus_dmamap_destroy(txr->hn_tx_data_dtag, txd->data_dmap); 5688 } 5689 5690 static void 5691 hn_txdesc_gc(struct hn_tx_ring *txr, struct hn_txdesc *txd) 5692 { 5693 5694 KASSERT(txd->refs == 0 || txd->refs == 1, 5695 ("invalid txd refs %d", txd->refs)); 5696 5697 /* Aggregated txds will be freed by their aggregating txd. */ 5698 if (txd->refs > 0 && (txd->flags & HN_TXD_FLAG_ONAGG) == 0) { 5699 int freed __diagused; 5700 5701 freed = hn_txdesc_put(txr, txd); 5702 KASSERT(freed, ("can't free txdesc")); 5703 } 5704 } 5705 5706 static void 5707 hn_tx_ring_destroy(struct hn_tx_ring *txr) 5708 { 5709 int i; 5710 5711 if (txr->hn_txdesc == NULL) 5712 return; 5713 5714 /* 5715 * NOTE: 5716 * Because the freeing of aggregated txds will be deferred 5717 * to the aggregating txd, two passes are used here: 5718 * - The first pass GCes any pending txds. This GC is necessary, 5719 * since if the channels are revoked, hypervisor will not 5720 * deliver send-done for all pending txds. 5721 * - The second pass frees the busdma stuffs, i.e. after all txds 5722 * were freed. 5723 */ 5724 for (i = 0; i < txr->hn_txdesc_cnt; ++i) 5725 hn_txdesc_gc(txr, &txr->hn_txdesc[i]); 5726 for (i = 0; i < txr->hn_txdesc_cnt; ++i) 5727 hn_txdesc_dmamap_destroy(&txr->hn_txdesc[i]); 5728 5729 if (txr->hn_tx_data_dtag != NULL) 5730 bus_dma_tag_destroy(txr->hn_tx_data_dtag); 5731 if (txr->hn_tx_rndis_dtag != NULL) 5732 bus_dma_tag_destroy(txr->hn_tx_rndis_dtag); 5733 5734 #ifdef HN_USE_TXDESC_BUFRING 5735 buf_ring_free(txr->hn_txdesc_br, M_DEVBUF); 5736 #endif 5737 5738 free(txr->hn_txdesc, M_DEVBUF); 5739 txr->hn_txdesc = NULL; 5740 5741 if (txr->hn_mbuf_br != NULL) 5742 buf_ring_free(txr->hn_mbuf_br, M_DEVBUF); 5743 5744 #ifndef HN_USE_TXDESC_BUFRING 5745 mtx_destroy(&txr->hn_txlist_spin); 5746 #endif 5747 mtx_destroy(&txr->hn_tx_lock); 5748 } 5749 5750 static int 5751 hn_create_tx_data(struct hn_softc *sc, int ring_cnt) 5752 { 5753 struct sysctl_oid_list *child; 5754 struct sysctl_ctx_list *ctx; 5755 int i; 5756 5757 /* 5758 * Create TXBUF for chimney sending. 5759 * 5760 * NOTE: It is shared by all channels. 5761 */ 5762 sc->hn_chim = contigmalloc(HN_CHIM_SIZE, M_DEVBUF, M_WAITOK | M_ZERO, 5763 0ul, ~0ul, PAGE_SIZE, 0); 5764 if (sc->hn_chim == NULL) { 5765 device_printf(sc->hn_dev, "allocate txbuf failed\n"); 5766 return (ENOMEM); 5767 } 5768 5769 sc->hn_tx_ring_cnt = ring_cnt; 5770 sc->hn_tx_ring_inuse = sc->hn_tx_ring_cnt; 5771 5772 sc->hn_tx_ring = malloc(sizeof(struct hn_tx_ring) * sc->hn_tx_ring_cnt, 5773 M_DEVBUF, M_WAITOK | M_ZERO); 5774 5775 ctx = device_get_sysctl_ctx(sc->hn_dev); 5776 child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->hn_dev)); 5777 5778 /* Create dev.hn.UNIT.tx sysctl tree */ 5779 sc->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "tx", 5780 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5781 5782 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 5783 int error; 5784 5785 error = hn_tx_ring_create(sc, i); 5786 if (error) 5787 return error; 5788 } 5789 5790 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "no_txdescs", 5791 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5792 __offsetof(struct hn_tx_ring, hn_no_txdescs), 5793 hn_tx_stat_ulong_sysctl, "LU", "# of times short of TX descs"); 5794 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "send_failed", 5795 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5796 __offsetof(struct hn_tx_ring, hn_send_failed), 5797 hn_tx_stat_ulong_sysctl, "LU", "# of hyper-v sending failure"); 5798 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txdma_failed", 5799 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5800 __offsetof(struct hn_tx_ring, hn_txdma_failed), 5801 hn_tx_stat_ulong_sysctl, "LU", "# of TX DMA failure"); 5802 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_flush_failed", 5803 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5804 __offsetof(struct hn_tx_ring, hn_flush_failed), 5805 hn_tx_stat_ulong_sysctl, "LU", 5806 "# of packet transmission aggregation flush failure"); 5807 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_collapsed", 5808 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5809 __offsetof(struct hn_tx_ring, hn_tx_collapsed), 5810 hn_tx_stat_ulong_sysctl, "LU", "# of TX mbuf collapsed"); 5811 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney", 5812 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5813 __offsetof(struct hn_tx_ring, hn_tx_chimney), 5814 hn_tx_stat_ulong_sysctl, "LU", "# of chimney send"); 5815 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_tried", 5816 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5817 __offsetof(struct hn_tx_ring, hn_tx_chimney_tried), 5818 hn_tx_stat_ulong_sysctl, "LU", "# of chimney send tries"); 5819 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_cnt", 5820 CTLFLAG_RD, &sc->hn_tx_ring[0].hn_txdesc_cnt, 0, 5821 "# of total TX descs"); 5822 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_chimney_max", 5823 CTLFLAG_RD, &sc->hn_chim_szmax, 0, 5824 "Chimney send packet size upper boundary"); 5825 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_size", 5826 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5827 hn_chim_size_sysctl, "I", "Chimney send packet size limit"); 5828 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "direct_tx_size", 5829 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5830 __offsetof(struct hn_tx_ring, hn_direct_tx_size), 5831 hn_tx_conf_int_sysctl, "I", 5832 "Size of the packet for direct transmission"); 5833 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "sched_tx", 5834 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5835 __offsetof(struct hn_tx_ring, hn_sched_tx), 5836 hn_tx_conf_int_sysctl, "I", 5837 "Always schedule transmission " 5838 "instead of doing direct transmission"); 5839 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_cnt", 5840 CTLFLAG_RD, &sc->hn_tx_ring_cnt, 0, "# created TX rings"); 5841 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_inuse", 5842 CTLFLAG_RD, &sc->hn_tx_ring_inuse, 0, "# used TX rings"); 5843 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "agg_szmax", 5844 CTLFLAG_RD, &sc->hn_tx_ring[0].hn_agg_szmax, 0, 5845 "Applied packet transmission aggregation size"); 5846 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_pktmax", 5847 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 5848 hn_txagg_pktmax_sysctl, "I", 5849 "Applied packet transmission aggregation packets"); 5850 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_align", 5851 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 5852 hn_txagg_align_sysctl, "I", 5853 "Applied packet transmission aggregation alignment"); 5854 5855 return 0; 5856 } 5857 5858 static void 5859 hn_set_chim_size(struct hn_softc *sc, int chim_size) 5860 { 5861 int i; 5862 5863 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5864 sc->hn_tx_ring[i].hn_chim_size = chim_size; 5865 } 5866 5867 static void 5868 hn_set_tso_maxsize(struct hn_softc *sc, int tso_maxlen, int mtu) 5869 { 5870 if_t ifp = sc->hn_ifp; 5871 u_int hw_tsomax; 5872 int tso_minlen; 5873 5874 HN_LOCK_ASSERT(sc); 5875 5876 if ((if_getcapabilities(ifp) & (IFCAP_TSO4 | IFCAP_TSO6)) == 0) 5877 return; 5878 5879 KASSERT(sc->hn_ndis_tso_sgmin >= 2, 5880 ("invalid NDIS tso sgmin %d", sc->hn_ndis_tso_sgmin)); 5881 tso_minlen = sc->hn_ndis_tso_sgmin * mtu; 5882 5883 KASSERT(sc->hn_ndis_tso_szmax >= tso_minlen && 5884 sc->hn_ndis_tso_szmax <= IP_MAXPACKET, 5885 ("invalid NDIS tso szmax %d", sc->hn_ndis_tso_szmax)); 5886 5887 if (tso_maxlen < tso_minlen) 5888 tso_maxlen = tso_minlen; 5889 else if (tso_maxlen > IP_MAXPACKET) 5890 tso_maxlen = IP_MAXPACKET; 5891 if (tso_maxlen > sc->hn_ndis_tso_szmax) 5892 tso_maxlen = sc->hn_ndis_tso_szmax; 5893 hw_tsomax = tso_maxlen - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 5894 5895 if (hn_xpnt_vf_isready(sc)) { 5896 if (hw_tsomax > if_gethwtsomax(sc->hn_vf_ifp)) 5897 hw_tsomax = if_gethwtsomax(sc->hn_vf_ifp); 5898 } 5899 if_sethwtsomax(ifp, hw_tsomax); 5900 if (bootverbose) 5901 if_printf(ifp, "TSO size max %u\n", if_gethwtsomax(ifp)); 5902 } 5903 5904 static void 5905 hn_fixup_tx_data(struct hn_softc *sc) 5906 { 5907 uint64_t csum_assist; 5908 int i; 5909 5910 hn_set_chim_size(sc, sc->hn_chim_szmax); 5911 if (hn_tx_chimney_size > 0 && 5912 hn_tx_chimney_size < sc->hn_chim_szmax) 5913 hn_set_chim_size(sc, hn_tx_chimney_size); 5914 5915 csum_assist = 0; 5916 if (sc->hn_caps & HN_CAP_IPCS) 5917 csum_assist |= CSUM_IP; 5918 if (sc->hn_caps & HN_CAP_TCP4CS) 5919 csum_assist |= CSUM_IP_TCP; 5920 if ((sc->hn_caps & HN_CAP_UDP4CS) && hn_enable_udp4cs) 5921 csum_assist |= CSUM_IP_UDP; 5922 if (sc->hn_caps & HN_CAP_TCP6CS) 5923 csum_assist |= CSUM_IP6_TCP; 5924 if ((sc->hn_caps & HN_CAP_UDP6CS) && hn_enable_udp6cs) 5925 csum_assist |= CSUM_IP6_UDP; 5926 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5927 sc->hn_tx_ring[i].hn_csum_assist = csum_assist; 5928 5929 if (sc->hn_caps & HN_CAP_HASHVAL) { 5930 /* 5931 * Support HASHVAL pktinfo on TX path. 5932 */ 5933 if (bootverbose) 5934 if_printf(sc->hn_ifp, "support HASHVAL pktinfo\n"); 5935 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5936 sc->hn_tx_ring[i].hn_tx_flags |= HN_TX_FLAG_HASHVAL; 5937 } 5938 } 5939 5940 static void 5941 hn_fixup_rx_data(struct hn_softc *sc) 5942 { 5943 5944 if (sc->hn_caps & HN_CAP_UDPHASH) { 5945 int i; 5946 5947 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 5948 sc->hn_rx_ring[i].hn_rx_flags |= HN_RX_FLAG_UDP_HASH; 5949 } 5950 } 5951 5952 static void 5953 hn_destroy_tx_data(struct hn_softc *sc) 5954 { 5955 int i; 5956 5957 if (sc->hn_chim != NULL) { 5958 if ((sc->hn_flags & HN_FLAG_CHIM_REF) == 0) { 5959 free(sc->hn_chim, M_DEVBUF); 5960 } else { 5961 device_printf(sc->hn_dev, 5962 "chimney sending buffer is referenced"); 5963 } 5964 sc->hn_chim = NULL; 5965 } 5966 5967 if (sc->hn_tx_ring_cnt == 0) 5968 return; 5969 5970 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5971 hn_tx_ring_destroy(&sc->hn_tx_ring[i]); 5972 5973 free(sc->hn_tx_ring, M_DEVBUF); 5974 sc->hn_tx_ring = NULL; 5975 5976 sc->hn_tx_ring_cnt = 0; 5977 sc->hn_tx_ring_inuse = 0; 5978 } 5979 5980 #ifdef HN_IFSTART_SUPPORT 5981 5982 static void 5983 hn_start_taskfunc(void *xtxr, int pending __unused) 5984 { 5985 struct hn_tx_ring *txr = xtxr; 5986 5987 mtx_lock(&txr->hn_tx_lock); 5988 hn_start_locked(txr, 0); 5989 mtx_unlock(&txr->hn_tx_lock); 5990 } 5991 5992 static int 5993 hn_start_locked(struct hn_tx_ring *txr, int len) 5994 { 5995 struct hn_softc *sc = txr->hn_sc; 5996 if_t ifp = sc->hn_ifp; 5997 int sched = 0; 5998 5999 KASSERT(hn_use_if_start, 6000 ("hn_start_locked is called, when if_start is disabled")); 6001 KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); 6002 mtx_assert(&txr->hn_tx_lock, MA_OWNED); 6003 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 6004 6005 if (__predict_false(txr->hn_suspended)) 6006 return (0); 6007 6008 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 6009 IFF_DRV_RUNNING) 6010 return (0); 6011 6012 while (!if_sendq_empty(ifp)) { 6013 struct hn_txdesc *txd; 6014 struct mbuf *m_head; 6015 int error; 6016 6017 m_head = if_dequeue(ifp); 6018 if (m_head == NULL) 6019 break; 6020 6021 if (len > 0 && m_head->m_pkthdr.len > len) { 6022 /* 6023 * This sending could be time consuming; let callers 6024 * dispatch this packet sending (and sending of any 6025 * following up packets) to tx taskqueue. 6026 */ 6027 if_sendq_prepend(ifp, m_head); 6028 sched = 1; 6029 break; 6030 } 6031 6032 #if defined(INET6) || defined(INET) 6033 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 6034 m_head = hn_tso_fixup(m_head); 6035 if (__predict_false(m_head == NULL)) { 6036 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6037 continue; 6038 } 6039 } else if (m_head->m_pkthdr.csum_flags & 6040 (CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP6_UDP | CSUM_IP6_TCP)) { 6041 m_head = hn_set_hlen(m_head); 6042 if (__predict_false(m_head == NULL)) { 6043 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6044 continue; 6045 } 6046 } 6047 #endif 6048 6049 txd = hn_txdesc_get(txr); 6050 if (txd == NULL) { 6051 txr->hn_no_txdescs++; 6052 if_sendq_prepend(ifp, m_head); 6053 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 6054 break; 6055 } 6056 6057 error = hn_encap(ifp, txr, txd, &m_head); 6058 if (error) { 6059 /* Both txd and m_head are freed */ 6060 KASSERT(txr->hn_agg_txd == NULL, 6061 ("encap failed w/ pending aggregating txdesc")); 6062 continue; 6063 } 6064 6065 if (txr->hn_agg_pktleft == 0) { 6066 if (txr->hn_agg_txd != NULL) { 6067 KASSERT(m_head == NULL, 6068 ("pending mbuf for aggregating txdesc")); 6069 error = hn_flush_txagg(ifp, txr); 6070 if (__predict_false(error)) { 6071 if_setdrvflagbits(ifp, 6072 IFF_DRV_OACTIVE, 0); 6073 break; 6074 } 6075 } else { 6076 KASSERT(m_head != NULL, ("mbuf was freed")); 6077 error = hn_txpkt(ifp, txr, txd); 6078 if (__predict_false(error)) { 6079 /* txd is freed, but m_head is not */ 6080 if_sendq_prepend(ifp, m_head); 6081 if_setdrvflagbits(ifp, 6082 IFF_DRV_OACTIVE, 0); 6083 break; 6084 } 6085 } 6086 } 6087 #ifdef INVARIANTS 6088 else { 6089 KASSERT(txr->hn_agg_txd != NULL, 6090 ("no aggregating txdesc")); 6091 KASSERT(m_head == NULL, 6092 ("pending mbuf for aggregating txdesc")); 6093 } 6094 #endif 6095 } 6096 6097 /* Flush pending aggerated transmission. */ 6098 if (txr->hn_agg_txd != NULL) 6099 hn_flush_txagg(ifp, txr); 6100 return (sched); 6101 } 6102 6103 static void 6104 hn_start(if_t ifp) 6105 { 6106 struct hn_softc *sc = if_getsoftc(ifp); 6107 struct hn_tx_ring *txr = &sc->hn_tx_ring[0]; 6108 6109 if (txr->hn_sched_tx) 6110 goto do_sched; 6111 6112 if (mtx_trylock(&txr->hn_tx_lock)) { 6113 int sched; 6114 6115 sched = hn_start_locked(txr, txr->hn_direct_tx_size); 6116 mtx_unlock(&txr->hn_tx_lock); 6117 if (!sched) 6118 return; 6119 } 6120 do_sched: 6121 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); 6122 } 6123 6124 static void 6125 hn_start_txeof_taskfunc(void *xtxr, int pending __unused) 6126 { 6127 struct hn_tx_ring *txr = xtxr; 6128 6129 mtx_lock(&txr->hn_tx_lock); 6130 if_setdrvflagbits(txr->hn_sc->hn_ifp, 0, IFF_DRV_OACTIVE); 6131 hn_start_locked(txr, 0); 6132 mtx_unlock(&txr->hn_tx_lock); 6133 } 6134 6135 static void 6136 hn_start_txeof(struct hn_tx_ring *txr) 6137 { 6138 struct hn_softc *sc = txr->hn_sc; 6139 if_t ifp = sc->hn_ifp; 6140 6141 KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); 6142 6143 if (txr->hn_sched_tx) 6144 goto do_sched; 6145 6146 if (mtx_trylock(&txr->hn_tx_lock)) { 6147 int sched; 6148 6149 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 6150 sched = hn_start_locked(txr, txr->hn_direct_tx_size); 6151 mtx_unlock(&txr->hn_tx_lock); 6152 if (sched) { 6153 taskqueue_enqueue(txr->hn_tx_taskq, 6154 &txr->hn_tx_task); 6155 } 6156 } else { 6157 do_sched: 6158 /* 6159 * Release the OACTIVE earlier, with the hope, that 6160 * others could catch up. The task will clear the 6161 * flag again with the hn_tx_lock to avoid possible 6162 * races. 6163 */ 6164 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 6165 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 6166 } 6167 } 6168 6169 #endif /* HN_IFSTART_SUPPORT */ 6170 6171 static int 6172 hn_xmit(struct hn_tx_ring *txr, int len) 6173 { 6174 struct hn_softc *sc = txr->hn_sc; 6175 if_t ifp = sc->hn_ifp; 6176 struct mbuf *m_head; 6177 int sched = 0; 6178 6179 mtx_assert(&txr->hn_tx_lock, MA_OWNED); 6180 #ifdef HN_IFSTART_SUPPORT 6181 KASSERT(hn_use_if_start == 0, 6182 ("hn_xmit is called, when if_start is enabled")); 6183 #endif 6184 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 6185 6186 if (__predict_false(txr->hn_suspended)) 6187 return (0); 6188 6189 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || txr->hn_oactive) 6190 return (0); 6191 6192 while ((m_head = drbr_peek(ifp, txr->hn_mbuf_br)) != NULL) { 6193 struct hn_txdesc *txd; 6194 int error; 6195 6196 if (len > 0 && m_head->m_pkthdr.len > len) { 6197 /* 6198 * This sending could be time consuming; let callers 6199 * dispatch this packet sending (and sending of any 6200 * following up packets) to tx taskqueue. 6201 */ 6202 drbr_putback(ifp, txr->hn_mbuf_br, m_head); 6203 sched = 1; 6204 break; 6205 } 6206 6207 txd = hn_txdesc_get(txr); 6208 if (txd == NULL) { 6209 txr->hn_no_txdescs++; 6210 drbr_putback(ifp, txr->hn_mbuf_br, m_head); 6211 txr->hn_oactive = 1; 6212 break; 6213 } 6214 6215 error = hn_encap(ifp, txr, txd, &m_head); 6216 if (error) { 6217 /* Both txd and m_head are freed; discard */ 6218 KASSERT(txr->hn_agg_txd == NULL, 6219 ("encap failed w/ pending aggregating txdesc")); 6220 drbr_advance(ifp, txr->hn_mbuf_br); 6221 continue; 6222 } 6223 6224 if (txr->hn_agg_pktleft == 0) { 6225 if (txr->hn_agg_txd != NULL) { 6226 KASSERT(m_head == NULL, 6227 ("pending mbuf for aggregating txdesc")); 6228 error = hn_flush_txagg(ifp, txr); 6229 if (__predict_false(error)) { 6230 txr->hn_oactive = 1; 6231 break; 6232 } 6233 } else { 6234 KASSERT(m_head != NULL, ("mbuf was freed")); 6235 error = hn_txpkt(ifp, txr, txd); 6236 if (__predict_false(error)) { 6237 /* txd is freed, but m_head is not */ 6238 drbr_putback(ifp, txr->hn_mbuf_br, 6239 m_head); 6240 txr->hn_oactive = 1; 6241 break; 6242 } 6243 } 6244 } 6245 #ifdef INVARIANTS 6246 else { 6247 KASSERT(txr->hn_agg_txd != NULL, 6248 ("no aggregating txdesc")); 6249 KASSERT(m_head == NULL, 6250 ("pending mbuf for aggregating txdesc")); 6251 } 6252 #endif 6253 6254 /* Sent */ 6255 drbr_advance(ifp, txr->hn_mbuf_br); 6256 } 6257 6258 /* Flush pending aggerated transmission. */ 6259 if (txr->hn_agg_txd != NULL) 6260 hn_flush_txagg(ifp, txr); 6261 return (sched); 6262 } 6263 6264 static int 6265 hn_transmit(if_t ifp, struct mbuf *m) 6266 { 6267 struct hn_softc *sc = if_getsoftc(ifp); 6268 struct hn_tx_ring *txr; 6269 int error, idx = 0; 6270 6271 if (sc->hn_xvf_flags & (HN_XVFFLAG_ENABLED | HN_XVFFLAG_SWITCHING)) { 6272 struct rm_priotracker pt; 6273 6274 rm_rlock(&sc->hn_vf_lock, &pt); 6275 if ((sc->hn_xvf_flags & HN_XVFFLAG_SWITCHING) || 6276 ((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) && 6277 sc->hn_vf_active_assoc != atomic_load_acq_int(&sc->hn_vf_assoc))) { 6278 rm_runlock(&sc->hn_vf_lock, &pt); 6279 m_freem(m); 6280 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6281 return (ENETDOWN); 6282 } 6283 if (__predict_true(sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 6284 struct mbuf *m_bpf = NULL; 6285 int obytes, omcast; 6286 6287 obytes = m->m_pkthdr.len; 6288 omcast = (m->m_flags & M_MCAST) != 0; 6289 6290 if (sc->hn_xvf_flags & HN_XVFFLAG_ACCBPF) { 6291 if (bpf_peers_present_if(ifp)) { 6292 m_bpf = m_copypacket(m, M_NOWAIT); 6293 if (m_bpf == NULL) { 6294 /* 6295 * Failed to grab a shallow 6296 * copy; tap now. 6297 */ 6298 ETHER_BPF_MTAP(ifp, m); 6299 } 6300 } 6301 } else { 6302 ETHER_BPF_MTAP(ifp, m); 6303 } 6304 6305 error = if_transmit(sc->hn_vf_ifp, m); 6306 rm_runlock(&sc->hn_vf_lock, &pt); 6307 6308 if (m_bpf != NULL) { 6309 if (!error) 6310 ETHER_BPF_MTAP(ifp, m_bpf); 6311 m_freem(m_bpf); 6312 } 6313 6314 if (error == ENOBUFS) { 6315 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6316 } else if (error) { 6317 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6318 } else { 6319 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 6320 if_inc_counter(ifp, IFCOUNTER_OBYTES, obytes); 6321 if (omcast) { 6322 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 6323 omcast); 6324 } 6325 } 6326 return (error); 6327 } 6328 rm_runlock(&sc->hn_vf_lock, &pt); 6329 } 6330 6331 #if defined(INET6) || defined(INET) 6332 /* 6333 * Perform TSO packet header fixup or get l2/l3 header length now, 6334 * since packet headers should be cache-hot. 6335 */ 6336 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 6337 m = hn_tso_fixup(m); 6338 if (__predict_false(m == NULL)) { 6339 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6340 return EIO; 6341 } 6342 } else if (m->m_pkthdr.csum_flags & 6343 (CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP6_UDP | CSUM_IP6_TCP)) { 6344 m = hn_set_hlen(m); 6345 if (__predict_false(m == NULL)) { 6346 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6347 return EIO; 6348 } 6349 } 6350 #endif 6351 6352 /* 6353 * Select the TX ring based on flowid 6354 */ 6355 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 6356 #ifdef RSS 6357 uint32_t bid; 6358 6359 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 6360 &bid) == 0) 6361 idx = bid % sc->hn_tx_ring_inuse; 6362 else 6363 #endif 6364 { 6365 #if defined(INET6) || defined(INET) 6366 int tcpsyn = 0; 6367 6368 if (m->m_pkthdr.len < 128 && 6369 (m->m_pkthdr.csum_flags & 6370 (CSUM_IP_TCP | CSUM_IP6_TCP)) && 6371 (m->m_pkthdr.csum_flags & CSUM_TSO) == 0) { 6372 m = hn_check_tcpsyn(m, &tcpsyn); 6373 if (__predict_false(m == NULL)) { 6374 if_inc_counter(ifp, 6375 IFCOUNTER_OERRORS, 1); 6376 return (EIO); 6377 } 6378 } 6379 #else 6380 const int tcpsyn = 0; 6381 #endif 6382 if (tcpsyn) 6383 idx = 0; 6384 else 6385 idx = m->m_pkthdr.flowid % sc->hn_tx_ring_inuse; 6386 } 6387 } 6388 txr = &sc->hn_tx_ring[idx]; 6389 6390 error = drbr_enqueue(ifp, txr->hn_mbuf_br, m); 6391 if (error) { 6392 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6393 return error; 6394 } 6395 6396 if (txr->hn_oactive) 6397 return 0; 6398 6399 if (txr->hn_sched_tx) 6400 goto do_sched; 6401 6402 if (mtx_trylock(&txr->hn_tx_lock)) { 6403 int sched; 6404 6405 sched = hn_xmit(txr, txr->hn_direct_tx_size); 6406 mtx_unlock(&txr->hn_tx_lock); 6407 if (!sched) 6408 return 0; 6409 } 6410 do_sched: 6411 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); 6412 return 0; 6413 } 6414 6415 static void 6416 hn_tx_ring_qflush(struct hn_tx_ring *txr) 6417 { 6418 struct mbuf *m; 6419 6420 mtx_lock(&txr->hn_tx_lock); 6421 while ((m = buf_ring_dequeue_sc(txr->hn_mbuf_br)) != NULL) 6422 m_freem(m); 6423 mtx_unlock(&txr->hn_tx_lock); 6424 } 6425 6426 static void 6427 hn_xmit_qflush(if_t ifp) 6428 { 6429 struct hn_softc *sc = if_getsoftc(ifp); 6430 struct rm_priotracker pt; 6431 int i; 6432 6433 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 6434 hn_tx_ring_qflush(&sc->hn_tx_ring[i]); 6435 if_qflush(ifp); 6436 6437 rm_rlock(&sc->hn_vf_lock, &pt); 6438 /* Transparent mode owns the VF's queues even while switching paths. */ 6439 if (hn_xpnt_vf && sc->hn_vf_ifp != NULL) 6440 if_getqflushfn(sc->hn_vf_ifp)(sc->hn_vf_ifp); 6441 rm_runlock(&sc->hn_vf_lock, &pt); 6442 } 6443 6444 static void 6445 hn_xmit_txeof(struct hn_tx_ring *txr) 6446 { 6447 6448 if (txr->hn_sched_tx) 6449 goto do_sched; 6450 6451 if (mtx_trylock(&txr->hn_tx_lock)) { 6452 int sched; 6453 6454 txr->hn_oactive = 0; 6455 sched = hn_xmit(txr, txr->hn_direct_tx_size); 6456 mtx_unlock(&txr->hn_tx_lock); 6457 if (sched) { 6458 taskqueue_enqueue(txr->hn_tx_taskq, 6459 &txr->hn_tx_task); 6460 } 6461 } else { 6462 do_sched: 6463 /* 6464 * Release the oactive earlier, with the hope, that 6465 * others could catch up. The task will clear the 6466 * oactive again with the hn_tx_lock to avoid possible 6467 * races. 6468 */ 6469 txr->hn_oactive = 0; 6470 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 6471 } 6472 } 6473 6474 static void 6475 hn_xmit_taskfunc(void *xtxr, int pending __unused) 6476 { 6477 struct hn_tx_ring *txr = xtxr; 6478 6479 mtx_lock(&txr->hn_tx_lock); 6480 hn_xmit(txr, 0); 6481 mtx_unlock(&txr->hn_tx_lock); 6482 } 6483 6484 static void 6485 hn_xmit_txeof_taskfunc(void *xtxr, int pending __unused) 6486 { 6487 struct hn_tx_ring *txr = xtxr; 6488 6489 mtx_lock(&txr->hn_tx_lock); 6490 txr->hn_oactive = 0; 6491 hn_xmit(txr, 0); 6492 mtx_unlock(&txr->hn_tx_lock); 6493 } 6494 6495 static int 6496 hn_chan_attach(struct hn_softc *sc, struct vmbus_channel *chan) 6497 { 6498 struct vmbus_chan_br cbr; 6499 struct hn_rx_ring *rxr; 6500 struct hn_tx_ring *txr = NULL; 6501 int idx, error; 6502 6503 idx = vmbus_chan_subidx(chan); 6504 6505 /* 6506 * Link this channel to RX/TX ring. 6507 */ 6508 KASSERT(idx >= 0 && idx < sc->hn_rx_ring_inuse, 6509 ("invalid channel index %d, should > 0 && < %d", 6510 idx, sc->hn_rx_ring_inuse)); 6511 rxr = &sc->hn_rx_ring[idx]; 6512 KASSERT((rxr->hn_rx_flags & HN_RX_FLAG_ATTACHED) == 0, 6513 ("RX ring %d already attached", idx)); 6514 rxr->hn_rx_flags |= HN_RX_FLAG_ATTACHED; 6515 rxr->hn_chan = chan; 6516 6517 if (bootverbose) { 6518 if_printf(sc->hn_ifp, "link RX ring %d to chan%u\n", 6519 idx, vmbus_chan_id(chan)); 6520 } 6521 6522 if (idx < sc->hn_tx_ring_inuse) { 6523 txr = &sc->hn_tx_ring[idx]; 6524 KASSERT((txr->hn_tx_flags & HN_TX_FLAG_ATTACHED) == 0, 6525 ("TX ring %d already attached", idx)); 6526 txr->hn_tx_flags |= HN_TX_FLAG_ATTACHED; 6527 6528 txr->hn_chan = chan; 6529 if (bootverbose) { 6530 if_printf(sc->hn_ifp, "link TX ring %d to chan%u\n", 6531 idx, vmbus_chan_id(chan)); 6532 } 6533 } 6534 6535 /* Bind this channel to a proper CPU. */ 6536 vmbus_chan_cpu_set(chan, HN_RING_IDX2CPU(sc, idx)); 6537 6538 /* 6539 * Open this channel 6540 */ 6541 cbr.cbr = rxr->hn_br; 6542 cbr.cbr_paddr = pmap_kextract((vm_offset_t)rxr->hn_br); 6543 cbr.cbr_txsz = HN_TXBR_SIZE; 6544 cbr.cbr_rxsz = HN_RXBR_SIZE; 6545 error = vmbus_chan_open_br(chan, &cbr, NULL, 0, hn_chan_callback, rxr); 6546 if (error) { 6547 if (error == EISCONN) { 6548 if_printf(sc->hn_ifp, "bufring is connected after " 6549 "chan%u open failure\n", vmbus_chan_id(chan)); 6550 rxr->hn_rx_flags |= HN_RX_FLAG_BR_REF; 6551 } else { 6552 if_printf(sc->hn_ifp, "open chan%u failed: %d\n", 6553 vmbus_chan_id(chan), error); 6554 } 6555 } 6556 return (error); 6557 } 6558 6559 static void 6560 hn_chan_detach(struct hn_softc *sc, struct vmbus_channel *chan) 6561 { 6562 struct hn_rx_ring *rxr; 6563 int idx, error; 6564 6565 idx = vmbus_chan_subidx(chan); 6566 6567 /* 6568 * Link this channel to RX/TX ring. 6569 */ 6570 KASSERT(idx >= 0 && idx < sc->hn_rx_ring_inuse, 6571 ("invalid channel index %d, should > 0 && < %d", 6572 idx, sc->hn_rx_ring_inuse)); 6573 rxr = &sc->hn_rx_ring[idx]; 6574 KASSERT((rxr->hn_rx_flags & HN_RX_FLAG_ATTACHED), 6575 ("RX ring %d is not attached", idx)); 6576 rxr->hn_rx_flags &= ~HN_RX_FLAG_ATTACHED; 6577 6578 if (idx < sc->hn_tx_ring_inuse) { 6579 struct hn_tx_ring *txr = &sc->hn_tx_ring[idx]; 6580 6581 KASSERT((txr->hn_tx_flags & HN_TX_FLAG_ATTACHED), 6582 ("TX ring %d is not attached attached", idx)); 6583 txr->hn_tx_flags &= ~HN_TX_FLAG_ATTACHED; 6584 } 6585 6586 /* 6587 * Close this channel. 6588 * 6589 * NOTE: 6590 * Channel closing does _not_ destroy the target channel. 6591 */ 6592 error = vmbus_chan_close_direct(chan); 6593 if (error == EISCONN) { 6594 if_printf(sc->hn_ifp, "chan%u bufring is connected " 6595 "after being closed\n", vmbus_chan_id(chan)); 6596 rxr->hn_rx_flags |= HN_RX_FLAG_BR_REF; 6597 } else if (error) { 6598 if_printf(sc->hn_ifp, "chan%u close failed: %d\n", 6599 vmbus_chan_id(chan), error); 6600 } 6601 } 6602 6603 static int 6604 hn_attach_subchans(struct hn_softc *sc) 6605 { 6606 struct vmbus_channel **subchans; 6607 int subchan_cnt = sc->hn_rx_ring_inuse - 1; 6608 int i, error = 0; 6609 6610 KASSERT(subchan_cnt > 0, ("no sub-channels")); 6611 6612 /* Attach the sub-channels. */ 6613 subchans = vmbus_subchan_get(sc->hn_prichan, subchan_cnt); 6614 for (i = 0; i < subchan_cnt; ++i) { 6615 int error1; 6616 6617 error1 = hn_chan_attach(sc, subchans[i]); 6618 if (error1) { 6619 error = error1; 6620 /* Move on; all channels will be detached later. */ 6621 } 6622 } 6623 vmbus_subchan_rel(subchans, subchan_cnt); 6624 6625 if (error) { 6626 if_printf(sc->hn_ifp, "sub-channels attach failed: %d\n", error); 6627 } else { 6628 if (bootverbose) { 6629 if_printf(sc->hn_ifp, "%d sub-channels attached\n", 6630 subchan_cnt); 6631 } 6632 } 6633 return (error); 6634 } 6635 6636 static void 6637 hn_detach_allchans(struct hn_softc *sc) 6638 { 6639 struct vmbus_channel **subchans; 6640 int subchan_cnt = sc->hn_rx_ring_inuse - 1; 6641 int i; 6642 6643 if (subchan_cnt == 0) 6644 goto back; 6645 6646 /* Detach the sub-channels. */ 6647 subchans = vmbus_subchan_get(sc->hn_prichan, subchan_cnt); 6648 for (i = 0; i < subchan_cnt; ++i) 6649 hn_chan_detach(sc, subchans[i]); 6650 vmbus_subchan_rel(subchans, subchan_cnt); 6651 6652 back: 6653 /* 6654 * Detach the primary channel, _after_ all sub-channels 6655 * are detached. 6656 */ 6657 hn_chan_detach(sc, sc->hn_prichan); 6658 6659 /* Wait for sub-channels to be destroyed, if any. */ 6660 vmbus_subchan_drain(sc->hn_prichan); 6661 6662 #ifdef INVARIANTS 6663 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 6664 KASSERT((sc->hn_rx_ring[i].hn_rx_flags & 6665 HN_RX_FLAG_ATTACHED) == 0, 6666 ("%dth RX ring is still attached", i)); 6667 } 6668 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 6669 KASSERT((sc->hn_tx_ring[i].hn_tx_flags & 6670 HN_TX_FLAG_ATTACHED) == 0, 6671 ("%dth TX ring is still attached", i)); 6672 } 6673 #endif 6674 } 6675 6676 static int 6677 hn_synth_alloc_subchans(struct hn_softc *sc, int *nsubch) 6678 { 6679 struct vmbus_channel **subchans; 6680 int nchan, rxr_cnt, error; 6681 6682 nchan = *nsubch + 1; 6683 if (nchan == 1) { 6684 /* 6685 * Multiple RX/TX rings are not requested. 6686 */ 6687 *nsubch = 0; 6688 return (0); 6689 } 6690 6691 /* 6692 * Query RSS capabilities, e.g. # of RX rings, and # of indirect 6693 * table entries. 6694 */ 6695 error = hn_rndis_query_rsscaps(sc, &rxr_cnt); 6696 if (error) { 6697 /* No RSS; this is benign. */ 6698 *nsubch = 0; 6699 return (0); 6700 } 6701 if (bootverbose) { 6702 if_printf(sc->hn_ifp, "RX rings offered %u, requested %d\n", 6703 rxr_cnt, nchan); 6704 } 6705 6706 if (nchan > rxr_cnt) 6707 nchan = rxr_cnt; 6708 if (nchan == 1) { 6709 if_printf(sc->hn_ifp, "only 1 channel is supported, no vRSS\n"); 6710 *nsubch = 0; 6711 return (0); 6712 } 6713 6714 /* 6715 * Allocate sub-channels from NVS. 6716 */ 6717 *nsubch = nchan - 1; 6718 error = hn_nvs_alloc_subchans(sc, nsubch); 6719 if (error || *nsubch == 0) { 6720 /* Failed to allocate sub-channels. */ 6721 *nsubch = 0; 6722 return (0); 6723 } 6724 6725 /* 6726 * Wait for all sub-channels to become ready before moving on. 6727 */ 6728 subchans = vmbus_subchan_get(sc->hn_prichan, *nsubch); 6729 vmbus_subchan_rel(subchans, *nsubch); 6730 return (0); 6731 } 6732 6733 static bool 6734 hn_synth_attachable(const struct hn_softc *sc) 6735 { 6736 int i; 6737 6738 if (sc->hn_flags & HN_FLAG_ERRORS) 6739 return (false); 6740 6741 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 6742 const struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 6743 6744 if (rxr->hn_rx_flags & HN_RX_FLAG_BR_REF) 6745 return (false); 6746 } 6747 return (true); 6748 } 6749 6750 /* 6751 * Make sure that the RX filter is zero after the successful 6752 * RNDIS initialization. 6753 * 6754 * NOTE: 6755 * Under certain conditions on certain versions of Hyper-V, 6756 * the RNDIS rxfilter is _not_ zero on the hypervisor side 6757 * after the successful RNDIS initialization, which breaks 6758 * the assumption of any following code (well, it breaks the 6759 * RNDIS API contract actually). Clear the RNDIS rxfilter 6760 * explicitly, drain packets sneaking through, and drain the 6761 * interrupt taskqueues scheduled due to the stealth packets. 6762 */ 6763 static void 6764 hn_rndis_init_fixat(struct hn_softc *sc, int nchan) 6765 { 6766 6767 hn_disable_rx(sc); 6768 hn_drain_rxtx(sc, nchan); 6769 } 6770 6771 static int 6772 hn_synth_attach(struct hn_softc *sc, int mtu) 6773 { 6774 #define ATTACHED_NVS 0x0002 6775 #define ATTACHED_RNDIS 0x0004 6776 6777 struct ndis_rssprm_toeplitz *rss = &sc->hn_rss; 6778 int error, nsubch, nchan = 1, i, rndis_inited; 6779 uint32_t old_caps, attached = 0; 6780 6781 KASSERT((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0, 6782 ("synthetic parts were attached")); 6783 6784 if (!hn_synth_attachable(sc)) 6785 return (ENXIO); 6786 6787 /* Save capabilities for later verification. */ 6788 old_caps = sc->hn_caps; 6789 sc->hn_caps = 0; 6790 6791 /* Clear RSS stuffs. */ 6792 sc->hn_rss_ind_size = 0; 6793 sc->hn_rss_hash = 0; 6794 sc->hn_rss_hcap = 0; 6795 6796 /* 6797 * Attach the primary channel _before_ attaching NVS and RNDIS. 6798 */ 6799 atomic_store_rel_int(&sc->hn_vf_assoc, 6800 (atomic_load_int(&sc->hn_vf_assoc) + HN_VF_ASSOC_GENINC) & 6801 ~HN_VF_ASSOC_ALLOCATED); 6802 error = hn_chan_attach(sc, sc->hn_prichan); 6803 if (error) 6804 goto failed; 6805 6806 /* 6807 * Attach NVS. 6808 */ 6809 error = hn_nvs_attach(sc, mtu); 6810 if (error) 6811 goto failed; 6812 attached |= ATTACHED_NVS; 6813 6814 /* 6815 * Attach RNDIS _after_ NVS is attached. 6816 */ 6817 error = hn_rndis_attach(sc, mtu, &rndis_inited); 6818 if (rndis_inited) 6819 attached |= ATTACHED_RNDIS; 6820 if (error) 6821 goto failed; 6822 6823 /* 6824 * Make sure capabilities are not changed. 6825 */ 6826 if (device_is_attached(sc->hn_dev) && old_caps != sc->hn_caps) { 6827 if_printf(sc->hn_ifp, "caps mismatch old 0x%08x, new 0x%08x\n", 6828 old_caps, sc->hn_caps); 6829 error = ENXIO; 6830 goto failed; 6831 } 6832 6833 /* 6834 * Allocate sub-channels for multi-TX/RX rings. 6835 * 6836 * NOTE: 6837 * The # of RX rings that can be used is equivalent to the # of 6838 * channels to be requested. 6839 */ 6840 nsubch = sc->hn_rx_ring_cnt - 1; 6841 error = hn_synth_alloc_subchans(sc, &nsubch); 6842 if (error) 6843 goto failed; 6844 /* NOTE: _Full_ synthetic parts detach is required now. */ 6845 sc->hn_flags |= HN_FLAG_SYNTH_ATTACHED; 6846 6847 /* 6848 * Set the # of TX/RX rings that could be used according to 6849 * the # of channels that NVS offered. 6850 */ 6851 nchan = nsubch + 1; 6852 hn_set_ring_inuse(sc, nchan); 6853 if (nchan == 1) { 6854 /* Only the primary channel can be used; done */ 6855 goto back; 6856 } 6857 6858 /* 6859 * Attach the sub-channels. 6860 * 6861 * NOTE: hn_set_ring_inuse() _must_ have been called. 6862 */ 6863 error = hn_attach_subchans(sc); 6864 if (error) 6865 goto failed; 6866 6867 /* 6868 * Configure RSS key and indirect table _after_ all sub-channels 6869 * are attached. 6870 */ 6871 if ((sc->hn_flags & HN_FLAG_HAS_RSSKEY) == 0) { 6872 /* 6873 * RSS key is not set yet; set it to the default RSS key. 6874 */ 6875 if (bootverbose) 6876 if_printf(sc->hn_ifp, "setup default RSS key\n"); 6877 rss_getkey(rss->rss_key); 6878 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 6879 } 6880 6881 if ((sc->hn_flags & HN_FLAG_HAS_RSSIND) == 0) { 6882 /* 6883 * RSS indirect table is not set yet; set it up in round- 6884 * robin fashion. 6885 */ 6886 if (bootverbose) { 6887 if_printf(sc->hn_ifp, "setup default RSS indirect " 6888 "table\n"); 6889 } 6890 for (i = 0; i < NDIS_HASH_INDCNT; ++i) { 6891 uint32_t subidx; 6892 6893 #ifdef RSS 6894 subidx = rss_get_indirection_to_bucket(i); 6895 #else 6896 subidx = i; 6897 #endif 6898 rss->rss_ind[i] = subidx % nchan; 6899 } 6900 sc->hn_flags |= HN_FLAG_HAS_RSSIND; 6901 } else { 6902 /* 6903 * # of usable channels may be changed, so we have to 6904 * make sure that all entries in RSS indirect table 6905 * are valid. 6906 * 6907 * NOTE: hn_set_ring_inuse() _must_ have been called. 6908 */ 6909 hn_rss_ind_fixup(sc); 6910 } 6911 6912 sc->hn_rss_hash = sc->hn_rss_hcap; 6913 if ((sc->hn_flags & HN_FLAG_RXVF) || 6914 (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 6915 /* NOTE: Don't reconfigure RSS; will do immediately. */ 6916 hn_vf_rss_fixup(sc, false); 6917 } 6918 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE); 6919 if (error) 6920 goto failed; 6921 back: 6922 /* 6923 * Fixup transmission aggregation setup. 6924 */ 6925 hn_set_txagg(sc); 6926 hn_rndis_init_fixat(sc, nchan); 6927 return (0); 6928 6929 failed: 6930 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 6931 hn_rndis_init_fixat(sc, nchan); 6932 hn_synth_detach(sc); 6933 } else { 6934 if (attached & ATTACHED_RNDIS) { 6935 hn_rndis_init_fixat(sc, nchan); 6936 hn_rndis_detach(sc); 6937 } 6938 if (attached & ATTACHED_NVS) 6939 hn_nvs_detach(sc); 6940 hn_chan_detach(sc, sc->hn_prichan); 6941 /* Restore old capabilities. */ 6942 sc->hn_caps = old_caps; 6943 } 6944 return (error); 6945 6946 #undef ATTACHED_RNDIS 6947 #undef ATTACHED_NVS 6948 } 6949 6950 /* 6951 * NOTE: 6952 * The interface must have been suspended though hn_suspend(), before 6953 * this function get called. 6954 */ 6955 static void 6956 hn_synth_detach(struct hn_softc *sc) 6957 { 6958 6959 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 6960 ("synthetic parts were not attached")); 6961 6962 /* Detach the RNDIS first. */ 6963 hn_rndis_detach(sc); 6964 6965 /* Detach NVS. */ 6966 hn_nvs_detach(sc); 6967 6968 /* Detach all of the channels. */ 6969 hn_detach_allchans(sc); 6970 6971 if (vmbus_current_version >= VMBUS_VERSION_WIN10 && sc->hn_rxbuf_gpadl != 0) { 6972 /* 6973 * Host is post-Win2016, disconnect RXBUF from primary channel here. 6974 */ 6975 int error; 6976 6977 error = vmbus_chan_gpadl_disconnect(sc->hn_prichan, 6978 sc->hn_rxbuf_gpadl); 6979 if (error) { 6980 if_printf(sc->hn_ifp, 6981 "rxbuf gpadl disconn failed: %d\n", error); 6982 sc->hn_flags |= HN_FLAG_RXBUF_REF; 6983 } 6984 sc->hn_rxbuf_gpadl = 0; 6985 } 6986 6987 if (vmbus_current_version >= VMBUS_VERSION_WIN10 && sc->hn_chim_gpadl != 0) { 6988 /* 6989 * Host is post-Win2016, disconnect chimney sending buffer from 6990 * primary channel here. 6991 */ 6992 int error; 6993 6994 error = vmbus_chan_gpadl_disconnect(sc->hn_prichan, 6995 sc->hn_chim_gpadl); 6996 if (error) { 6997 if_printf(sc->hn_ifp, 6998 "chim gpadl disconn failed: %d\n", error); 6999 sc->hn_flags |= HN_FLAG_CHIM_REF; 7000 } 7001 sc->hn_chim_gpadl = 0; 7002 } 7003 sc->hn_flags &= ~HN_FLAG_SYNTH_ATTACHED; 7004 } 7005 7006 static void 7007 hn_set_ring_inuse(struct hn_softc *sc, int ring_cnt) 7008 { 7009 KASSERT(ring_cnt > 0 && ring_cnt <= sc->hn_rx_ring_cnt, 7010 ("invalid ring count %d", ring_cnt)); 7011 7012 if (sc->hn_tx_ring_cnt > ring_cnt) 7013 sc->hn_tx_ring_inuse = ring_cnt; 7014 else 7015 sc->hn_tx_ring_inuse = sc->hn_tx_ring_cnt; 7016 sc->hn_rx_ring_inuse = ring_cnt; 7017 7018 #ifdef RSS 7019 if (sc->hn_rx_ring_inuse != rss_getnumbuckets()) { 7020 if_printf(sc->hn_ifp, "# of RX rings (%d) does not match " 7021 "# of RSS buckets (%d)\n", sc->hn_rx_ring_inuse, 7022 rss_getnumbuckets()); 7023 } 7024 #endif 7025 7026 if (bootverbose) { 7027 if_printf(sc->hn_ifp, "%d TX ring, %d RX ring\n", 7028 sc->hn_tx_ring_inuse, sc->hn_rx_ring_inuse); 7029 } 7030 } 7031 7032 static void 7033 hn_chan_drain(struct hn_softc *sc, struct vmbus_channel *chan) 7034 { 7035 7036 /* 7037 * NOTE: 7038 * The TX bufring will not be drained by the hypervisor, 7039 * if the primary channel is revoked. 7040 */ 7041 while (!vmbus_chan_rx_empty(chan) || 7042 (!vmbus_chan_is_revoked(sc->hn_prichan) && 7043 !vmbus_chan_tx_empty(chan))) 7044 pause("waitch", 1); 7045 vmbus_chan_intr_drain(chan); 7046 } 7047 7048 static void 7049 hn_disable_rx(struct hn_softc *sc) 7050 { 7051 7052 /* 7053 * Disable RX by clearing RX filter forcefully. 7054 */ 7055 sc->hn_rx_filter = NDIS_PACKET_TYPE_NONE; 7056 hn_rndis_set_rxfilter(sc, sc->hn_rx_filter); /* ignore error */ 7057 7058 /* 7059 * Give RNDIS enough time to flush all pending data packets. 7060 */ 7061 pause("waitrx", (200 * hz) / 1000); 7062 } 7063 7064 /* 7065 * NOTE: 7066 * RX/TX _must_ have been suspended/disabled, before this function 7067 * is called. 7068 */ 7069 static void 7070 hn_drain_rxtx(struct hn_softc *sc, int nchan) 7071 { 7072 struct vmbus_channel **subch = NULL; 7073 int nsubch; 7074 7075 /* 7076 * Drain RX/TX bufrings and interrupts. 7077 */ 7078 nsubch = nchan - 1; 7079 if (nsubch > 0) 7080 subch = vmbus_subchan_get(sc->hn_prichan, nsubch); 7081 7082 if (subch != NULL) { 7083 int i; 7084 7085 for (i = 0; i < nsubch; ++i) 7086 hn_chan_drain(sc, subch[i]); 7087 } 7088 hn_chan_drain(sc, sc->hn_prichan); 7089 7090 if (subch != NULL) 7091 vmbus_subchan_rel(subch, nsubch); 7092 } 7093 7094 static void 7095 hn_suspend_data(struct hn_softc *sc) 7096 { 7097 struct hn_tx_ring *txr; 7098 int i; 7099 7100 HN_LOCK_ASSERT(sc); 7101 7102 /* 7103 * Suspend TX. 7104 */ 7105 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 7106 txr = &sc->hn_tx_ring[i]; 7107 7108 mtx_lock(&txr->hn_tx_lock); 7109 txr->hn_suspended = 1; 7110 mtx_unlock(&txr->hn_tx_lock); 7111 /* No one is able send more packets now. */ 7112 7113 /* 7114 * Wait for all pending sends to finish. 7115 * 7116 * NOTE: 7117 * We will _not_ receive all pending send-done, if the 7118 * primary channel is revoked. 7119 */ 7120 while (hn_tx_ring_pending(txr) && 7121 !vmbus_chan_is_revoked(sc->hn_prichan)) 7122 pause("hnwtx", 1 /* 1 tick */); 7123 } 7124 7125 /* 7126 * Disable RX. 7127 */ 7128 hn_disable_rx(sc); 7129 7130 /* 7131 * Drain RX/TX. 7132 */ 7133 hn_drain_rxtx(sc, sc->hn_rx_ring_inuse); 7134 7135 /* 7136 * Drain any pending TX tasks. 7137 * 7138 * NOTE: 7139 * The above hn_drain_rxtx() can dispatch TX tasks, so the TX 7140 * tasks will have to be drained _after_ the above hn_drain_rxtx(). 7141 */ 7142 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 7143 txr = &sc->hn_tx_ring[i]; 7144 7145 taskqueue_drain(txr->hn_tx_taskq, &txr->hn_tx_task); 7146 taskqueue_drain(txr->hn_tx_taskq, &txr->hn_txeof_task); 7147 } 7148 } 7149 7150 static void 7151 hn_suspend_mgmt_taskfunc(void *xsc, int pending __unused) 7152 { 7153 7154 ((struct hn_softc *)xsc)->hn_mgmt_taskq = NULL; 7155 } 7156 7157 static void 7158 hn_suspend_mgmt(struct hn_softc *sc) 7159 { 7160 struct task task; 7161 7162 HN_LOCK_ASSERT(sc); 7163 7164 /* 7165 * Make sure that hn_mgmt_taskq0 can nolonger be accessed 7166 * through hn_mgmt_taskq. 7167 */ 7168 TASK_INIT(&task, 0, hn_suspend_mgmt_taskfunc, sc); 7169 vmbus_chan_run_task(sc->hn_prichan, &task); 7170 7171 /* 7172 * Make sure that all pending management tasks are completed. 7173 */ 7174 taskqueue_drain(sc->hn_mgmt_taskq0, &sc->hn_netchg_init); 7175 taskqueue_drain_timeout(sc->hn_mgmt_taskq0, &sc->hn_netchg_status); 7176 taskqueue_drain_all(sc->hn_mgmt_taskq0); 7177 } 7178 7179 static void 7180 hn_suspend(struct hn_softc *sc) 7181 { 7182 7183 /* Disable polling. */ 7184 hn_polling(sc, 0); 7185 7186 /* 7187 * If the non-transparent mode VF is activated, the synthetic 7188 * device is receiving packets, so the data path of the 7189 * synthetic device must be suspended. 7190 */ 7191 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) || 7192 (sc->hn_flags & HN_FLAG_RXVF)) 7193 hn_suspend_data(sc); 7194 hn_suspend_mgmt(sc); 7195 } 7196 7197 static void 7198 hn_resume_tx(struct hn_softc *sc, int tx_ring_cnt) 7199 { 7200 int i; 7201 7202 KASSERT(tx_ring_cnt <= sc->hn_tx_ring_cnt, 7203 ("invalid TX ring count %d", tx_ring_cnt)); 7204 7205 for (i = 0; i < tx_ring_cnt; ++i) { 7206 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 7207 7208 mtx_lock(&txr->hn_tx_lock); 7209 txr->hn_suspended = 0; 7210 mtx_unlock(&txr->hn_tx_lock); 7211 } 7212 } 7213 7214 static void 7215 hn_resume_data(struct hn_softc *sc) 7216 { 7217 int i; 7218 7219 HN_LOCK_ASSERT(sc); 7220 7221 /* 7222 * Re-enable RX. 7223 */ 7224 hn_rxfilter_config(sc); 7225 7226 /* 7227 * Make sure to clear suspend status on "all" TX rings, 7228 * since hn_tx_ring_inuse can be changed after 7229 * hn_suspend_data(). 7230 */ 7231 hn_resume_tx(sc, sc->hn_tx_ring_cnt); 7232 7233 #ifdef HN_IFSTART_SUPPORT 7234 if (!hn_use_if_start) 7235 #endif 7236 { 7237 /* 7238 * Flush unused drbrs, since hn_tx_ring_inuse may be 7239 * reduced. 7240 */ 7241 for (i = sc->hn_tx_ring_inuse; i < sc->hn_tx_ring_cnt; ++i) 7242 hn_tx_ring_qflush(&sc->hn_tx_ring[i]); 7243 } 7244 7245 /* 7246 * Kick start TX. 7247 */ 7248 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 7249 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 7250 7251 /* 7252 * Use txeof task, so that any pending oactive can be 7253 * cleared properly. 7254 */ 7255 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 7256 } 7257 } 7258 7259 static void 7260 hn_resume_mgmt(struct hn_softc *sc) 7261 { 7262 7263 sc->hn_mgmt_taskq = sc->hn_mgmt_taskq0; 7264 7265 /* 7266 * Kick off network change detection, if it was pending. 7267 * If no network change was pending, start link status 7268 * checks, which is more lightweight than network change 7269 * detection. 7270 */ 7271 if (sc->hn_link_flags & HN_LINK_FLAG_NETCHG) 7272 hn_change_network(sc); 7273 else 7274 hn_update_link_status(sc); 7275 } 7276 7277 static void 7278 hn_resume(struct hn_softc *sc) 7279 { 7280 7281 /* 7282 * If the non-transparent mode VF is activated, the synthetic 7283 * device have to receive packets, so the data path of the 7284 * synthetic device must be resumed. 7285 */ 7286 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) || 7287 (sc->hn_flags & HN_FLAG_RXVF)) 7288 hn_resume_data(sc); 7289 7290 /* 7291 * Don't resume link status change if VF is activated. 7292 * - In the non-transparent VF mode, the synthetic device marks 7293 * link down until the VF is deactivated; i.e. VF is down. 7294 * - In transparent VF mode, VF's media status is used until 7295 * the VF is deactivated. 7296 */ 7297 if ((sc->hn_flags & HN_FLAG_RXVF) == 0 && 7298 !(sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) 7299 hn_resume_mgmt(sc); 7300 7301 /* 7302 * Re-enable polling if this interface is running and 7303 * the polling is requested. 7304 */ 7305 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) && sc->hn_pollhz > 0) 7306 hn_polling(sc, sc->hn_pollhz); 7307 } 7308 7309 static void 7310 hn_rndis_rx_status(struct hn_softc *sc, const void *data, int dlen) 7311 { 7312 const struct rndis_status_msg *msg; 7313 int ofs; 7314 7315 if (dlen < sizeof(*msg)) { 7316 if_printf(sc->hn_ifp, "invalid RNDIS status\n"); 7317 return; 7318 } 7319 msg = data; 7320 7321 switch (msg->rm_status) { 7322 case RNDIS_STATUS_MEDIA_CONNECT: 7323 case RNDIS_STATUS_MEDIA_DISCONNECT: 7324 hn_update_link_status(sc); 7325 break; 7326 7327 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 7328 case RNDIS_STATUS_LINK_SPEED_CHANGE: 7329 /* Not really useful; ignore. */ 7330 break; 7331 7332 case RNDIS_STATUS_NETWORK_CHANGE: 7333 ofs = RNDIS_STBUFOFFSET_ABS(msg->rm_stbufoffset); 7334 if (dlen < ofs + msg->rm_stbuflen || 7335 msg->rm_stbuflen < sizeof(uint32_t)) { 7336 if_printf(sc->hn_ifp, "network changed\n"); 7337 } else { 7338 uint32_t change; 7339 7340 memcpy(&change, ((const uint8_t *)msg) + ofs, 7341 sizeof(change)); 7342 if_printf(sc->hn_ifp, "network changed, change %u\n", 7343 change); 7344 } 7345 hn_change_network(sc); 7346 break; 7347 7348 default: 7349 if_printf(sc->hn_ifp, "unknown RNDIS status 0x%08x\n", 7350 msg->rm_status); 7351 break; 7352 } 7353 } 7354 7355 static int 7356 hn_rndis_rxinfo(const void *info_data, int info_dlen, struct hn_rxinfo *info) 7357 { 7358 const struct rndis_pktinfo *pi = info_data; 7359 uint32_t mask = 0; 7360 7361 while (info_dlen != 0) { 7362 const void *data; 7363 uint32_t dlen; 7364 7365 if (__predict_false(info_dlen < sizeof(*pi))) 7366 return (EINVAL); 7367 if (__predict_false(info_dlen < pi->rm_size)) 7368 return (EINVAL); 7369 info_dlen -= pi->rm_size; 7370 7371 if (__predict_false(pi->rm_size & RNDIS_PKTINFO_SIZE_ALIGNMASK)) 7372 return (EINVAL); 7373 if (__predict_false(pi->rm_size < pi->rm_pktinfooffset)) 7374 return (EINVAL); 7375 dlen = pi->rm_size - pi->rm_pktinfooffset; 7376 data = pi->rm_data; 7377 7378 if (pi->rm_internal == 1) { 7379 switch (pi->rm_type) { 7380 case NDIS_PKTINFO_IT_PKTINFO_ID: 7381 if (__predict_false(dlen < NDIS_PKTINFOID_SZ)) 7382 return (EINVAL); 7383 info->pktinfo_id = 7384 (const struct packet_info_id *)data; 7385 mask |= HN_RXINFO_PKTINFO_ID; 7386 break; 7387 7388 default: 7389 goto next; 7390 } 7391 } else { 7392 switch (pi->rm_type) { 7393 case NDIS_PKTINFO_TYPE_VLAN: 7394 if (__predict_false(dlen 7395 < NDIS_VLAN_INFO_SIZE)) 7396 return (EINVAL); 7397 info->vlan_info = (const uint32_t *)data; 7398 mask |= HN_RXINFO_VLAN; 7399 break; 7400 7401 case NDIS_PKTINFO_TYPE_CSUM: 7402 if (__predict_false(dlen 7403 < NDIS_RXCSUM_INFO_SIZE)) 7404 return (EINVAL); 7405 info->csum_info = (const uint32_t *)data; 7406 mask |= HN_RXINFO_CSUM; 7407 break; 7408 7409 case HN_NDIS_PKTINFO_TYPE_HASHVAL: 7410 if (__predict_false(dlen 7411 < HN_NDIS_HASH_VALUE_SIZE)) 7412 return (EINVAL); 7413 info->hash_value = (const uint32_t *)data; 7414 mask |= HN_RXINFO_HASHVAL; 7415 break; 7416 7417 case HN_NDIS_PKTINFO_TYPE_HASHINF: 7418 if (__predict_false(dlen 7419 < HN_NDIS_HASH_INFO_SIZE)) 7420 return (EINVAL); 7421 info->hash_info = (const uint32_t *)data; 7422 mask |= HN_RXINFO_HASHINF; 7423 break; 7424 7425 default: 7426 goto next; 7427 } 7428 } 7429 7430 if (mask == HN_RXINFO_ALL) { 7431 /* All found; done */ 7432 break; 7433 } 7434 next: 7435 pi = (const struct rndis_pktinfo *) 7436 ((const uint8_t *)pi + pi->rm_size); 7437 } 7438 7439 /* 7440 * Final fixup. 7441 * - If there is no hash value, invalidate the hash info. 7442 */ 7443 if ((mask & HN_RXINFO_HASHVAL) == 0) 7444 info->hash_info = NULL; 7445 return (0); 7446 } 7447 7448 static __inline bool 7449 hn_rndis_check_overlap(int off, int len, int check_off, int check_len) 7450 { 7451 7452 if (off < check_off) { 7453 if (__predict_true(off + len <= check_off)) 7454 return (false); 7455 } else if (off > check_off) { 7456 if (__predict_true(check_off + check_len <= off)) 7457 return (false); 7458 } 7459 return (true); 7460 } 7461 7462 static __inline void 7463 hn_rsc_add_data(struct hn_rx_ring *rxr, const void *data, 7464 uint32_t len, struct hn_rxinfo *info) 7465 { 7466 uint32_t cnt = rxr->rsc.cnt; 7467 7468 if (cnt) { 7469 rxr->rsc.pktlen += len; 7470 } else { 7471 rxr->rsc.vlan_info = info->vlan_info; 7472 rxr->rsc.csum_info = info->csum_info; 7473 rxr->rsc.hash_info = info->hash_info; 7474 rxr->rsc.hash_value = info->hash_value; 7475 rxr->rsc.pktlen = len; 7476 } 7477 7478 rxr->rsc.frag_data[cnt] = data; 7479 rxr->rsc.frag_len[cnt] = len; 7480 rxr->rsc.cnt++; 7481 } 7482 7483 static void 7484 hn_rndis_rx_data(struct hn_rx_ring *rxr, const void *data, int dlen) 7485 { 7486 const struct rndis_packet_msg *pkt; 7487 struct hn_rxinfo info; 7488 int data_off, pktinfo_off, data_len, pktinfo_len; 7489 bool rsc_more= false; 7490 7491 /* 7492 * Check length. 7493 */ 7494 if (__predict_false(dlen < sizeof(*pkt))) { 7495 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg\n"); 7496 return; 7497 } 7498 pkt = data; 7499 7500 if (__predict_false(dlen < pkt->rm_len)) { 7501 if_printf(rxr->hn_ifp, "truncated RNDIS packet msg, " 7502 "dlen %d, msglen %u\n", dlen, pkt->rm_len); 7503 return; 7504 } 7505 if (__predict_false(pkt->rm_len < 7506 pkt->rm_datalen + pkt->rm_oobdatalen + pkt->rm_pktinfolen)) { 7507 if_printf(rxr->hn_ifp, "invalid RNDIS packet msglen, " 7508 "msglen %u, data %u, oob %u, pktinfo %u\n", 7509 pkt->rm_len, pkt->rm_datalen, pkt->rm_oobdatalen, 7510 pkt->rm_pktinfolen); 7511 return; 7512 } 7513 if (__predict_false(pkt->rm_datalen == 0)) { 7514 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, no data\n"); 7515 return; 7516 } 7517 7518 /* 7519 * Check offests. 7520 */ 7521 #define IS_OFFSET_INVALID(ofs) \ 7522 ((ofs) < RNDIS_PACKET_MSG_OFFSET_MIN || \ 7523 ((ofs) & RNDIS_PACKET_MSG_OFFSET_ALIGNMASK)) 7524 7525 /* XXX Hyper-V does not meet data offset alignment requirement */ 7526 if (__predict_false(pkt->rm_dataoffset < RNDIS_PACKET_MSG_OFFSET_MIN)) { 7527 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7528 "data offset %u\n", pkt->rm_dataoffset); 7529 return; 7530 } 7531 if (__predict_false(pkt->rm_oobdataoffset > 0 && 7532 IS_OFFSET_INVALID(pkt->rm_oobdataoffset))) { 7533 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7534 "oob offset %u\n", pkt->rm_oobdataoffset); 7535 return; 7536 } 7537 if (__predict_true(pkt->rm_pktinfooffset > 0) && 7538 __predict_false(IS_OFFSET_INVALID(pkt->rm_pktinfooffset))) { 7539 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7540 "pktinfo offset %u\n", pkt->rm_pktinfooffset); 7541 return; 7542 } 7543 7544 #undef IS_OFFSET_INVALID 7545 7546 data_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_dataoffset); 7547 data_len = pkt->rm_datalen; 7548 pktinfo_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_pktinfooffset); 7549 pktinfo_len = pkt->rm_pktinfolen; 7550 7551 /* 7552 * Check OOB coverage. 7553 */ 7554 if (__predict_false(pkt->rm_oobdatalen != 0)) { 7555 int oob_off, oob_len; 7556 7557 if_printf(rxr->hn_ifp, "got oobdata\n"); 7558 oob_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_oobdataoffset); 7559 oob_len = pkt->rm_oobdatalen; 7560 7561 if (__predict_false(oob_off + oob_len > pkt->rm_len)) { 7562 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7563 "oob overflow, msglen %u, oob abs %d len %d\n", 7564 pkt->rm_len, oob_off, oob_len); 7565 return; 7566 } 7567 7568 /* 7569 * Check against data. 7570 */ 7571 if (hn_rndis_check_overlap(oob_off, oob_len, 7572 data_off, data_len)) { 7573 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7574 "oob overlaps data, oob abs %d len %d, " 7575 "data abs %d len %d\n", 7576 oob_off, oob_len, data_off, data_len); 7577 return; 7578 } 7579 7580 /* 7581 * Check against pktinfo. 7582 */ 7583 if (pktinfo_len != 0 && 7584 hn_rndis_check_overlap(oob_off, oob_len, 7585 pktinfo_off, pktinfo_len)) { 7586 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7587 "oob overlaps pktinfo, oob abs %d len %d, " 7588 "pktinfo abs %d len %d\n", 7589 oob_off, oob_len, pktinfo_off, pktinfo_len); 7590 return; 7591 } 7592 } 7593 7594 /* 7595 * Check per-packet-info coverage and find useful per-packet-info. 7596 */ 7597 info.vlan_info = NULL; 7598 info.csum_info = NULL; 7599 info.hash_info = NULL; 7600 info.pktinfo_id = NULL; 7601 7602 if (__predict_true(pktinfo_len != 0)) { 7603 bool overlap; 7604 int error; 7605 7606 if (__predict_false(pktinfo_off + pktinfo_len > pkt->rm_len)) { 7607 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7608 "pktinfo overflow, msglen %u, " 7609 "pktinfo abs %d len %d\n", 7610 pkt->rm_len, pktinfo_off, pktinfo_len); 7611 return; 7612 } 7613 7614 /* 7615 * Check packet info coverage. 7616 */ 7617 overlap = hn_rndis_check_overlap(pktinfo_off, pktinfo_len, 7618 data_off, data_len); 7619 if (__predict_false(overlap)) { 7620 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7621 "pktinfo overlap data, pktinfo abs %d len %d, " 7622 "data abs %d len %d\n", 7623 pktinfo_off, pktinfo_len, data_off, data_len); 7624 return; 7625 } 7626 7627 /* 7628 * Find useful per-packet-info. 7629 */ 7630 error = hn_rndis_rxinfo(((const uint8_t *)pkt) + pktinfo_off, 7631 pktinfo_len, &info); 7632 if (__predict_false(error)) { 7633 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg " 7634 "pktinfo\n"); 7635 return; 7636 } 7637 } 7638 7639 if (__predict_false(data_off + data_len > pkt->rm_len)) { 7640 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7641 "data overflow, msglen %u, data abs %d len %d\n", 7642 pkt->rm_len, data_off, data_len); 7643 return; 7644 } 7645 7646 /* Identify RSC fragments, drop invalid packets */ 7647 if ((info.pktinfo_id != NULL) && 7648 (info.pktinfo_id->flag & HN_NDIS_PKTINFO_SUBALLOC)) { 7649 if (info.pktinfo_id->flag & HN_NDIS_PKTINFO_1ST_FRAG) { 7650 rxr->rsc.cnt = 0; 7651 rxr->hn_rsc_pkts++; 7652 } else if (rxr->rsc.cnt == 0) 7653 goto drop; 7654 7655 rsc_more = true; 7656 7657 if (info.pktinfo_id->flag & HN_NDIS_PKTINFO_LAST_FRAG) 7658 rsc_more = false; 7659 7660 if (rsc_more && rxr->rsc.is_last) 7661 goto drop; 7662 } else { 7663 rxr->rsc.cnt = 0; 7664 } 7665 7666 if (__predict_false(rxr->rsc.cnt >= HN_NVS_RSC_MAX)) 7667 goto drop; 7668 7669 /* Store data in per rx ring structure */ 7670 hn_rsc_add_data(rxr,((const uint8_t *)pkt) + data_off, 7671 data_len, &info); 7672 7673 if (rsc_more) 7674 return; 7675 7676 hn_rxpkt(rxr); 7677 rxr->rsc.cnt = 0; 7678 return; 7679 drop: 7680 rxr->hn_rsc_drop++; 7681 return; 7682 } 7683 7684 static __inline void 7685 hn_rndis_rxpkt(struct hn_rx_ring *rxr, const void *data, int dlen) 7686 { 7687 const struct rndis_msghdr *hdr; 7688 7689 if (__predict_false(dlen < sizeof(*hdr))) { 7690 if_printf(rxr->hn_ifp, "invalid RNDIS msg\n"); 7691 return; 7692 } 7693 hdr = data; 7694 7695 if (__predict_true(hdr->rm_type == REMOTE_NDIS_PACKET_MSG)) { 7696 /* Hot data path. */ 7697 hn_rndis_rx_data(rxr, data, dlen); 7698 /* Done! */ 7699 return; 7700 } 7701 7702 if (hdr->rm_type == REMOTE_NDIS_INDICATE_STATUS_MSG) 7703 hn_rndis_rx_status(if_getsoftc(rxr->hn_ifp), data, dlen); 7704 else 7705 hn_rndis_rx_ctrl(if_getsoftc(rxr->hn_ifp), data, dlen); 7706 } 7707 7708 static void 7709 hn_nvs_handle_notify(struct hn_softc *sc, const struct vmbus_chanpkt_hdr *pkt) 7710 { 7711 const struct hn_nvs_hdr *hdr; 7712 7713 if (VMBUS_CHANPKT_DATALEN(pkt) < sizeof(*hdr)) { 7714 if_printf(sc->hn_ifp, "invalid nvs notify\n"); 7715 return; 7716 } 7717 hdr = VMBUS_CHANPKT_CONST_DATA(pkt); 7718 7719 if (hdr->nvs_type == HN_NVS_TYPE_VFASSOC_NOTE) { 7720 const struct hn_nvs_vfassoc *assoc; 7721 u_int state; 7722 7723 if (VMBUS_CHANPKT_DATALEN(pkt) < sizeof(*assoc)) { 7724 if_printf(sc->hn_ifp, "short VF association notification\n"); 7725 return; 7726 } 7727 assoc = (const struct hn_nvs_vfassoc *)hdr; 7728 if (assoc->nvs_alloc > 1) { 7729 if_printf(sc->hn_ifp, "invalid VF association notification\n"); 7730 return; 7731 } 7732 /* The serial is diagnostic; hn_ismyvf() matches the VF by MAC. */ 7733 /* Preserve withdrawals even when the worker coalesces notices. */ 7734 state = (atomic_load_int(&sc->hn_vf_assoc) + HN_VF_ASSOC_GENINC) & 7735 ~HN_VF_ASSOC_ALLOCATED; 7736 atomic_store_rel_int(&sc->hn_vf_assoc, state | assoc->nvs_alloc); 7737 if (bootverbose) 7738 if_printf(sc->hn_ifp, "VF %u %s\n", assoc->nvs_serial, 7739 assoc->nvs_alloc ? "associated" : "withdrawn"); 7740 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 7741 return; 7742 } 7743 if (hdr->nvs_type == HN_NVS_TYPE_TXTBL_NOTE) { 7744 /* Useless; ignore */ 7745 return; 7746 } 7747 if_printf(sc->hn_ifp, "got notify, nvs type %u\n", hdr->nvs_type); 7748 } 7749 7750 static void 7751 hn_nvs_handle_comp(struct hn_softc *sc, struct vmbus_channel *chan, 7752 const struct vmbus_chanpkt_hdr *pkt) 7753 { 7754 struct hn_nvs_sendctx *sndc; 7755 7756 sndc = (struct hn_nvs_sendctx *)(uintptr_t)pkt->cph_xactid; 7757 sndc->hn_cb(sndc, sc, chan, VMBUS_CHANPKT_CONST_DATA(pkt), 7758 VMBUS_CHANPKT_DATALEN(pkt)); 7759 /* 7760 * NOTE: 7761 * 'sndc' CAN NOT be accessed anymore, since it can be freed by 7762 * its callback. 7763 */ 7764 } 7765 7766 static void 7767 hn_nvs_handle_rxbuf(struct hn_rx_ring *rxr, struct vmbus_channel *chan, 7768 const struct vmbus_chanpkt_hdr *pkthdr) 7769 { 7770 struct epoch_tracker et; 7771 const struct vmbus_chanpkt_rxbuf *pkt; 7772 const struct hn_nvs_hdr *nvs_hdr; 7773 int count, i, hlen; 7774 7775 if (__predict_false(VMBUS_CHANPKT_DATALEN(pkthdr) < sizeof(*nvs_hdr))) { 7776 if_printf(rxr->hn_ifp, "invalid nvs RNDIS\n"); 7777 return; 7778 } 7779 nvs_hdr = VMBUS_CHANPKT_CONST_DATA(pkthdr); 7780 7781 /* Make sure that this is a RNDIS message. */ 7782 if (__predict_false(nvs_hdr->nvs_type != HN_NVS_TYPE_RNDIS)) { 7783 if_printf(rxr->hn_ifp, "nvs type %u, not RNDIS\n", 7784 nvs_hdr->nvs_type); 7785 return; 7786 } 7787 7788 hlen = VMBUS_CHANPKT_GETLEN(pkthdr->cph_hlen); 7789 if (__predict_false(hlen < sizeof(*pkt))) { 7790 if_printf(rxr->hn_ifp, "invalid rxbuf chanpkt\n"); 7791 return; 7792 } 7793 pkt = (const struct vmbus_chanpkt_rxbuf *)pkthdr; 7794 7795 if (__predict_false(pkt->cp_rxbuf_id != HN_NVS_RXBUF_SIG)) { 7796 if_printf(rxr->hn_ifp, "invalid rxbuf_id 0x%08x\n", 7797 pkt->cp_rxbuf_id); 7798 return; 7799 } 7800 7801 count = pkt->cp_rxbuf_cnt; 7802 if (__predict_false(hlen < 7803 __offsetof(struct vmbus_chanpkt_rxbuf, cp_rxbuf[count]))) { 7804 if_printf(rxr->hn_ifp, "invalid rxbuf_cnt %d\n", count); 7805 return; 7806 } 7807 7808 NET_EPOCH_ENTER(et); 7809 /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */ 7810 for (i = 0; i < count; ++i) { 7811 int ofs, len; 7812 7813 ofs = pkt->cp_rxbuf[i].rb_ofs; 7814 len = pkt->cp_rxbuf[i].rb_len; 7815 if (__predict_false(ofs + len > HN_RXBUF_SIZE)) { 7816 if_printf(rxr->hn_ifp, "%dth RNDIS msg overflow rxbuf, " 7817 "ofs %d, len %d\n", i, ofs, len); 7818 continue; 7819 } 7820 7821 rxr->rsc.is_last = (i == (count - 1)); 7822 hn_rndis_rxpkt(rxr, rxr->hn_rxbuf + ofs, len); 7823 } 7824 NET_EPOCH_EXIT(et); 7825 7826 /* 7827 * Ack the consumed RXBUF associated w/ this channel packet, 7828 * so that this RXBUF can be recycled by the hypervisor. 7829 */ 7830 hn_nvs_ack_rxbuf(rxr, chan, pkt->cp_hdr.cph_xactid); 7831 } 7832 7833 static void 7834 hn_nvs_ack_rxbuf(struct hn_rx_ring *rxr, struct vmbus_channel *chan, 7835 uint64_t tid) 7836 { 7837 struct hn_nvs_rndis_ack ack; 7838 int retries, error; 7839 7840 ack.nvs_type = HN_NVS_TYPE_RNDIS_ACK; 7841 ack.nvs_status = HN_NVS_STATUS_OK; 7842 7843 retries = 0; 7844 again: 7845 error = vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP, 7846 VMBUS_CHANPKT_FLAG_NONE, &ack, sizeof(ack), tid); 7847 if (__predict_false(error == EAGAIN)) { 7848 /* 7849 * NOTE: 7850 * This should _not_ happen in real world, since the 7851 * consumption of the TX bufring from the TX path is 7852 * controlled. 7853 */ 7854 if (rxr->hn_ack_failed == 0) 7855 if_printf(rxr->hn_ifp, "RXBUF ack retry\n"); 7856 rxr->hn_ack_failed++; 7857 retries++; 7858 if (retries < 10) { 7859 DELAY(100); 7860 goto again; 7861 } 7862 /* RXBUF leaks! */ 7863 if_printf(rxr->hn_ifp, "RXBUF ack failed\n"); 7864 } 7865 } 7866 7867 static void 7868 hn_chan_callback(struct vmbus_channel *chan, void *xrxr) 7869 { 7870 struct hn_rx_ring *rxr = xrxr; 7871 struct hn_softc *sc = if_getsoftc(rxr->hn_ifp); 7872 7873 for (;;) { 7874 struct vmbus_chanpkt_hdr *pkt = rxr->hn_pktbuf; 7875 int error, pktlen; 7876 7877 pktlen = rxr->hn_pktbuf_len; 7878 error = vmbus_chan_recv_pkt(chan, pkt, &pktlen); 7879 if (__predict_false(error == ENOBUFS)) { 7880 void *nbuf; 7881 int nlen; 7882 7883 /* 7884 * Expand channel packet buffer. 7885 * 7886 * XXX 7887 * Use M_WAITOK here, since allocation failure 7888 * is fatal. 7889 */ 7890 nlen = rxr->hn_pktbuf_len * 2; 7891 while (nlen < pktlen) 7892 nlen *= 2; 7893 nbuf = malloc(nlen, M_DEVBUF, M_WAITOK); 7894 7895 if_printf(rxr->hn_ifp, "expand pktbuf %d -> %d\n", 7896 rxr->hn_pktbuf_len, nlen); 7897 7898 free(rxr->hn_pktbuf, M_DEVBUF); 7899 rxr->hn_pktbuf = nbuf; 7900 rxr->hn_pktbuf_len = nlen; 7901 /* Retry! */ 7902 continue; 7903 } else if (__predict_false(error == EAGAIN)) { 7904 /* No more channel packets; done! */ 7905 break; 7906 } 7907 KASSERT(!error, ("vmbus_chan_recv_pkt failed: %d", error)); 7908 7909 switch (pkt->cph_type) { 7910 case VMBUS_CHANPKT_TYPE_COMP: 7911 hn_nvs_handle_comp(sc, chan, pkt); 7912 break; 7913 7914 case VMBUS_CHANPKT_TYPE_RXBUF: 7915 hn_nvs_handle_rxbuf(rxr, chan, pkt); 7916 break; 7917 7918 case VMBUS_CHANPKT_TYPE_INBAND: 7919 if (chan == sc->hn_prichan) 7920 hn_nvs_handle_notify(sc, pkt); 7921 break; 7922 7923 default: 7924 if_printf(rxr->hn_ifp, "unknown chan pkt %u\n", 7925 pkt->cph_type); 7926 break; 7927 } 7928 } 7929 hn_chan_rollup(rxr, rxr->hn_txr); 7930 } 7931 7932 static void 7933 hn_sysinit(void *arg __unused) 7934 { 7935 int i; 7936 7937 hn_udpcs_fixup = counter_u64_alloc(M_WAITOK); 7938 7939 #ifdef HN_IFSTART_SUPPORT 7940 /* 7941 * Don't use ifnet.if_start if transparent VF mode is requested; 7942 * mainly due to the IFF_DRV_OACTIVE flag. 7943 */ 7944 if (hn_xpnt_vf && hn_use_if_start) { 7945 hn_use_if_start = 0; 7946 printf("hn: transparent VF mode, if_transmit will be used, " 7947 "instead of if_start\n"); 7948 } 7949 #endif 7950 if (hn_xpnt_vf_attwait < HN_XPNT_VF_ATTWAIT_MIN) { 7951 printf("hn: invalid transparent VF attach routing " 7952 "wait timeout %d, reset to %d\n", 7953 hn_xpnt_vf_attwait, HN_XPNT_VF_ATTWAIT_MIN); 7954 hn_xpnt_vf_attwait = HN_XPNT_VF_ATTWAIT_MIN; 7955 } 7956 7957 /* 7958 * Initialize VF map. 7959 */ 7960 rm_init_flags(&hn_vfmap_lock, "hn_vfmap", RM_SLEEPABLE); 7961 hn_vfmap_size = HN_VFMAP_SIZE_DEF; 7962 hn_vfmap = malloc(sizeof(if_t) * hn_vfmap_size, M_DEVBUF, 7963 M_WAITOK | M_ZERO); 7964 7965 /* 7966 * Fix the # of TX taskqueues. 7967 */ 7968 if (hn_tx_taskq_cnt <= 0) 7969 hn_tx_taskq_cnt = 1; 7970 else if (hn_tx_taskq_cnt > mp_ncpus) 7971 hn_tx_taskq_cnt = mp_ncpus; 7972 7973 /* 7974 * Fix the TX taskqueue mode. 7975 */ 7976 switch (hn_tx_taskq_mode) { 7977 case HN_TX_TASKQ_M_INDEP: 7978 case HN_TX_TASKQ_M_GLOBAL: 7979 case HN_TX_TASKQ_M_EVTTQ: 7980 break; 7981 default: 7982 hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; 7983 break; 7984 } 7985 7986 if (vm_guest != VM_GUEST_HV) 7987 return; 7988 7989 if (hn_tx_taskq_mode != HN_TX_TASKQ_M_GLOBAL) 7990 return; 7991 7992 hn_tx_taskque = malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), 7993 M_DEVBUF, M_WAITOK); 7994 for (i = 0; i < hn_tx_taskq_cnt; ++i) { 7995 hn_tx_taskque[i] = taskqueue_create("hn_tx", M_WAITOK, 7996 taskqueue_thread_enqueue, &hn_tx_taskque[i]); 7997 taskqueue_start_threads(&hn_tx_taskque[i], 1, PI_NET, 7998 "hn tx%d", i); 7999 } 8000 } 8001 SYSINIT(hn_sysinit, SI_SUB_DRIVERS, SI_ORDER_SECOND, hn_sysinit, NULL); 8002 8003 static void 8004 hn_sysuninit(void *arg __unused) 8005 { 8006 8007 if (hn_tx_taskque != NULL) { 8008 int i; 8009 8010 for (i = 0; i < hn_tx_taskq_cnt; ++i) 8011 taskqueue_free(hn_tx_taskque[i]); 8012 free(hn_tx_taskque, M_DEVBUF); 8013 } 8014 8015 if (hn_vfmap != NULL) 8016 free(hn_vfmap, M_DEVBUF); 8017 rm_destroy(&hn_vfmap_lock); 8018 8019 counter_u64_free(hn_udpcs_fixup); 8020 } 8021 SYSUNINIT(hn_sysuninit, SI_SUB_DRIVERS, SI_ORDER_SECOND, hn_sysuninit, NULL); 8022