1 /*- 2 * Copyright (c) 2010-2012 Citrix Inc. 3 * Copyright (c) 2009-2012,2016-2017 Microsoft Corp. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /*- 30 * Copyright (c) 2004-2006 Kip Macy 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 45 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 52 * SUCH DAMAGE. 53 */ 54 55 #include <sys/cdefs.h> 56 #include "opt_hn.h" 57 #include "opt_inet6.h" 58 #include "opt_inet.h" 59 #include "opt_rss.h" 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/bus.h> 64 #include <sys/counter.h> 65 #include <sys/kernel.h> 66 #include <sys/limits.h> 67 #include <sys/malloc.h> 68 #include <sys/mbuf.h> 69 #include <sys/module.h> 70 #include <sys/queue.h> 71 #include <sys/lock.h> 72 #include <sys/proc.h> 73 #include <sys/rmlock.h> 74 #include <sys/sbuf.h> 75 #include <sys/sched.h> 76 #include <sys/smp.h> 77 #include <sys/socket.h> 78 #include <sys/sockio.h> 79 #include <sys/sx.h> 80 #include <sys/sysctl.h> 81 #include <sys/taskqueue.h> 82 #include <sys/buf_ring.h> 83 #include <sys/eventhandler.h> 84 #include <sys/epoch.h> 85 86 #include <vm/vm.h> 87 #include <vm/vm_extern.h> 88 #include <vm/pmap.h> 89 90 #include <machine/atomic.h> 91 #include <machine/in_cksum.h> 92 93 #include <net/bpf.h> 94 #include <net/ethernet.h> 95 #include <net/if.h> 96 #include <net/if_dl.h> 97 #include <net/if_media.h> 98 #include <net/if_types.h> 99 #include <net/if_var.h> 100 #include <net/rndis.h> 101 #ifdef RSS 102 #include <net/rss_config.h> 103 #endif 104 105 #include <netinet/in_systm.h> 106 #include <netinet/in.h> 107 #include <netinet/ip.h> 108 #include <netinet/ip6.h> 109 #include <netinet/tcp.h> 110 #include <netinet/tcp_lro.h> 111 #include <netinet/udp.h> 112 113 #include <dev/hyperv/include/hyperv.h> 114 #include <dev/hyperv/include/hyperv_busdma.h> 115 #include <dev/hyperv/include/vmbus.h> 116 #include <dev/hyperv/include/vmbus_xact.h> 117 118 #include <dev/hyperv/netvsc/ndis.h> 119 #include <dev/hyperv/netvsc/if_hnreg.h> 120 #include <dev/hyperv/netvsc/if_hnvar.h> 121 #include <dev/hyperv/netvsc/hn_nvs.h> 122 #include <dev/hyperv/netvsc/hn_rndis.h> 123 124 #include "vmbus_if.h" 125 126 #define HN_IFSTART_SUPPORT 127 128 #define HN_RING_CNT_DEF_MAX 8 129 130 #define HN_VFMAP_SIZE_DEF 8 131 132 #define HN_XPNT_VF_ATTWAIT_MIN 2 /* seconds */ 133 134 /* YYY should get it from the underlying channel */ 135 #define HN_TX_DESC_CNT 512 136 137 #define HN_RNDIS_PKT_LEN \ 138 (sizeof(struct rndis_packet_msg) + \ 139 HN_RNDIS_PKTINFO_SIZE(HN_NDIS_HASH_VALUE_SIZE) + \ 140 HN_RNDIS_PKTINFO_SIZE(NDIS_VLAN_INFO_SIZE) + \ 141 HN_RNDIS_PKTINFO_SIZE(NDIS_LSO2_INFO_SIZE) + \ 142 HN_RNDIS_PKTINFO_SIZE(NDIS_TXCSUM_INFO_SIZE)) 143 #define HN_RNDIS_PKT_BOUNDARY PAGE_SIZE 144 #define HN_RNDIS_PKT_ALIGN CACHE_LINE_SIZE 145 146 #define HN_TX_DATA_BOUNDARY PAGE_SIZE 147 #define HN_TX_DATA_MAXSIZE IP_MAXPACKET 148 #define HN_TX_DATA_SEGSIZE PAGE_SIZE 149 /* -1 for RNDIS packet message */ 150 #define HN_TX_DATA_SEGCNT_MAX (HN_GPACNT_MAX - 1) 151 152 #define HN_DIRECT_TX_SIZE_DEF 128 153 154 #define HN_EARLY_TXEOF_THRESH 8 155 156 #define HN_PKTBUF_LEN_DEF (16 * 1024) 157 158 #define HN_LROENT_CNT_DEF 128 159 160 #define HN_LRO_LENLIM_MULTIRX_DEF (12 * ETHERMTU) 161 #define HN_LRO_LENLIM_DEF (25 * ETHERMTU) 162 /* YYY 2*MTU is a bit rough, but should be good enough. */ 163 #define HN_LRO_LENLIM_MIN(ifp) (2 * if_getmtu(ifp)) 164 165 #define HN_LRO_ACKCNT_DEF 1 166 167 #define HN_LOCK_INIT(sc) \ 168 sx_init(&(sc)->hn_lock, device_get_nameunit((sc)->hn_dev)) 169 #define HN_LOCK_DESTROY(sc) sx_destroy(&(sc)->hn_lock) 170 #define HN_LOCK_ASSERT(sc) sx_assert(&(sc)->hn_lock, SA_XLOCKED) 171 #define HN_LOCK(sc) \ 172 do { \ 173 while (sx_try_xlock(&(sc)->hn_lock) == 0) { \ 174 /* Relinquish cpu to avoid deadlock */ \ 175 sched_relinquish(curthread); \ 176 DELAY(1000); \ 177 } \ 178 } while (0) 179 #define HN_UNLOCK(sc) sx_xunlock(&(sc)->hn_lock) 180 181 #define HN_CSUM_IP_MASK (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP) 182 #define HN_CSUM_IP6_MASK (CSUM_IP6_TCP | CSUM_IP6_UDP) 183 #define HN_CSUM_IP_HWASSIST(sc) \ 184 ((sc)->hn_tx_ring[0].hn_csum_assist & HN_CSUM_IP_MASK) 185 #define HN_CSUM_IP6_HWASSIST(sc) \ 186 ((sc)->hn_tx_ring[0].hn_csum_assist & HN_CSUM_IP6_MASK) 187 188 #define HN_PKTSIZE_MIN(align) \ 189 roundup2(ETHER_MIN_LEN + ETHER_VLAN_ENCAP_LEN - ETHER_CRC_LEN + \ 190 HN_RNDIS_PKT_LEN, (align)) 191 #define HN_PKTSIZE(m, align) \ 192 roundup2((m)->m_pkthdr.len + HN_RNDIS_PKT_LEN, (align)) 193 194 #ifdef RSS 195 #define HN_RING_IDX2CPU(sc, idx) rss_getcpu((idx) % rss_getnumbuckets()) 196 #else 197 #define HN_RING_IDX2CPU(sc, idx) (((sc)->hn_cpu + (idx)) % mp_ncpus) 198 #endif 199 200 struct hn_txdesc { 201 #ifndef HN_USE_TXDESC_BUFRING 202 SLIST_ENTRY(hn_txdesc) link; 203 #endif 204 STAILQ_ENTRY(hn_txdesc) agg_link; 205 206 /* Aggregated txdescs, in sending order. */ 207 STAILQ_HEAD(, hn_txdesc) agg_list; 208 209 /* The oldest packet, if transmission aggregation happens. */ 210 struct mbuf *m; 211 struct hn_tx_ring *txr; 212 int refs; 213 uint32_t flags; /* HN_TXD_FLAG_ */ 214 struct hn_nvs_sendctx send_ctx; 215 uint32_t chim_index; 216 int chim_size; 217 218 bus_dmamap_t data_dmap; 219 220 bus_addr_t rndis_pkt_paddr; 221 struct rndis_packet_msg *rndis_pkt; 222 bus_dmamap_t rndis_pkt_dmap; 223 }; 224 225 #define HN_TXD_FLAG_ONLIST 0x0001 226 #define HN_TXD_FLAG_DMAMAP 0x0002 227 #define HN_TXD_FLAG_ONAGG 0x0004 228 229 #define HN_NDIS_PKTINFO_SUBALLOC 0x01 230 #define HN_NDIS_PKTINFO_1ST_FRAG 0x02 231 #define HN_NDIS_PKTINFO_LAST_FRAG 0x04 232 233 struct packet_info_id { 234 uint8_t ver; 235 uint8_t flag; 236 uint16_t pkt_id; 237 }; 238 239 #define NDIS_PKTINFOID_SZ sizeof(struct packet_info_id) 240 241 242 struct hn_rxinfo { 243 const uint32_t *vlan_info; 244 const uint32_t *csum_info; 245 const uint32_t *hash_info; 246 const uint32_t *hash_value; 247 const struct packet_info_id *pktinfo_id; 248 }; 249 250 struct hn_rxvf_setarg { 251 struct hn_rx_ring *rxr; 252 if_t vf_ifp; 253 }; 254 255 #define HN_RXINFO_VLAN 0x0001 256 #define HN_RXINFO_CSUM 0x0002 257 #define HN_RXINFO_HASHINF 0x0004 258 #define HN_RXINFO_HASHVAL 0x0008 259 #define HN_RXINFO_PKTINFO_ID 0x0010 260 #define HN_RXINFO_ALL \ 261 (HN_RXINFO_VLAN | \ 262 HN_RXINFO_CSUM | \ 263 HN_RXINFO_HASHINF | \ 264 HN_RXINFO_HASHVAL | \ 265 HN_RXINFO_PKTINFO_ID) 266 267 static int hn_probe(device_t); 268 static int hn_attach(device_t); 269 static int hn_detach(device_t); 270 static int hn_shutdown(device_t); 271 static void hn_chan_callback(struct vmbus_channel *, 272 void *); 273 274 static void hn_init(void *); 275 static int hn_ioctl(if_t, u_long, caddr_t); 276 #ifdef HN_IFSTART_SUPPORT 277 static void hn_start(if_t); 278 #endif 279 static int hn_transmit(if_t, struct mbuf *); 280 static void hn_xmit_qflush(if_t); 281 static int hn_ifmedia_upd(if_t); 282 static void hn_ifmedia_sts(if_t, 283 struct ifmediareq *); 284 285 static void hn_ifnet_event(void *, if_t, int); 286 static void hn_ifaddr_event(void *, if_t); 287 static void hn_ifnet_attevent(void *, if_t); 288 static void hn_ifnet_detevent(void *, if_t); 289 static void hn_ifnet_lnkevent(void *, if_t, int); 290 291 static bool hn_ismyvf(const struct hn_softc *, 292 const if_t); 293 static void hn_rxvf_change(struct hn_softc *, 294 if_t, bool); 295 static void hn_rxvf_set(struct hn_softc *, if_t); 296 static void hn_rxvf_set_task(void *, int); 297 static void hn_xpnt_vf_input(if_t, struct mbuf *); 298 static int hn_xpnt_vf_iocsetflags(struct hn_softc *); 299 static int hn_xpnt_vf_iocsetcaps(struct hn_softc *, 300 struct ifreq *); 301 static void hn_xpnt_vf_saveifflags(struct hn_softc *); 302 static bool hn_xpnt_vf_isready(struct hn_softc *); 303 static void hn_xpnt_vf_setready(struct hn_softc *); 304 static void hn_xpnt_vf_init_taskfunc(void *, int); 305 static void hn_xpnt_vf_init(struct hn_softc *); 306 static void hn_xpnt_vf_setenable(struct hn_softc *); 307 static void hn_xpnt_vf_setdisable(struct hn_softc *, bool); 308 static void hn_vf_rss_fixup(struct hn_softc *, bool); 309 static void hn_vf_rss_restore(struct hn_softc *); 310 311 static int hn_rndis_rxinfo(const void *, int, 312 struct hn_rxinfo *); 313 static void hn_rndis_rx_data(struct hn_rx_ring *, 314 const void *, int); 315 static void hn_rndis_rx_status(struct hn_softc *, 316 const void *, int); 317 static void hn_rndis_init_fixat(struct hn_softc *, int); 318 319 static void hn_nvs_handle_notify(struct hn_softc *, 320 const struct vmbus_chanpkt_hdr *); 321 static void hn_nvs_handle_comp(struct hn_softc *, 322 struct vmbus_channel *, 323 const struct vmbus_chanpkt_hdr *); 324 static void hn_nvs_handle_rxbuf(struct hn_rx_ring *, 325 struct vmbus_channel *, 326 const struct vmbus_chanpkt_hdr *); 327 static void hn_nvs_ack_rxbuf(struct hn_rx_ring *, 328 struct vmbus_channel *, uint64_t); 329 330 static int hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS); 331 static int hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS); 332 static int hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS); 333 static int hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS); 334 static int hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS); 335 static int hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); 336 static int hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); 337 static int hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS); 338 static int hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS); 339 static int hn_caps_sysctl(SYSCTL_HANDLER_ARGS); 340 static int hn_hwassist_sysctl(SYSCTL_HANDLER_ARGS); 341 static int hn_rxfilter_sysctl(SYSCTL_HANDLER_ARGS); 342 #ifndef RSS 343 static int hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS); 344 static int hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS); 345 #endif 346 static int hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS); 347 static int hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS); 348 static int hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS); 349 static int hn_txagg_size_sysctl(SYSCTL_HANDLER_ARGS); 350 static int hn_txagg_pkts_sysctl(SYSCTL_HANDLER_ARGS); 351 static int hn_txagg_pktmax_sysctl(SYSCTL_HANDLER_ARGS); 352 static int hn_txagg_align_sysctl(SYSCTL_HANDLER_ARGS); 353 static int hn_polling_sysctl(SYSCTL_HANDLER_ARGS); 354 static int hn_vf_sysctl(SYSCTL_HANDLER_ARGS); 355 static int hn_rxvf_sysctl(SYSCTL_HANDLER_ARGS); 356 static int hn_vflist_sysctl(SYSCTL_HANDLER_ARGS); 357 static int hn_vfmap_sysctl(SYSCTL_HANDLER_ARGS); 358 static int hn_xpnt_vf_accbpf_sysctl(SYSCTL_HANDLER_ARGS); 359 static int hn_xpnt_vf_enabled_sysctl(SYSCTL_HANDLER_ARGS); 360 361 static void hn_stop(struct hn_softc *, bool); 362 static void hn_init_locked(struct hn_softc *); 363 static int hn_chan_attach(struct hn_softc *, 364 struct vmbus_channel *); 365 static void hn_chan_detach(struct hn_softc *, 366 struct vmbus_channel *); 367 static int hn_attach_subchans(struct hn_softc *); 368 static void hn_detach_allchans(struct hn_softc *); 369 static void hn_chan_rollup(struct hn_rx_ring *, 370 struct hn_tx_ring *); 371 static void hn_set_ring_inuse(struct hn_softc *, int); 372 static int hn_synth_attach(struct hn_softc *, int); 373 static void hn_synth_detach(struct hn_softc *); 374 static int hn_synth_alloc_subchans(struct hn_softc *, 375 int *); 376 static bool hn_synth_attachable(const struct hn_softc *); 377 static void hn_suspend(struct hn_softc *); 378 static void hn_suspend_data(struct hn_softc *); 379 static void hn_suspend_mgmt(struct hn_softc *); 380 static void hn_resume(struct hn_softc *); 381 static void hn_resume_data(struct hn_softc *); 382 static void hn_resume_mgmt(struct hn_softc *); 383 static void hn_suspend_mgmt_taskfunc(void *, int); 384 static void hn_chan_drain(struct hn_softc *, 385 struct vmbus_channel *); 386 static void hn_disable_rx(struct hn_softc *); 387 static void hn_drain_rxtx(struct hn_softc *, int); 388 static void hn_polling(struct hn_softc *, u_int); 389 static void hn_chan_polling(struct vmbus_channel *, u_int); 390 static void hn_mtu_change_fixup(struct hn_softc *); 391 392 static void hn_update_link_status(struct hn_softc *); 393 static void hn_change_network(struct hn_softc *); 394 static void hn_link_taskfunc(void *, int); 395 static void hn_netchg_init_taskfunc(void *, int); 396 static void hn_netchg_status_taskfunc(void *, int); 397 static void hn_link_status(struct hn_softc *); 398 399 static int hn_create_rx_data(struct hn_softc *, int); 400 static void hn_destroy_rx_data(struct hn_softc *); 401 static int hn_check_iplen(const struct mbuf *, int); 402 static void hn_rxpkt_proto(const struct mbuf *, int *, int *); 403 static int hn_set_rxfilter(struct hn_softc *, uint32_t); 404 static int hn_rxfilter_config(struct hn_softc *); 405 static int hn_rss_reconfig(struct hn_softc *); 406 static void hn_rss_ind_fixup(struct hn_softc *); 407 static void hn_rss_mbuf_hash(struct hn_softc *, uint32_t); 408 static int hn_rxpkt(struct hn_rx_ring *); 409 static uint32_t hn_rss_type_fromndis(uint32_t); 410 static uint32_t hn_rss_type_tondis(uint32_t); 411 412 static int hn_tx_ring_create(struct hn_softc *, int); 413 static void hn_tx_ring_destroy(struct hn_tx_ring *); 414 static int hn_create_tx_data(struct hn_softc *, int); 415 static void hn_fixup_tx_data(struct hn_softc *); 416 static void hn_fixup_rx_data(struct hn_softc *); 417 static void hn_destroy_tx_data(struct hn_softc *); 418 static void hn_txdesc_dmamap_destroy(struct hn_txdesc *); 419 static void hn_txdesc_gc(struct hn_tx_ring *, 420 struct hn_txdesc *); 421 static int hn_encap(if_t, struct hn_tx_ring *, 422 struct hn_txdesc *, struct mbuf **); 423 static int hn_txpkt(if_t, struct hn_tx_ring *, 424 struct hn_txdesc *); 425 static void hn_set_chim_size(struct hn_softc *, int); 426 static void hn_set_tso_maxsize(struct hn_softc *, int, int); 427 static bool hn_tx_ring_pending(struct hn_tx_ring *); 428 static void hn_tx_ring_qflush(struct hn_tx_ring *); 429 static void hn_resume_tx(struct hn_softc *, int); 430 static void hn_set_txagg(struct hn_softc *); 431 static void *hn_try_txagg(if_t, 432 struct hn_tx_ring *, struct hn_txdesc *, 433 int); 434 static int hn_get_txswq_depth(const struct hn_tx_ring *); 435 static void hn_txpkt_done(struct hn_nvs_sendctx *, 436 struct hn_softc *, struct vmbus_channel *, 437 const void *, int); 438 static int hn_txpkt_sglist(struct hn_tx_ring *, 439 struct hn_txdesc *); 440 static int hn_txpkt_chim(struct hn_tx_ring *, 441 struct hn_txdesc *); 442 static int hn_xmit(struct hn_tx_ring *, int); 443 static void hn_xmit_taskfunc(void *, int); 444 static void hn_xmit_txeof(struct hn_tx_ring *); 445 static void hn_xmit_txeof_taskfunc(void *, int); 446 #ifdef HN_IFSTART_SUPPORT 447 static int hn_start_locked(struct hn_tx_ring *, int); 448 static void hn_start_taskfunc(void *, int); 449 static void hn_start_txeof(struct hn_tx_ring *); 450 static void hn_start_txeof_taskfunc(void *, int); 451 #endif 452 453 static int hn_rsc_sysctl(SYSCTL_HANDLER_ARGS); 454 455 SYSCTL_NODE(_hw, OID_AUTO, hn, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 456 "Hyper-V network interface"); 457 458 /* Trust tcp segment verification on host side. */ 459 static int hn_trust_hosttcp = 1; 460 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hosttcp, CTLFLAG_RDTUN, 461 &hn_trust_hosttcp, 0, 462 "Trust tcp segment verification on host side, " 463 "when csum info is missing (global setting)"); 464 465 /* Trust udp datagrams verification on host side. */ 466 static int hn_trust_hostudp = 1; 467 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostudp, CTLFLAG_RDTUN, 468 &hn_trust_hostudp, 0, 469 "Trust udp datagram verification on host side, " 470 "when csum info is missing (global setting)"); 471 472 /* Trust ip packets verification on host side. */ 473 static int hn_trust_hostip = 1; 474 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostip, CTLFLAG_RDTUN, 475 &hn_trust_hostip, 0, 476 "Trust ip packet verification on host side, " 477 "when csum info is missing (global setting)"); 478 479 /* 480 * Offload UDP/IPv4 checksum. 481 */ 482 static int hn_enable_udp4cs = 1; 483 SYSCTL_INT(_hw_hn, OID_AUTO, enable_udp4cs, CTLFLAG_RDTUN, 484 &hn_enable_udp4cs, 0, "Offload UDP/IPv4 checksum"); 485 486 /* 487 * Offload UDP/IPv6 checksum. 488 */ 489 static int hn_enable_udp6cs = 1; 490 SYSCTL_INT(_hw_hn, OID_AUTO, enable_udp6cs, CTLFLAG_RDTUN, 491 &hn_enable_udp6cs, 0, "Offload UDP/IPv6 checksum"); 492 493 /* Stats. */ 494 static counter_u64_t hn_udpcs_fixup; 495 SYSCTL_COUNTER_U64(_hw_hn, OID_AUTO, udpcs_fixup, CTLFLAG_RW, 496 &hn_udpcs_fixup, "# of UDP checksum fixup"); 497 498 /* 499 * See hn_set_hlen(). 500 * 501 * This value is for Azure. For Hyper-V, set this above 502 * 65536 to disable UDP datagram checksum fixup. 503 */ 504 static int hn_udpcs_fixup_mtu = 1420; 505 SYSCTL_INT(_hw_hn, OID_AUTO, udpcs_fixup_mtu, CTLFLAG_RWTUN, 506 &hn_udpcs_fixup_mtu, 0, "UDP checksum fixup MTU threshold"); 507 508 /* Limit TSO burst size */ 509 static int hn_tso_maxlen = IP_MAXPACKET; 510 SYSCTL_INT(_hw_hn, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, 511 &hn_tso_maxlen, 0, "TSO burst limit"); 512 513 /* Limit chimney send size */ 514 static int hn_tx_chimney_size = 0; 515 SYSCTL_INT(_hw_hn, OID_AUTO, tx_chimney_size, CTLFLAG_RDTUN, 516 &hn_tx_chimney_size, 0, "Chimney send packet size limit"); 517 518 /* Limit the size of packet for direct transmission */ 519 static int hn_direct_tx_size = HN_DIRECT_TX_SIZE_DEF; 520 SYSCTL_INT(_hw_hn, OID_AUTO, direct_tx_size, CTLFLAG_RDTUN, 521 &hn_direct_tx_size, 0, "Size of the packet for direct transmission"); 522 523 /* # of LRO entries per RX ring */ 524 #if defined(INET) || defined(INET6) 525 static int hn_lro_entry_count = HN_LROENT_CNT_DEF; 526 SYSCTL_INT(_hw_hn, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN, 527 &hn_lro_entry_count, 0, "LRO entry count"); 528 #endif 529 530 static int hn_tx_taskq_cnt = 1; 531 SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_cnt, CTLFLAG_RDTUN, 532 &hn_tx_taskq_cnt, 0, "# of TX taskqueues"); 533 534 #define HN_TX_TASKQ_M_INDEP 0 535 #define HN_TX_TASKQ_M_GLOBAL 1 536 #define HN_TX_TASKQ_M_EVTTQ 2 537 538 static int hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; 539 SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_mode, CTLFLAG_RDTUN, 540 &hn_tx_taskq_mode, 0, "TX taskqueue modes: " 541 "0 - independent, 1 - share global tx taskqs, 2 - share event taskqs"); 542 543 #ifndef HN_USE_TXDESC_BUFRING 544 static int hn_use_txdesc_bufring = 0; 545 #else 546 static int hn_use_txdesc_bufring = 1; 547 #endif 548 SYSCTL_INT(_hw_hn, OID_AUTO, use_txdesc_bufring, CTLFLAG_RD, 549 &hn_use_txdesc_bufring, 0, "Use buf_ring for TX descriptors"); 550 551 #ifdef HN_IFSTART_SUPPORT 552 /* Use ifnet.if_start instead of ifnet.if_transmit */ 553 static int hn_use_if_start = 0; 554 SYSCTL_INT(_hw_hn, OID_AUTO, use_if_start, CTLFLAG_RDTUN, 555 &hn_use_if_start, 0, "Use if_start TX method"); 556 #endif 557 558 /* # of channels to use */ 559 static int hn_chan_cnt = 0; 560 SYSCTL_INT(_hw_hn, OID_AUTO, chan_cnt, CTLFLAG_RDTUN, 561 &hn_chan_cnt, 0, 562 "# of channels to use; each channel has one RX ring and one TX ring"); 563 564 /* # of transmit rings to use */ 565 static int hn_tx_ring_cnt = 0; 566 SYSCTL_INT(_hw_hn, OID_AUTO, tx_ring_cnt, CTLFLAG_RDTUN, 567 &hn_tx_ring_cnt, 0, "# of TX rings to use"); 568 569 /* Software TX ring deptch */ 570 static int hn_tx_swq_depth = 0; 571 SYSCTL_INT(_hw_hn, OID_AUTO, tx_swq_depth, CTLFLAG_RDTUN, 572 &hn_tx_swq_depth, 0, "Depth of IFQ or BUFRING"); 573 574 /* Enable sorted LRO, and the depth of the per-channel mbuf queue */ 575 static u_int hn_lro_mbufq_depth = 0; 576 SYSCTL_UINT(_hw_hn, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN, 577 &hn_lro_mbufq_depth, 0, "Depth of LRO mbuf queue"); 578 579 /* Packet transmission aggregation size limit */ 580 static int hn_tx_agg_size = -1; 581 SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_size, CTLFLAG_RDTUN, 582 &hn_tx_agg_size, 0, "Packet transmission aggregation size limit"); 583 584 /* Packet transmission aggregation count limit */ 585 static int hn_tx_agg_pkts = -1; 586 SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_pkts, CTLFLAG_RDTUN, 587 &hn_tx_agg_pkts, 0, "Packet transmission aggregation packet limit"); 588 589 /* VF list */ 590 SYSCTL_PROC(_hw_hn, OID_AUTO, vflist, 591 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_NEEDGIANT, 0, 0, 592 hn_vflist_sysctl, "A", 593 "VF list"); 594 595 /* VF mapping */ 596 SYSCTL_PROC(_hw_hn, OID_AUTO, vfmap, 597 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_NEEDGIANT, 0, 0, 598 hn_vfmap_sysctl, "A", 599 "VF mapping"); 600 601 /* Transparent VF */ 602 static int hn_xpnt_vf = 1; 603 SYSCTL_INT(_hw_hn, OID_AUTO, vf_transparent, CTLFLAG_RDTUN, 604 &hn_xpnt_vf, 0, "Transparent VF mod"); 605 606 /* Accurate BPF support for Transparent VF */ 607 static int hn_xpnt_vf_accbpf = 0; 608 SYSCTL_INT(_hw_hn, OID_AUTO, vf_xpnt_accbpf, CTLFLAG_RDTUN, 609 &hn_xpnt_vf_accbpf, 0, "Accurate BPF for transparent VF"); 610 611 /* Extra wait for transparent VF attach routing; unit seconds. */ 612 static int hn_xpnt_vf_attwait = HN_XPNT_VF_ATTWAIT_MIN; 613 SYSCTL_INT(_hw_hn, OID_AUTO, vf_xpnt_attwait, CTLFLAG_RWTUN, 614 &hn_xpnt_vf_attwait, 0, 615 "Extra wait for transparent VF attach routing; unit: seconds"); 616 617 static u_int hn_cpu_index; /* next CPU for channel */ 618 static struct taskqueue **hn_tx_taskque;/* shared TX taskqueues */ 619 620 static struct rmlock hn_vfmap_lock; 621 static int hn_vfmap_size; 622 static if_t *hn_vfmap; 623 624 #ifndef RSS 625 static const uint8_t 626 hn_rss_key_default[NDIS_HASH_KEYSIZE_TOEPLITZ] = { 627 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 628 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 629 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 630 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 631 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa 632 }; 633 #endif /* !RSS */ 634 635 static const struct hyperv_guid hn_guid = { 636 .hv_guid = { 637 0x63, 0x51, 0x61, 0xf8, 0x3e, 0xdf, 0xc5, 0x46, 638 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e } 639 }; 640 641 static device_method_t hn_methods[] = { 642 /* Device interface */ 643 DEVMETHOD(device_probe, hn_probe), 644 DEVMETHOD(device_attach, hn_attach), 645 DEVMETHOD(device_detach, hn_detach), 646 DEVMETHOD(device_shutdown, hn_shutdown), 647 DEVMETHOD_END 648 }; 649 650 static driver_t hn_driver = { 651 "hn", 652 hn_methods, 653 sizeof(struct hn_softc) 654 }; 655 656 DRIVER_MODULE(hn, vmbus, hn_driver, 0, 0); 657 MODULE_VERSION(hn, 1); 658 MODULE_DEPEND(hn, vmbus, 1, 1, 1); 659 660 static void 661 hn_set_lro_lenlim(struct hn_softc *sc, int lenlim) 662 { 663 int i; 664 665 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 666 sc->hn_rx_ring[i].hn_lro.lro_length_lim = lenlim; 667 } 668 669 static int 670 hn_txpkt_sglist(struct hn_tx_ring *txr, struct hn_txdesc *txd) 671 { 672 673 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID && 674 txd->chim_size == 0, ("invalid rndis sglist txd")); 675 return (hn_nvs_send_rndis_sglist(txr->hn_chan, HN_NVS_RNDIS_MTYPE_DATA, 676 &txd->send_ctx, txr->hn_gpa, txr->hn_gpa_cnt)); 677 } 678 679 static int 680 hn_txpkt_chim(struct hn_tx_ring *txr, struct hn_txdesc *txd) 681 { 682 struct hn_nvs_rndis rndis; 683 684 KASSERT(txd->chim_index != HN_NVS_CHIM_IDX_INVALID && 685 txd->chim_size > 0, ("invalid rndis chim txd")); 686 687 rndis.nvs_type = HN_NVS_TYPE_RNDIS; 688 rndis.nvs_rndis_mtype = HN_NVS_RNDIS_MTYPE_DATA; 689 rndis.nvs_chim_idx = txd->chim_index; 690 rndis.nvs_chim_sz = txd->chim_size; 691 692 return (hn_nvs_send(txr->hn_chan, VMBUS_CHANPKT_FLAG_RC, 693 &rndis, sizeof(rndis), &txd->send_ctx)); 694 } 695 696 static __inline uint32_t 697 hn_chim_alloc(struct hn_softc *sc) 698 { 699 int i, bmap_cnt = sc->hn_chim_bmap_cnt; 700 u_long *bmap = sc->hn_chim_bmap; 701 uint32_t ret = HN_NVS_CHIM_IDX_INVALID; 702 703 for (i = 0; i < bmap_cnt; ++i) { 704 int idx; 705 706 idx = ffsl(~bmap[i]); 707 if (idx == 0) 708 continue; 709 710 --idx; /* ffsl is 1-based */ 711 KASSERT(i * LONG_BIT + idx < sc->hn_chim_cnt, 712 ("invalid i %d and idx %d", i, idx)); 713 714 if (atomic_testandset_long(&bmap[i], idx)) 715 continue; 716 717 ret = i * LONG_BIT + idx; 718 break; 719 } 720 return (ret); 721 } 722 723 static __inline void 724 hn_chim_free(struct hn_softc *sc, uint32_t chim_idx) 725 { 726 u_long mask; 727 uint32_t idx; 728 729 idx = chim_idx / LONG_BIT; 730 KASSERT(idx < sc->hn_chim_bmap_cnt, 731 ("invalid chimney index 0x%x", chim_idx)); 732 733 mask = 1UL << (chim_idx % LONG_BIT); 734 KASSERT(sc->hn_chim_bmap[idx] & mask, 735 ("index bitmap 0x%lx, chimney index %u, " 736 "bitmap idx %d, bitmask 0x%lx", 737 sc->hn_chim_bmap[idx], chim_idx, idx, mask)); 738 739 atomic_clear_long(&sc->hn_chim_bmap[idx], mask); 740 } 741 742 #if defined(INET6) || defined(INET) 743 744 #define PULLUP_HDR(m, len) \ 745 do { \ 746 if (__predict_false((m)->m_len < (len))) { \ 747 (m) = m_pullup((m), (len)); \ 748 if ((m) == NULL) \ 749 return (NULL); \ 750 } \ 751 } while (0) 752 753 /* 754 * NOTE: If this function failed, the m_head would be freed. 755 */ 756 static __inline struct mbuf * 757 hn_tso_fixup(struct mbuf *m_head) 758 { 759 struct ether_vlan_header *evl; 760 struct tcphdr *th; 761 int ehlen; 762 763 KASSERT(M_WRITABLE(m_head), ("TSO mbuf not writable")); 764 765 PULLUP_HDR(m_head, sizeof(*evl)); 766 evl = mtod(m_head, struct ether_vlan_header *); 767 if (evl->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) 768 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 769 else 770 ehlen = ETHER_HDR_LEN; 771 m_head->m_pkthdr.l2hlen = ehlen; 772 773 #ifdef INET 774 if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { 775 struct ip *ip; 776 int iphlen; 777 778 PULLUP_HDR(m_head, ehlen + sizeof(*ip)); 779 ip = mtodo(m_head, ehlen); 780 iphlen = ip->ip_hl << 2; 781 m_head->m_pkthdr.l3hlen = iphlen; 782 783 PULLUP_HDR(m_head, ehlen + iphlen + sizeof(*th)); 784 th = mtodo(m_head, ehlen + iphlen); 785 786 ip->ip_len = 0; 787 ip->ip_sum = 0; 788 th->th_sum = in_pseudo(ip->ip_src.s_addr, 789 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 790 } 791 #endif 792 #if defined(INET6) && defined(INET) 793 else 794 #endif 795 #ifdef INET6 796 { 797 struct ip6_hdr *ip6; 798 799 PULLUP_HDR(m_head, ehlen + sizeof(*ip6)); 800 ip6 = mtodo(m_head, ehlen); 801 if (ip6->ip6_nxt != IPPROTO_TCP) { 802 m_freem(m_head); 803 return (NULL); 804 } 805 m_head->m_pkthdr.l3hlen = sizeof(*ip6); 806 807 PULLUP_HDR(m_head, ehlen + sizeof(*ip6) + sizeof(*th)); 808 th = mtodo(m_head, ehlen + sizeof(*ip6)); 809 810 ip6->ip6_plen = 0; 811 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 812 } 813 #endif 814 return (m_head); 815 } 816 817 /* 818 * NOTE: If this function failed, the m_head would be freed. 819 */ 820 static __inline struct mbuf * 821 hn_set_hlen(struct mbuf *m_head) 822 { 823 const struct ether_vlan_header *evl; 824 int ehlen; 825 826 PULLUP_HDR(m_head, sizeof(*evl)); 827 evl = mtod(m_head, const struct ether_vlan_header *); 828 if (evl->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) 829 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 830 else 831 ehlen = ETHER_HDR_LEN; 832 m_head->m_pkthdr.l2hlen = ehlen; 833 834 #ifdef INET 835 if (m_head->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP_UDP)) { 836 const struct ip *ip; 837 int iphlen; 838 839 PULLUP_HDR(m_head, ehlen + sizeof(*ip)); 840 ip = mtodo(m_head, ehlen); 841 iphlen = ip->ip_hl << 2; 842 m_head->m_pkthdr.l3hlen = iphlen; 843 844 /* 845 * UDP checksum offload does not work in Azure, if the 846 * following conditions meet: 847 * - sizeof(IP hdr + UDP hdr + payload) > 1420. 848 * - IP_DF is not set in the IP hdr. 849 * 850 * Fallback to software checksum for these UDP datagrams. 851 */ 852 if ((m_head->m_pkthdr.csum_flags & CSUM_IP_UDP) && 853 m_head->m_pkthdr.len > hn_udpcs_fixup_mtu + ehlen && 854 (ntohs(ip->ip_off) & IP_DF) == 0) { 855 uint16_t off = ehlen + iphlen; 856 857 counter_u64_add(hn_udpcs_fixup, 1); 858 PULLUP_HDR(m_head, off + sizeof(struct udphdr)); 859 *(uint16_t *)(m_head->m_data + off + 860 m_head->m_pkthdr.csum_data) = in_cksum_skip( 861 m_head, m_head->m_pkthdr.len, off); 862 m_head->m_pkthdr.csum_flags &= ~CSUM_IP_UDP; 863 } 864 } 865 #endif 866 #if defined(INET6) && defined(INET) 867 else 868 #endif 869 #ifdef INET6 870 { 871 const struct ip6_hdr *ip6; 872 873 PULLUP_HDR(m_head, ehlen + sizeof(*ip6)); 874 ip6 = mtodo(m_head, ehlen); 875 if (ip6->ip6_nxt != IPPROTO_TCP && 876 ip6->ip6_nxt != IPPROTO_UDP) { 877 m_freem(m_head); 878 return (NULL); 879 } 880 m_head->m_pkthdr.l3hlen = sizeof(*ip6); 881 } 882 #endif 883 return (m_head); 884 } 885 886 /* 887 * NOTE: If this function failed, the m_head would be freed. 888 */ 889 static __inline struct mbuf * 890 hn_check_tcpsyn(struct mbuf *m_head, int *tcpsyn) 891 { 892 const struct tcphdr *th; 893 int ehlen, iphlen; 894 895 *tcpsyn = 0; 896 ehlen = m_head->m_pkthdr.l2hlen; 897 iphlen = m_head->m_pkthdr.l3hlen; 898 899 PULLUP_HDR(m_head, ehlen + iphlen + sizeof(*th)); 900 th = mtodo(m_head, ehlen + iphlen); 901 if (th->th_flags & TH_SYN) 902 *tcpsyn = 1; 903 return (m_head); 904 } 905 906 #undef PULLUP_HDR 907 908 #endif /* INET6 || INET */ 909 910 static int 911 hn_set_rxfilter(struct hn_softc *sc, uint32_t filter) 912 { 913 int error = 0; 914 915 HN_LOCK_ASSERT(sc); 916 917 if (sc->hn_rx_filter != filter) { 918 error = hn_rndis_set_rxfilter(sc, filter); 919 if (!error) 920 sc->hn_rx_filter = filter; 921 } 922 return (error); 923 } 924 925 static int 926 hn_rxfilter_config(struct hn_softc *sc) 927 { 928 if_t ifp = sc->hn_ifp; 929 uint32_t filter; 930 931 HN_LOCK_ASSERT(sc); 932 933 /* 934 * If the non-transparent mode VF is activated, we don't know how 935 * its RX filter is configured, so stick the synthetic device in 936 * the promiscous mode. 937 */ 938 if ((if_getflags(ifp) & IFF_PROMISC) || (sc->hn_flags & HN_FLAG_RXVF)) { 939 filter = NDIS_PACKET_TYPE_PROMISCUOUS; 940 } else { 941 filter = NDIS_PACKET_TYPE_DIRECTED; 942 if (if_getflags(ifp) & IFF_BROADCAST) 943 filter |= NDIS_PACKET_TYPE_BROADCAST; 944 /* TODO: support multicast list */ 945 if ((if_getflags(ifp) & IFF_ALLMULTI) || 946 !if_maddr_empty(ifp)) 947 filter |= NDIS_PACKET_TYPE_ALL_MULTICAST; 948 } 949 return (hn_set_rxfilter(sc, filter)); 950 } 951 952 static void 953 hn_set_txagg(struct hn_softc *sc) 954 { 955 uint32_t size, pkts; 956 int i; 957 958 /* 959 * Setup aggregation size. 960 */ 961 if (sc->hn_agg_size < 0) 962 size = UINT32_MAX; 963 else 964 size = sc->hn_agg_size; 965 966 if (sc->hn_rndis_agg_size < size) 967 size = sc->hn_rndis_agg_size; 968 969 /* NOTE: We only aggregate packets using chimney sending buffers. */ 970 if (size > (uint32_t)sc->hn_chim_szmax) 971 size = sc->hn_chim_szmax; 972 973 if (size <= 2 * HN_PKTSIZE_MIN(sc->hn_rndis_agg_align)) { 974 /* Disable */ 975 size = 0; 976 pkts = 0; 977 goto done; 978 } 979 980 /* NOTE: Type of the per TX ring setting is 'int'. */ 981 if (size > INT_MAX) 982 size = INT_MAX; 983 984 /* 985 * Setup aggregation packet count. 986 */ 987 if (sc->hn_agg_pkts < 0) 988 pkts = UINT32_MAX; 989 else 990 pkts = sc->hn_agg_pkts; 991 992 if (sc->hn_rndis_agg_pkts < pkts) 993 pkts = sc->hn_rndis_agg_pkts; 994 995 if (pkts <= 1) { 996 /* Disable */ 997 size = 0; 998 pkts = 0; 999 goto done; 1000 } 1001 1002 /* NOTE: Type of the per TX ring setting is 'short'. */ 1003 if (pkts > SHRT_MAX) 1004 pkts = SHRT_MAX; 1005 1006 done: 1007 /* NOTE: Type of the per TX ring setting is 'short'. */ 1008 if (sc->hn_rndis_agg_align > SHRT_MAX) { 1009 /* Disable */ 1010 size = 0; 1011 pkts = 0; 1012 } 1013 1014 if (bootverbose) { 1015 if_printf(sc->hn_ifp, "TX agg size %u, pkts %u, align %u\n", 1016 size, pkts, sc->hn_rndis_agg_align); 1017 } 1018 1019 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 1020 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 1021 1022 mtx_lock(&txr->hn_tx_lock); 1023 txr->hn_agg_szmax = size; 1024 txr->hn_agg_pktmax = pkts; 1025 txr->hn_agg_align = sc->hn_rndis_agg_align; 1026 mtx_unlock(&txr->hn_tx_lock); 1027 } 1028 } 1029 1030 static int 1031 hn_get_txswq_depth(const struct hn_tx_ring *txr) 1032 { 1033 1034 KASSERT(txr->hn_txdesc_cnt > 0, ("tx ring is not setup yet")); 1035 if (hn_tx_swq_depth < txr->hn_txdesc_cnt) 1036 return txr->hn_txdesc_cnt; 1037 return hn_tx_swq_depth; 1038 } 1039 1040 static int 1041 hn_rss_reconfig(struct hn_softc *sc) 1042 { 1043 int error; 1044 1045 HN_LOCK_ASSERT(sc); 1046 1047 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 1048 return (ENXIO); 1049 1050 /* 1051 * Disable RSS first. 1052 * 1053 * NOTE: 1054 * Direct reconfiguration by setting the UNCHG flags does 1055 * _not_ work properly. 1056 */ 1057 if (bootverbose) 1058 if_printf(sc->hn_ifp, "disable RSS\n"); 1059 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_DISABLE); 1060 if (error) { 1061 if_printf(sc->hn_ifp, "RSS disable failed\n"); 1062 return (error); 1063 } 1064 1065 /* 1066 * Reenable the RSS w/ the updated RSS key or indirect 1067 * table. 1068 */ 1069 if (bootverbose) 1070 if_printf(sc->hn_ifp, "reconfig RSS\n"); 1071 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE); 1072 if (error) { 1073 if_printf(sc->hn_ifp, "RSS reconfig failed\n"); 1074 return (error); 1075 } 1076 return (0); 1077 } 1078 1079 static void 1080 hn_rss_ind_fixup(struct hn_softc *sc) 1081 { 1082 struct ndis_rssprm_toeplitz *rss = &sc->hn_rss; 1083 int i, nchan; 1084 1085 nchan = sc->hn_rx_ring_inuse; 1086 KASSERT(nchan > 1, ("invalid # of channels %d", nchan)); 1087 1088 /* 1089 * Check indirect table to make sure that all channels in it 1090 * can be used. 1091 */ 1092 for (i = 0; i < NDIS_HASH_INDCNT; ++i) { 1093 if (rss->rss_ind[i] >= nchan) { 1094 if_printf(sc->hn_ifp, 1095 "RSS indirect table %d fixup: %u -> %d\n", 1096 i, rss->rss_ind[i], nchan - 1); 1097 rss->rss_ind[i] = nchan - 1; 1098 } 1099 } 1100 } 1101 1102 static int 1103 hn_ifmedia_upd(if_t ifp __unused) 1104 { 1105 1106 /* Ignore since autoselect is the only defined and valid media */ 1107 return (0); 1108 } 1109 1110 static void 1111 hn_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) 1112 { 1113 struct hn_softc *sc = if_getsoftc(ifp); 1114 1115 ifmr->ifm_status = IFM_AVALID; 1116 ifmr->ifm_active = IFM_ETHER; 1117 1118 if ((sc->hn_link_flags & HN_LINK_FLAG_LINKUP) == 0) { 1119 ifmr->ifm_active |= IFM_NONE; 1120 return; 1121 } 1122 ifmr->ifm_status |= IFM_ACTIVE; 1123 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1124 } 1125 1126 static void 1127 hn_rxvf_set_task(void *xarg, int pending __unused) 1128 { 1129 struct hn_rxvf_setarg *arg = xarg; 1130 1131 arg->rxr->hn_rxvf_ifp = arg->vf_ifp; 1132 } 1133 1134 static void 1135 hn_rxvf_set(struct hn_softc *sc, if_t vf_ifp) 1136 { 1137 struct hn_rx_ring *rxr; 1138 struct hn_rxvf_setarg arg; 1139 struct task task; 1140 int i; 1141 1142 HN_LOCK_ASSERT(sc); 1143 1144 TASK_INIT(&task, 0, hn_rxvf_set_task, &arg); 1145 1146 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 1147 rxr = &sc->hn_rx_ring[i]; 1148 1149 if (i < sc->hn_rx_ring_inuse) { 1150 arg.rxr = rxr; 1151 arg.vf_ifp = vf_ifp; 1152 vmbus_chan_run_task(rxr->hn_chan, &task); 1153 } else { 1154 rxr->hn_rxvf_ifp = vf_ifp; 1155 } 1156 } 1157 } 1158 1159 static bool 1160 hn_ismyvf(const struct hn_softc *sc, const if_t ifp) 1161 { 1162 if_t hn_ifp; 1163 1164 hn_ifp = sc->hn_ifp; 1165 1166 if (ifp == hn_ifp) 1167 return (false); 1168 1169 if (if_getalloctype(ifp) != IFT_ETHER) 1170 return (false); 1171 1172 /* Ignore lagg/vlan interfaces */ 1173 if (strcmp(if_getdname(ifp), "lagg") == 0 || 1174 strcmp(if_getdname(ifp), "vlan") == 0) 1175 return (false); 1176 1177 /* 1178 * During detach events if_getifaddr(ifp) might be NULL. 1179 * Make sure the bcmp() below doesn't panic on that: 1180 */ 1181 if (if_getifaddr(ifp) == NULL || if_getifaddr(hn_ifp) == NULL) 1182 return (false); 1183 1184 if (bcmp(if_getlladdr(ifp), if_getlladdr(hn_ifp), ETHER_ADDR_LEN) != 0) 1185 return (false); 1186 1187 return (true); 1188 } 1189 1190 static void 1191 hn_rxvf_change(struct hn_softc *sc, if_t ifp, bool rxvf) 1192 { 1193 if_t hn_ifp; 1194 1195 HN_LOCK(sc); 1196 1197 if (!(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 1198 goto out; 1199 1200 if (!hn_ismyvf(sc, ifp)) 1201 goto out; 1202 hn_ifp = sc->hn_ifp; 1203 1204 if (rxvf) { 1205 if (sc->hn_flags & HN_FLAG_RXVF) 1206 goto out; 1207 1208 sc->hn_flags |= HN_FLAG_RXVF; 1209 hn_rxfilter_config(sc); 1210 } else { 1211 if (!(sc->hn_flags & HN_FLAG_RXVF)) 1212 goto out; 1213 1214 sc->hn_flags &= ~HN_FLAG_RXVF; 1215 if (if_getdrvflags(hn_ifp) & IFF_DRV_RUNNING) 1216 hn_rxfilter_config(sc); 1217 else 1218 hn_set_rxfilter(sc, NDIS_PACKET_TYPE_NONE); 1219 } 1220 1221 hn_nvs_set_datapath(sc, 1222 rxvf ? HN_NVS_DATAPATH_VF : HN_NVS_DATAPATH_SYNTH); 1223 1224 hn_rxvf_set(sc, rxvf ? ifp : NULL); 1225 1226 if (rxvf) { 1227 hn_vf_rss_fixup(sc, true); 1228 hn_suspend_mgmt(sc); 1229 sc->hn_link_flags &= 1230 ~(HN_LINK_FLAG_LINKUP | HN_LINK_FLAG_NETCHG); 1231 if_link_state_change(hn_ifp, LINK_STATE_DOWN); 1232 } else { 1233 hn_vf_rss_restore(sc); 1234 hn_resume_mgmt(sc); 1235 } 1236 1237 devctl_notify("HYPERV_NIC_VF", if_name(hn_ifp), 1238 rxvf ? "VF_UP" : "VF_DOWN", NULL); 1239 1240 if (bootverbose) { 1241 if_printf(hn_ifp, "datapath is switched %s %s\n", 1242 rxvf ? "to" : "from", if_name(ifp)); 1243 } 1244 out: 1245 HN_UNLOCK(sc); 1246 } 1247 1248 static void 1249 hn_ifnet_event(void *arg, if_t ifp, int event) 1250 { 1251 1252 if (event != IFNET_EVENT_UP && event != IFNET_EVENT_DOWN) 1253 return; 1254 hn_rxvf_change(arg, ifp, event == IFNET_EVENT_UP); 1255 } 1256 1257 static void 1258 hn_ifaddr_event(void *arg, if_t ifp) 1259 { 1260 1261 hn_rxvf_change(arg, ifp, if_getflags(ifp) & IFF_UP); 1262 } 1263 1264 static int 1265 hn_xpnt_vf_iocsetcaps(struct hn_softc *sc, struct ifreq *ifr __unused) 1266 { 1267 if_t ifp, vf_ifp; 1268 1269 HN_LOCK_ASSERT(sc); 1270 ifp = sc->hn_ifp; 1271 vf_ifp = sc->hn_vf_ifp; 1272 1273 /* 1274 * Just sync up with VF's enabled capabilities. 1275 */ 1276 if_setcapenable(ifp, if_getcapenable(vf_ifp)); 1277 if_sethwassist(ifp, if_gethwassist(vf_ifp)); 1278 1279 return (0); 1280 } 1281 1282 static int 1283 hn_xpnt_vf_iocsetflags(struct hn_softc *sc) 1284 { 1285 if_t vf_ifp; 1286 struct ifreq ifr; 1287 1288 HN_LOCK_ASSERT(sc); 1289 vf_ifp = sc->hn_vf_ifp; 1290 1291 memset(&ifr, 0, sizeof(ifr)); 1292 strlcpy(ifr.ifr_name, if_name(vf_ifp), sizeof(ifr.ifr_name)); 1293 ifr.ifr_flags = if_getflags(vf_ifp) & 0xffff; 1294 ifr.ifr_flagshigh = if_getflags(vf_ifp) >> 16; 1295 return (ifhwioctl(SIOCSIFFLAGS, vf_ifp, (caddr_t)&ifr, curthread)); 1296 } 1297 1298 static void 1299 hn_xpnt_vf_saveifflags(struct hn_softc *sc) 1300 { 1301 if_t ifp = sc->hn_ifp; 1302 int allmulti = 0; 1303 1304 HN_LOCK_ASSERT(sc); 1305 1306 /* XXX vlan(4) style mcast addr maintenance */ 1307 if (!if_maddr_empty(ifp)) 1308 allmulti = IFF_ALLMULTI; 1309 1310 /* Always set the VF's if_flags */ 1311 if_setflags(sc->hn_vf_ifp, if_getflags(ifp) | allmulti); 1312 } 1313 1314 static void 1315 hn_xpnt_vf_input(if_t vf_ifp, struct mbuf *m) 1316 { 1317 struct rm_priotracker pt; 1318 if_t hn_ifp = NULL; 1319 struct mbuf *mn; 1320 1321 /* 1322 * XXX racy, if hn(4) ever detached. 1323 */ 1324 rm_rlock(&hn_vfmap_lock, &pt); 1325 if (if_getindex(vf_ifp) < hn_vfmap_size) 1326 hn_ifp = hn_vfmap[if_getindex(vf_ifp)]; 1327 rm_runlock(&hn_vfmap_lock, &pt); 1328 1329 if (hn_ifp != NULL) { 1330 for (mn = m; mn != NULL; mn = mn->m_nextpkt) { 1331 /* 1332 * Allow tapping on the VF. 1333 */ 1334 ETHER_BPF_MTAP(vf_ifp, mn); 1335 1336 /* 1337 * Update VF stats. 1338 */ 1339 if ((if_getcapenable(vf_ifp) & IFCAP_HWSTATS) == 0) { 1340 if_inc_counter(vf_ifp, IFCOUNTER_IBYTES, 1341 mn->m_pkthdr.len); 1342 } 1343 /* 1344 * XXX IFCOUNTER_IMCAST 1345 * This stat updating is kinda invasive, since it 1346 * requires two checks on the mbuf: the length check 1347 * and the ethernet header check. As of this write, 1348 * all multicast packets go directly to hn(4), which 1349 * makes imcast stat updating in the VF a try in vian. 1350 */ 1351 1352 /* 1353 * Fix up rcvif and increase hn(4)'s ipackets. 1354 */ 1355 mn->m_pkthdr.rcvif = hn_ifp; 1356 if_inc_counter(hn_ifp, IFCOUNTER_IPACKETS, 1); 1357 } 1358 /* 1359 * Go through hn(4)'s if_input. 1360 */ 1361 if_input(hn_ifp, m); 1362 } else { 1363 /* 1364 * In the middle of the transition; free this 1365 * mbuf chain. 1366 */ 1367 while (m != NULL) { 1368 mn = m->m_nextpkt; 1369 m->m_nextpkt = NULL; 1370 m_freem(m); 1371 m = mn; 1372 } 1373 } 1374 } 1375 1376 static void 1377 hn_mtu_change_fixup(struct hn_softc *sc) 1378 { 1379 if_t ifp; 1380 1381 HN_LOCK_ASSERT(sc); 1382 ifp = sc->hn_ifp; 1383 1384 hn_set_tso_maxsize(sc, hn_tso_maxlen, if_getmtu(ifp)); 1385 if (sc->hn_rx_ring[0].hn_lro.lro_length_lim < HN_LRO_LENLIM_MIN(ifp)) 1386 hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MIN(ifp)); 1387 } 1388 1389 static uint32_t 1390 hn_rss_type_fromndis(uint32_t rss_hash) 1391 { 1392 uint32_t types = 0; 1393 1394 if (rss_hash & NDIS_HASH_IPV4) 1395 types |= RSS_TYPE_IPV4; 1396 if (rss_hash & NDIS_HASH_TCP_IPV4) 1397 types |= RSS_TYPE_TCP_IPV4; 1398 if (rss_hash & NDIS_HASH_IPV6) 1399 types |= RSS_TYPE_IPV6; 1400 if (rss_hash & NDIS_HASH_IPV6_EX) 1401 types |= RSS_TYPE_IPV6_EX; 1402 if (rss_hash & NDIS_HASH_TCP_IPV6) 1403 types |= RSS_TYPE_TCP_IPV6; 1404 if (rss_hash & NDIS_HASH_TCP_IPV6_EX) 1405 types |= RSS_TYPE_TCP_IPV6_EX; 1406 if (rss_hash & NDIS_HASH_UDP_IPV4_X) 1407 types |= RSS_TYPE_UDP_IPV4; 1408 return (types); 1409 } 1410 1411 static uint32_t 1412 hn_rss_type_tondis(uint32_t types) 1413 { 1414 uint32_t rss_hash = 0; 1415 1416 KASSERT((types & (RSS_TYPE_UDP_IPV6 | RSS_TYPE_UDP_IPV6_EX)) == 0, 1417 ("UDP6 and UDP6EX are not supported")); 1418 1419 if (types & RSS_TYPE_IPV4) 1420 rss_hash |= NDIS_HASH_IPV4; 1421 if (types & RSS_TYPE_TCP_IPV4) 1422 rss_hash |= NDIS_HASH_TCP_IPV4; 1423 if (types & RSS_TYPE_IPV6) 1424 rss_hash |= NDIS_HASH_IPV6; 1425 if (types & RSS_TYPE_IPV6_EX) 1426 rss_hash |= NDIS_HASH_IPV6_EX; 1427 if (types & RSS_TYPE_TCP_IPV6) 1428 rss_hash |= NDIS_HASH_TCP_IPV6; 1429 if (types & RSS_TYPE_TCP_IPV6_EX) 1430 rss_hash |= NDIS_HASH_TCP_IPV6_EX; 1431 if (types & RSS_TYPE_UDP_IPV4) 1432 rss_hash |= NDIS_HASH_UDP_IPV4_X; 1433 return (rss_hash); 1434 } 1435 1436 static void 1437 hn_rss_mbuf_hash(struct hn_softc *sc, uint32_t mbuf_hash) 1438 { 1439 int i; 1440 1441 HN_LOCK_ASSERT(sc); 1442 1443 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1444 sc->hn_rx_ring[i].hn_mbuf_hash = mbuf_hash; 1445 } 1446 1447 static void 1448 hn_vf_rss_fixup(struct hn_softc *sc, bool reconf) 1449 { 1450 if_t ifp, vf_ifp; 1451 struct ifrsshash ifrh; 1452 struct ifrsskey ifrk; 1453 int error; 1454 uint32_t my_types, diff_types, mbuf_types = 0; 1455 1456 HN_LOCK_ASSERT(sc); 1457 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 1458 ("%s: synthetic parts are not attached", if_name(sc->hn_ifp))); 1459 1460 if (sc->hn_rx_ring_inuse == 1) { 1461 /* No RSS on synthetic parts; done. */ 1462 return; 1463 } 1464 if ((sc->hn_rss_hcap & NDIS_HASH_FUNCTION_TOEPLITZ) == 0) { 1465 /* Synthetic parts do not support Toeplitz; done. */ 1466 return; 1467 } 1468 1469 ifp = sc->hn_ifp; 1470 vf_ifp = sc->hn_vf_ifp; 1471 1472 /* 1473 * Extract VF's RSS key. Only 40 bytes key for Toeplitz is 1474 * supported. 1475 */ 1476 memset(&ifrk, 0, sizeof(ifrk)); 1477 strlcpy(ifrk.ifrk_name, if_name(vf_ifp), sizeof(ifrk.ifrk_name)); 1478 error = ifhwioctl(SIOCGIFRSSKEY, vf_ifp, (caddr_t)&ifrk, curthread); 1479 if (error) { 1480 if_printf(ifp, "%s SIOCGIFRSSKEY failed: %d\n", 1481 if_name(vf_ifp), error); 1482 goto done; 1483 } 1484 if (ifrk.ifrk_func != RSS_FUNC_TOEPLITZ) { 1485 if_printf(ifp, "%s RSS function %u is not Toeplitz\n", 1486 if_name(vf_ifp), ifrk.ifrk_func); 1487 goto done; 1488 } 1489 if (ifrk.ifrk_keylen != NDIS_HASH_KEYSIZE_TOEPLITZ) { 1490 if_printf(ifp, "%s invalid RSS Toeplitz key length %d\n", 1491 if_name(vf_ifp), ifrk.ifrk_keylen); 1492 goto done; 1493 } 1494 1495 /* 1496 * Extract VF's RSS hash. Only Toeplitz is supported. 1497 */ 1498 memset(&ifrh, 0, sizeof(ifrh)); 1499 strlcpy(ifrh.ifrh_name, if_name(vf_ifp), sizeof(ifrh.ifrh_name)); 1500 error = ifhwioctl(SIOCGIFRSSHASH, vf_ifp, (caddr_t)&ifrh, curthread); 1501 if (error) { 1502 if_printf(ifp, "%s SIOCGRSSHASH failed: %d\n", 1503 if_name(vf_ifp), error); 1504 goto done; 1505 } 1506 if (ifrh.ifrh_func != RSS_FUNC_TOEPLITZ) { 1507 if_printf(ifp, "%s RSS function %u is not Toeplitz\n", 1508 if_name(vf_ifp), ifrh.ifrh_func); 1509 goto done; 1510 } 1511 1512 my_types = hn_rss_type_fromndis(sc->hn_rss_hcap); 1513 if ((ifrh.ifrh_types & my_types) == 0) { 1514 /* This disables RSS; ignore it then */ 1515 if_printf(ifp, "%s intersection of RSS types failed. " 1516 "VF %#x, mine %#x\n", if_name(vf_ifp), 1517 ifrh.ifrh_types, my_types); 1518 goto done; 1519 } 1520 1521 diff_types = my_types ^ ifrh.ifrh_types; 1522 my_types &= ifrh.ifrh_types; 1523 mbuf_types = my_types; 1524 1525 /* 1526 * Detect RSS hash value/type confliction. 1527 * 1528 * NOTE: 1529 * We don't disable the hash type, but stop delivery the hash 1530 * value/type through mbufs on RX path. 1531 * 1532 * XXX If HN_CAP_UDPHASH is set in hn_caps, then UDP 4-tuple 1533 * hash is delivered with type of TCP_IPV4. This means if 1534 * UDP_IPV4 is enabled, then TCP_IPV4 should be forced, at 1535 * least to hn_mbuf_hash. However, given that _all_ of the 1536 * NICs implement TCP_IPV4, this will _not_ impose any issues 1537 * here. 1538 */ 1539 if ((my_types & RSS_TYPE_IPV4) && 1540 (diff_types & ifrh.ifrh_types & 1541 (RSS_TYPE_TCP_IPV4 | RSS_TYPE_UDP_IPV4))) { 1542 /* Conflict; disable IPV4 hash type/value delivery. */ 1543 if_printf(ifp, "disable IPV4 mbuf hash delivery\n"); 1544 mbuf_types &= ~RSS_TYPE_IPV4; 1545 } 1546 if ((my_types & RSS_TYPE_IPV6) && 1547 (diff_types & ifrh.ifrh_types & 1548 (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 | 1549 RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX | 1550 RSS_TYPE_IPV6_EX))) { 1551 /* Conflict; disable IPV6 hash type/value delivery. */ 1552 if_printf(ifp, "disable IPV6 mbuf hash delivery\n"); 1553 mbuf_types &= ~RSS_TYPE_IPV6; 1554 } 1555 if ((my_types & RSS_TYPE_IPV6_EX) && 1556 (diff_types & ifrh.ifrh_types & 1557 (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 | 1558 RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX | 1559 RSS_TYPE_IPV6))) { 1560 /* Conflict; disable IPV6_EX hash type/value delivery. */ 1561 if_printf(ifp, "disable IPV6_EX mbuf hash delivery\n"); 1562 mbuf_types &= ~RSS_TYPE_IPV6_EX; 1563 } 1564 if ((my_types & RSS_TYPE_TCP_IPV6) && 1565 (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6_EX)) { 1566 /* Conflict; disable TCP_IPV6 hash type/value delivery. */ 1567 if_printf(ifp, "disable TCP_IPV6 mbuf hash delivery\n"); 1568 mbuf_types &= ~RSS_TYPE_TCP_IPV6; 1569 } 1570 if ((my_types & RSS_TYPE_TCP_IPV6_EX) && 1571 (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6)) { 1572 /* Conflict; disable TCP_IPV6_EX hash type/value delivery. */ 1573 if_printf(ifp, "disable TCP_IPV6_EX mbuf hash delivery\n"); 1574 mbuf_types &= ~RSS_TYPE_TCP_IPV6_EX; 1575 } 1576 if ((my_types & RSS_TYPE_UDP_IPV6) && 1577 (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6_EX)) { 1578 /* Conflict; disable UDP_IPV6 hash type/value delivery. */ 1579 if_printf(ifp, "disable UDP_IPV6 mbuf hash delivery\n"); 1580 mbuf_types &= ~RSS_TYPE_UDP_IPV6; 1581 } 1582 if ((my_types & RSS_TYPE_UDP_IPV6_EX) && 1583 (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6)) { 1584 /* Conflict; disable UDP_IPV6_EX hash type/value delivery. */ 1585 if_printf(ifp, "disable UDP_IPV6_EX mbuf hash delivery\n"); 1586 mbuf_types &= ~RSS_TYPE_UDP_IPV6_EX; 1587 } 1588 1589 /* 1590 * Indirect table does not matter. 1591 */ 1592 1593 sc->hn_rss_hash = (sc->hn_rss_hcap & NDIS_HASH_FUNCTION_MASK) | 1594 hn_rss_type_tondis(my_types); 1595 memcpy(sc->hn_rss.rss_key, ifrk.ifrk_key, sizeof(sc->hn_rss.rss_key)); 1596 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 1597 1598 if (reconf) { 1599 error = hn_rss_reconfig(sc); 1600 if (error) { 1601 /* XXX roll-back? */ 1602 if_printf(ifp, "hn_rss_reconfig failed: %d\n", error); 1603 /* XXX keep going. */ 1604 } 1605 } 1606 done: 1607 /* Hash deliverability for mbufs. */ 1608 hn_rss_mbuf_hash(sc, hn_rss_type_tondis(mbuf_types)); 1609 } 1610 1611 static void 1612 hn_vf_rss_restore(struct hn_softc *sc) 1613 { 1614 1615 HN_LOCK_ASSERT(sc); 1616 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 1617 ("%s: synthetic parts are not attached", if_name(sc->hn_ifp))); 1618 1619 if (sc->hn_rx_ring_inuse == 1) 1620 goto done; 1621 1622 /* 1623 * Restore hash types. Key does _not_ matter. 1624 */ 1625 if (sc->hn_rss_hash != sc->hn_rss_hcap) { 1626 int error; 1627 1628 sc->hn_rss_hash = sc->hn_rss_hcap; 1629 error = hn_rss_reconfig(sc); 1630 if (error) { 1631 if_printf(sc->hn_ifp, "hn_rss_reconfig failed: %d\n", 1632 error); 1633 /* XXX keep going. */ 1634 } 1635 } 1636 done: 1637 /* Hash deliverability for mbufs. */ 1638 hn_rss_mbuf_hash(sc, NDIS_HASH_ALL); 1639 } 1640 1641 static void 1642 hn_xpnt_vf_setready(struct hn_softc *sc) 1643 { 1644 if_t ifp, vf_ifp; 1645 struct ifreq ifr; 1646 1647 HN_LOCK_ASSERT(sc); 1648 ifp = sc->hn_ifp; 1649 vf_ifp = sc->hn_vf_ifp; 1650 1651 /* 1652 * Mark the VF ready. 1653 */ 1654 sc->hn_vf_rdytick = 0; 1655 1656 /* 1657 * Save information for restoration. 1658 */ 1659 sc->hn_saved_caps = if_getcapabilities(ifp); 1660 sc->hn_saved_tsomax = if_gethwtsomax(ifp); 1661 sc->hn_saved_tsosegcnt = if_gethwtsomaxsegcount(ifp); 1662 sc->hn_saved_tsosegsz = if_gethwtsomaxsegsize(ifp); 1663 sc->hn_saved_capenable = if_getcapenable(ifp); 1664 sc->hn_saved_hwassist = if_gethwassist(ifp); 1665 1666 /* 1667 * Intersect supported/enabled capabilities. 1668 * 1669 * NOTE: 1670 * if_hwassist is not changed here. 1671 */ 1672 if_setcapabilitiesbit(ifp, 0, if_getcapabilities(vf_ifp)); 1673 if_setcapenablebit(ifp, 0, if_getcapabilities(ifp)); 1674 1675 /* 1676 * Fix TSO settings. 1677 */ 1678 if (if_gethwtsomax(ifp) > if_gethwtsomax(vf_ifp)) 1679 if_sethwtsomax(ifp, if_gethwtsomax(vf_ifp)); 1680 if (if_gethwtsomaxsegcount(ifp) > if_gethwtsomaxsegcount(vf_ifp)) 1681 if_sethwtsomaxsegcount(ifp, if_gethwtsomaxsegcount(vf_ifp)); 1682 if (if_gethwtsomaxsegsize(ifp) > if_gethwtsomaxsegsize(vf_ifp)) 1683 if_sethwtsomaxsegsize(ifp, if_gethwtsomaxsegsize(vf_ifp)); 1684 1685 /* 1686 * Change VF's enabled capabilities. 1687 */ 1688 memset(&ifr, 0, sizeof(ifr)); 1689 strlcpy(ifr.ifr_name, if_name(vf_ifp), sizeof(ifr.ifr_name)); 1690 ifr.ifr_reqcap = if_getcapenable(ifp); 1691 hn_xpnt_vf_iocsetcaps(sc, &ifr); 1692 1693 if (if_getmtu(ifp) != ETHERMTU) { 1694 int error; 1695 1696 /* 1697 * Change VF's MTU. 1698 */ 1699 memset(&ifr, 0, sizeof(ifr)); 1700 strlcpy(ifr.ifr_name, if_name(vf_ifp), sizeof(ifr.ifr_name)); 1701 ifr.ifr_mtu = if_getmtu(ifp); 1702 error = ifhwioctl(SIOCSIFMTU, vf_ifp, (caddr_t)&ifr, curthread); 1703 if (error) { 1704 if_printf(ifp, "%s SIOCSIFMTU %u failed\n", 1705 if_name(vf_ifp), if_getmtu(ifp)); 1706 if (if_getmtu(ifp) > ETHERMTU) { 1707 if_printf(ifp, "change MTU to %d\n", ETHERMTU); 1708 1709 /* 1710 * XXX 1711 * No need to adjust the synthetic parts' MTU; 1712 * failure of the adjustment will cause us 1713 * infinite headache. 1714 */ 1715 if_setmtu(ifp, ETHERMTU); 1716 hn_mtu_change_fixup(sc); 1717 } 1718 } 1719 } 1720 } 1721 1722 static bool 1723 hn_xpnt_vf_isready(struct hn_softc *sc) 1724 { 1725 1726 HN_LOCK_ASSERT(sc); 1727 1728 if (!hn_xpnt_vf || sc->hn_vf_ifp == NULL) 1729 return (false); 1730 1731 if (sc->hn_vf_rdytick == 0) 1732 return (true); 1733 1734 if (sc->hn_vf_rdytick > ticks) 1735 return (false); 1736 1737 /* Mark VF as ready. */ 1738 hn_xpnt_vf_setready(sc); 1739 return (true); 1740 } 1741 1742 static void 1743 hn_xpnt_vf_setenable(struct hn_softc *sc) 1744 { 1745 int i; 1746 1747 HN_LOCK_ASSERT(sc); 1748 1749 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 1750 rm_wlock(&sc->hn_vf_lock); 1751 sc->hn_xvf_flags |= HN_XVFFLAG_ENABLED; 1752 rm_wunlock(&sc->hn_vf_lock); 1753 1754 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1755 sc->hn_rx_ring[i].hn_rx_flags |= HN_RX_FLAG_XPNT_VF; 1756 } 1757 1758 static void 1759 hn_xpnt_vf_setdisable(struct hn_softc *sc, bool clear_vf) 1760 { 1761 int i; 1762 1763 HN_LOCK_ASSERT(sc); 1764 1765 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 1766 rm_wlock(&sc->hn_vf_lock); 1767 sc->hn_xvf_flags &= ~HN_XVFFLAG_ENABLED; 1768 if (clear_vf) 1769 sc->hn_vf_ifp = NULL; 1770 rm_wunlock(&sc->hn_vf_lock); 1771 1772 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1773 sc->hn_rx_ring[i].hn_rx_flags &= ~HN_RX_FLAG_XPNT_VF; 1774 } 1775 1776 static void 1777 hn_xpnt_vf_init(struct hn_softc *sc) 1778 { 1779 int error; 1780 1781 HN_LOCK_ASSERT(sc); 1782 1783 KASSERT((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) == 0, 1784 ("%s: transparent VF was enabled", if_name(sc->hn_ifp))); 1785 1786 if (bootverbose) { 1787 if_printf(sc->hn_ifp, "try bringing up %s\n", 1788 if_name(sc->hn_vf_ifp)); 1789 } 1790 1791 /* 1792 * Bring the VF up. 1793 */ 1794 hn_xpnt_vf_saveifflags(sc); 1795 if_setflagbits(sc->hn_ifp, IFF_UP, 0); 1796 error = hn_xpnt_vf_iocsetflags(sc); 1797 if (error) { 1798 if_printf(sc->hn_ifp, "bringing up %s failed: %d\n", 1799 if_name(sc->hn_vf_ifp), error); 1800 return; 1801 } 1802 1803 /* 1804 * NOTE: 1805 * Datapath setting must happen _after_ bringing the VF up. 1806 */ 1807 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_VF); 1808 1809 /* 1810 * NOTE: 1811 * Fixup RSS related bits _after_ the VF is brought up, since 1812 * many VFs generate RSS key during it's initialization. 1813 */ 1814 hn_vf_rss_fixup(sc, true); 1815 1816 /* Mark transparent mode VF as enabled. */ 1817 hn_xpnt_vf_setenable(sc); 1818 } 1819 1820 static void 1821 hn_xpnt_vf_init_taskfunc(void *xsc, int pending __unused) 1822 { 1823 struct hn_softc *sc = xsc; 1824 1825 HN_LOCK(sc); 1826 1827 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 1828 goto done; 1829 if (sc->hn_vf_ifp == NULL) 1830 goto done; 1831 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 1832 goto done; 1833 1834 if (sc->hn_vf_rdytick != 0) { 1835 /* Mark VF as ready. */ 1836 hn_xpnt_vf_setready(sc); 1837 } 1838 1839 if (if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) { 1840 /* 1841 * Delayed VF initialization. 1842 */ 1843 if (bootverbose) { 1844 if_printf(sc->hn_ifp, "delayed initialize %s\n", 1845 if_name(sc->hn_vf_ifp)); 1846 } 1847 hn_xpnt_vf_init(sc); 1848 } 1849 done: 1850 HN_UNLOCK(sc); 1851 } 1852 1853 static void 1854 hn_ifnet_attevent(void *xsc, if_t ifp) 1855 { 1856 struct hn_softc *sc = xsc; 1857 1858 HN_LOCK(sc); 1859 1860 if (!(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 1861 goto done; 1862 1863 if (!hn_ismyvf(sc, ifp)) 1864 goto done; 1865 1866 if (sc->hn_vf_ifp != NULL) { 1867 if_printf(sc->hn_ifp, "%s was attached as VF\n", 1868 if_name(sc->hn_vf_ifp)); 1869 goto done; 1870 } 1871 1872 if (hn_xpnt_vf && if_getstartfn(ifp) != NULL) { 1873 /* 1874 * ifnet.if_start is _not_ supported by transparent 1875 * mode VF; mainly due to the IFF_DRV_OACTIVE flag. 1876 */ 1877 if_printf(sc->hn_ifp, "%s uses if_start, which is unsupported " 1878 "in transparent VF mode.\n", if_name(sc->hn_vf_ifp)); 1879 1880 goto done; 1881 } 1882 1883 rm_wlock(&hn_vfmap_lock); 1884 1885 if (if_getindex(ifp) >= hn_vfmap_size) { 1886 if_t *newmap; 1887 int newsize; 1888 1889 newsize = if_getindex(ifp) + HN_VFMAP_SIZE_DEF; 1890 newmap = malloc(sizeof(if_t) * newsize, M_DEVBUF, 1891 M_WAITOK | M_ZERO); 1892 1893 memcpy(newmap, hn_vfmap, 1894 sizeof(if_t) * hn_vfmap_size); 1895 free(hn_vfmap, M_DEVBUF); 1896 hn_vfmap = newmap; 1897 hn_vfmap_size = newsize; 1898 } 1899 KASSERT(hn_vfmap[if_getindex(ifp)] == NULL, 1900 ("%s: ifindex %d was mapped to %s", 1901 if_name(ifp), if_getindex(ifp), if_name(hn_vfmap[if_getindex(ifp)]))); 1902 hn_vfmap[if_getindex(ifp)] = sc->hn_ifp; 1903 1904 rm_wunlock(&hn_vfmap_lock); 1905 1906 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 1907 rm_wlock(&sc->hn_vf_lock); 1908 KASSERT((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) == 0, 1909 ("%s: transparent VF was enabled", if_name(sc->hn_ifp))); 1910 sc->hn_vf_ifp = ifp; 1911 rm_wunlock(&sc->hn_vf_lock); 1912 1913 if (hn_xpnt_vf) { 1914 int wait_ticks; 1915 1916 /* 1917 * Install if_input for vf_ifp, which does vf_ifp -> hn_ifp. 1918 * Save vf_ifp's current if_input for later restoration. 1919 */ 1920 sc->hn_vf_input = if_getinputfn(ifp); 1921 if_setinputfn(ifp, hn_xpnt_vf_input); 1922 1923 /* 1924 * Stop link status management; use the VF's. 1925 */ 1926 hn_suspend_mgmt(sc); 1927 1928 /* 1929 * Give VF sometime to complete its attach routing. 1930 */ 1931 wait_ticks = hn_xpnt_vf_attwait * hz; 1932 sc->hn_vf_rdytick = ticks + wait_ticks; 1933 1934 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 1935 wait_ticks); 1936 } 1937 done: 1938 HN_UNLOCK(sc); 1939 } 1940 1941 static void 1942 hn_ifnet_detevent(void *xsc, if_t ifp) 1943 { 1944 struct hn_softc *sc = xsc; 1945 1946 HN_LOCK(sc); 1947 1948 if (sc->hn_vf_ifp == NULL) 1949 goto done; 1950 1951 if (!hn_ismyvf(sc, ifp)) 1952 goto done; 1953 1954 if (hn_xpnt_vf) { 1955 /* 1956 * Make sure that the delayed initialization is not running. 1957 * 1958 * NOTE: 1959 * - This lock _must_ be released, since the hn_vf_init task 1960 * will try holding this lock. 1961 * - It is safe to release this lock here, since the 1962 * hn_ifnet_attevent() is interlocked by the hn_vf_ifp. 1963 * 1964 * XXX racy, if hn(4) ever detached. 1965 */ 1966 HN_UNLOCK(sc); 1967 taskqueue_drain_timeout(sc->hn_vf_taskq, &sc->hn_vf_init); 1968 HN_LOCK(sc); 1969 1970 KASSERT(sc->hn_vf_input != NULL, ("%s VF input is not saved", 1971 if_name(sc->hn_ifp))); 1972 if_setinputfn(ifp, sc->hn_vf_input); 1973 sc->hn_vf_input = NULL; 1974 1975 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) && 1976 (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) 1977 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_SYNTH); 1978 1979 if (sc->hn_vf_rdytick == 0) { 1980 /* 1981 * The VF was ready; restore some settings. 1982 */ 1983 if_setcapabilities(ifp, sc->hn_saved_caps); 1984 1985 if_sethwtsomax(ifp, sc->hn_saved_tsomax); 1986 if_sethwtsomaxsegcount(sc->hn_ifp, 1987 sc->hn_saved_tsosegcnt); 1988 if_sethwtsomaxsegsize(ifp, sc->hn_saved_tsosegsz); 1989 1990 if_setcapenable(ifp, sc->hn_saved_capenable); 1991 if_sethwassist(ifp, sc->hn_saved_hwassist); 1992 } 1993 1994 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 1995 /* 1996 * Restore RSS settings. 1997 */ 1998 hn_vf_rss_restore(sc); 1999 2000 /* 2001 * Resume link status management, which was suspended 2002 * by hn_ifnet_attevent(). 2003 */ 2004 hn_resume_mgmt(sc); 2005 } 2006 } 2007 2008 /* Mark transparent mode VF as disabled. */ 2009 hn_xpnt_vf_setdisable(sc, true /* clear hn_vf_ifp */); 2010 2011 rm_wlock(&hn_vfmap_lock); 2012 2013 KASSERT(if_getindex(ifp) < hn_vfmap_size, 2014 ("ifindex %d, vfmapsize %d", if_getindex(ifp), hn_vfmap_size)); 2015 if (hn_vfmap[if_getindex(ifp)] != NULL) { 2016 KASSERT(hn_vfmap[if_getindex(ifp)] == sc->hn_ifp, 2017 ("%s: ifindex %d was mapped to %s", 2018 if_name(ifp), if_getindex(ifp), 2019 if_name(hn_vfmap[if_getindex(ifp)]))); 2020 hn_vfmap[if_getindex(ifp)] = NULL; 2021 } 2022 2023 rm_wunlock(&hn_vfmap_lock); 2024 done: 2025 HN_UNLOCK(sc); 2026 } 2027 2028 static void 2029 hn_ifnet_lnkevent(void *xsc, if_t ifp, int link_state) 2030 { 2031 struct hn_softc *sc = xsc; 2032 2033 if (sc->hn_vf_ifp == ifp) 2034 if_link_state_change(sc->hn_ifp, link_state); 2035 } 2036 2037 static int 2038 hn_tsomax_sysctl(SYSCTL_HANDLER_ARGS) 2039 { 2040 struct hn_softc *sc = arg1; 2041 unsigned int tsomax; 2042 int error; 2043 2044 tsomax = if_gethwtsomax(sc->hn_ifp); 2045 error = sysctl_handle_int(oidp, &tsomax, 0, req); 2046 return error; 2047 } 2048 2049 static int 2050 hn_tsomaxsegcnt_sysctl(SYSCTL_HANDLER_ARGS) 2051 { 2052 struct hn_softc *sc = arg1; 2053 unsigned int tsomaxsegcnt; 2054 int error; 2055 2056 tsomaxsegcnt = if_gethwtsomaxsegcount(sc->hn_ifp); 2057 error = sysctl_handle_int(oidp, &tsomaxsegcnt, 0, req); 2058 return error; 2059 } 2060 2061 static int 2062 hn_tsomaxsegsz_sysctl(SYSCTL_HANDLER_ARGS) 2063 { 2064 struct hn_softc *sc = arg1; 2065 unsigned int tsomaxsegsz; 2066 int error; 2067 2068 tsomaxsegsz = if_gethwtsomaxsegsize(sc->hn_ifp); 2069 error = sysctl_handle_int(oidp, &tsomaxsegsz, 0, req); 2070 return error; 2071 } 2072 2073 static int 2074 hn_probe(device_t dev) 2075 { 2076 2077 if (VMBUS_PROBE_GUID(device_get_parent(dev), dev, &hn_guid) == 0) { 2078 device_set_desc(dev, "Hyper-V Network Interface"); 2079 return BUS_PROBE_DEFAULT; 2080 } 2081 return ENXIO; 2082 } 2083 2084 static int 2085 hn_attach(device_t dev) 2086 { 2087 struct hn_softc *sc = device_get_softc(dev); 2088 struct sysctl_oid_list *child; 2089 struct sysctl_ctx_list *ctx; 2090 uint8_t eaddr[ETHER_ADDR_LEN]; 2091 if_t ifp = NULL; 2092 int error, ring_cnt, tx_ring_cnt; 2093 uint32_t mtu; 2094 2095 sc->hn_dev = dev; 2096 sc->hn_prichan = vmbus_get_channel(dev); 2097 HN_LOCK_INIT(sc); 2098 rm_init(&sc->hn_vf_lock, "hnvf"); 2099 if (hn_xpnt_vf && hn_xpnt_vf_accbpf) 2100 sc->hn_xvf_flags |= HN_XVFFLAG_ACCBPF; 2101 2102 /* 2103 * Initialize these tunables once. 2104 */ 2105 sc->hn_agg_size = hn_tx_agg_size; 2106 sc->hn_agg_pkts = hn_tx_agg_pkts; 2107 2108 /* 2109 * Setup taskqueue for transmission. 2110 */ 2111 if (hn_tx_taskq_mode == HN_TX_TASKQ_M_INDEP) { 2112 int i; 2113 2114 sc->hn_tx_taskqs = 2115 malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), 2116 M_DEVBUF, M_WAITOK); 2117 for (i = 0; i < hn_tx_taskq_cnt; ++i) { 2118 sc->hn_tx_taskqs[i] = taskqueue_create("hn_tx", 2119 M_WAITOK, taskqueue_thread_enqueue, 2120 &sc->hn_tx_taskqs[i]); 2121 taskqueue_start_threads(&sc->hn_tx_taskqs[i], 1, PI_NET, 2122 "%s tx%d", device_get_nameunit(dev), i); 2123 } 2124 } else if (hn_tx_taskq_mode == HN_TX_TASKQ_M_GLOBAL) { 2125 sc->hn_tx_taskqs = hn_tx_taskque; 2126 } 2127 2128 /* 2129 * Setup taskqueue for mangement tasks, e.g. link status. 2130 */ 2131 sc->hn_mgmt_taskq0 = taskqueue_create("hn_mgmt", M_WAITOK, 2132 taskqueue_thread_enqueue, &sc->hn_mgmt_taskq0); 2133 taskqueue_start_threads(&sc->hn_mgmt_taskq0, 1, PI_NET, "%s mgmt", 2134 device_get_nameunit(dev)); 2135 TASK_INIT(&sc->hn_link_task, 0, hn_link_taskfunc, sc); 2136 TASK_INIT(&sc->hn_netchg_init, 0, hn_netchg_init_taskfunc, sc); 2137 TIMEOUT_TASK_INIT(sc->hn_mgmt_taskq0, &sc->hn_netchg_status, 0, 2138 hn_netchg_status_taskfunc, sc); 2139 2140 if (hn_xpnt_vf) { 2141 /* 2142 * Setup taskqueue for VF tasks, e.g. delayed VF bringing up. 2143 */ 2144 sc->hn_vf_taskq = taskqueue_create("hn_vf", M_WAITOK, 2145 taskqueue_thread_enqueue, &sc->hn_vf_taskq); 2146 taskqueue_start_threads(&sc->hn_vf_taskq, 1, PI_NET, "%s vf", 2147 device_get_nameunit(dev)); 2148 TIMEOUT_TASK_INIT(sc->hn_vf_taskq, &sc->hn_vf_init, 0, 2149 hn_xpnt_vf_init_taskfunc, sc); 2150 } 2151 2152 /* 2153 * Allocate ifnet and setup its name earlier, so that if_printf 2154 * can be used by functions, which will be called after 2155 * ether_ifattach(). 2156 */ 2157 ifp = sc->hn_ifp = if_alloc(IFT_ETHER); 2158 if_setsoftc(ifp, sc); 2159 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2160 2161 /* 2162 * Initialize ifmedia earlier so that it can be unconditionally 2163 * destroyed, if error happened later on. 2164 */ 2165 ifmedia_init(&sc->hn_media, 0, hn_ifmedia_upd, hn_ifmedia_sts); 2166 2167 /* 2168 * Figure out the # of RX rings (ring_cnt) and the # of TX rings 2169 * to use (tx_ring_cnt). 2170 * 2171 * NOTE: 2172 * The # of RX rings to use is same as the # of channels to use. 2173 */ 2174 ring_cnt = hn_chan_cnt; 2175 if (ring_cnt <= 0) { 2176 /* Default */ 2177 ring_cnt = mp_ncpus; 2178 if (ring_cnt > HN_RING_CNT_DEF_MAX) 2179 ring_cnt = HN_RING_CNT_DEF_MAX; 2180 } else if (ring_cnt > mp_ncpus) { 2181 ring_cnt = mp_ncpus; 2182 } 2183 #ifdef RSS 2184 if (ring_cnt > rss_getnumbuckets()) 2185 ring_cnt = rss_getnumbuckets(); 2186 #endif 2187 2188 tx_ring_cnt = hn_tx_ring_cnt; 2189 if (tx_ring_cnt <= 0 || tx_ring_cnt > ring_cnt) 2190 tx_ring_cnt = ring_cnt; 2191 #ifdef HN_IFSTART_SUPPORT 2192 if (hn_use_if_start) { 2193 /* ifnet.if_start only needs one TX ring. */ 2194 tx_ring_cnt = 1; 2195 } 2196 #endif 2197 2198 /* 2199 * Set the leader CPU for channels. 2200 */ 2201 sc->hn_cpu = atomic_fetchadd_int(&hn_cpu_index, ring_cnt) % mp_ncpus; 2202 2203 /* 2204 * Create enough TX/RX rings, even if only limited number of 2205 * channels can be allocated. 2206 */ 2207 error = hn_create_tx_data(sc, tx_ring_cnt); 2208 if (error) 2209 goto failed; 2210 error = hn_create_rx_data(sc, ring_cnt); 2211 if (error) 2212 goto failed; 2213 2214 /* 2215 * Create transaction context for NVS and RNDIS transactions. 2216 */ 2217 sc->hn_xact = vmbus_xact_ctx_create(bus_get_dma_tag(dev), 2218 HN_XACT_REQ_SIZE, HN_XACT_RESP_SIZE, 0); 2219 if (sc->hn_xact == NULL) { 2220 error = ENXIO; 2221 goto failed; 2222 } 2223 2224 /* 2225 * Install orphan handler for the revocation of this device's 2226 * primary channel. 2227 * 2228 * NOTE: 2229 * The processing order is critical here: 2230 * Install the orphan handler, _before_ testing whether this 2231 * device's primary channel has been revoked or not. 2232 */ 2233 vmbus_chan_set_orphan(sc->hn_prichan, sc->hn_xact); 2234 if (vmbus_chan_is_revoked(sc->hn_prichan)) { 2235 error = ENXIO; 2236 goto failed; 2237 } 2238 2239 /* 2240 * Attach the synthetic parts, i.e. NVS and RNDIS. 2241 */ 2242 error = hn_synth_attach(sc, ETHERMTU); 2243 if (error) 2244 goto failed; 2245 2246 error = hn_rndis_get_eaddr(sc, eaddr); 2247 if (error) 2248 goto failed; 2249 2250 error = hn_rndis_get_mtu(sc, &mtu); 2251 if (error) 2252 mtu = ETHERMTU; 2253 else if (bootverbose) 2254 device_printf(dev, "RNDIS mtu %u\n", mtu); 2255 2256 if (sc->hn_rx_ring_inuse > 1) { 2257 /* 2258 * Reduce TCP segment aggregation limit for multiple 2259 * RX rings to increase ACK timeliness. 2260 */ 2261 hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MULTIRX_DEF); 2262 } 2263 2264 /* 2265 * Fixup TX/RX stuffs after synthetic parts are attached. 2266 */ 2267 hn_fixup_tx_data(sc); 2268 hn_fixup_rx_data(sc); 2269 2270 ctx = device_get_sysctl_ctx(dev); 2271 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 2272 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "nvs_version", CTLFLAG_RD, 2273 &sc->hn_nvs_ver, 0, "NVS version"); 2274 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "ndis_version", 2275 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2276 hn_ndis_version_sysctl, "A", "NDIS version"); 2277 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "caps", 2278 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2279 hn_caps_sysctl, "A", "capabilities"); 2280 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "hwassist", 2281 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2282 hn_hwassist_sysctl, "A", "hwassist"); 2283 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_max", 2284 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomax_sysctl, 2285 "IU", "max TSO size"); 2286 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_maxsegcnt", 2287 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomaxsegcnt_sysctl, 2288 "IU", "max # of TSO segments"); 2289 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_maxsegsz", 2290 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomaxsegsz_sysctl, 2291 "IU", "max size of TSO segment"); 2292 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxfilter", 2293 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2294 hn_rxfilter_sysctl, "A", "rxfilter"); 2295 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hash", 2296 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2297 hn_rss_hash_sysctl, "A", "RSS hash"); 2298 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hashcap", 2299 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2300 hn_rss_hcap_sysctl, "A", "RSS hash capabilities"); 2301 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "mbuf_hash", 2302 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2303 hn_rss_mbuf_sysctl, "A", "RSS hash for mbufs"); 2304 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rss_ind_size", 2305 CTLFLAG_RD, &sc->hn_rss_ind_size, 0, "RSS indirect entry count"); 2306 #ifndef RSS 2307 /* 2308 * Don't allow RSS key/indirect table changes, if RSS is defined. 2309 */ 2310 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_key", 2311 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2312 hn_rss_key_sysctl, "IU", "RSS key"); 2313 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_ind", 2314 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2315 hn_rss_ind_sysctl, "IU", "RSS indirect table"); 2316 #endif 2317 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_size", 2318 CTLFLAG_RD, &sc->hn_rndis_agg_size, 0, 2319 "RNDIS offered packet transmission aggregation size limit"); 2320 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_pkts", 2321 CTLFLAG_RD, &sc->hn_rndis_agg_pkts, 0, 2322 "RNDIS offered packet transmission aggregation count limit"); 2323 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_align", 2324 CTLFLAG_RD, &sc->hn_rndis_agg_align, 0, 2325 "RNDIS packet transmission aggregation alignment"); 2326 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_size", 2327 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2328 hn_txagg_size_sysctl, "I", 2329 "Packet transmission aggregation size, 0 -- disable, -1 -- auto"); 2330 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_pkts", 2331 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2332 hn_txagg_pkts_sysctl, "I", 2333 "Packet transmission aggregation packets, " 2334 "0 -- disable, -1 -- auto"); 2335 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "polling", 2336 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2337 hn_polling_sysctl, "I", 2338 "Polling frequency: [100,1000000], 0 disable polling"); 2339 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf", 2340 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2341 hn_vf_sysctl, "A", "Virtual Function's name"); 2342 if (!hn_xpnt_vf) { 2343 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxvf", 2344 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2345 hn_rxvf_sysctl, "A", "activated Virtual Function's name"); 2346 } else { 2347 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf_xpnt_enabled", 2348 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2349 hn_xpnt_vf_enabled_sysctl, "I", 2350 "Transparent VF enabled"); 2351 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf_xpnt_accbpf", 2352 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2353 hn_xpnt_vf_accbpf_sysctl, "I", 2354 "Accurate BPF for transparent VF"); 2355 } 2356 2357 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rsc_switch", 2358 CTLTYPE_UINT | CTLFLAG_RW, sc, 0, hn_rsc_sysctl, "A", 2359 "switch to rsc"); 2360 2361 /* 2362 * Setup the ifmedia, which has been initialized earlier. 2363 */ 2364 ifmedia_add(&sc->hn_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2365 ifmedia_set(&sc->hn_media, IFM_ETHER | IFM_AUTO); 2366 /* XXX ifmedia_set really should do this for us */ 2367 sc->hn_media.ifm_media = sc->hn_media.ifm_cur->ifm_media; 2368 2369 /* 2370 * Setup the ifnet for this interface. 2371 */ 2372 2373 if_setbaudrate(ifp, IF_Gbps(10)); 2374 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2375 if_setioctlfn(ifp, hn_ioctl); 2376 if_setinitfn(ifp, hn_init); 2377 #ifdef HN_IFSTART_SUPPORT 2378 if (hn_use_if_start) { 2379 int qdepth = hn_get_txswq_depth(&sc->hn_tx_ring[0]); 2380 2381 if_setstartfn(ifp, hn_start); 2382 if_setsendqlen(ifp, qdepth); 2383 if_setsendqready(ifp); 2384 } else 2385 #endif 2386 { 2387 if_settransmitfn(ifp, hn_transmit); 2388 if_setqflushfn(ifp, hn_xmit_qflush); 2389 } 2390 2391 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM | IFCAP_LRO | IFCAP_LINKSTATE, 0); 2392 #ifdef foo 2393 /* We can't diff IPv6 packets from IPv4 packets on RX path. */ 2394 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM_IPV6, 0); 2395 #endif 2396 if (sc->hn_caps & HN_CAP_VLAN) { 2397 /* XXX not sure about VLAN_MTU. */ 2398 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); 2399 } 2400 2401 if_sethwassist(ifp, sc->hn_tx_ring[0].hn_csum_assist); 2402 if (if_gethwassist(ifp) & HN_CSUM_IP_MASK) 2403 if_setcapabilitiesbit(ifp, IFCAP_TXCSUM, 0); 2404 if (if_gethwassist(ifp) & HN_CSUM_IP6_MASK) 2405 if_setcapabilitiesbit(ifp, IFCAP_TXCSUM_IPV6, 0); 2406 if (sc->hn_caps & HN_CAP_TSO4) { 2407 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 2408 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 2409 } 2410 if (sc->hn_caps & HN_CAP_TSO6) { 2411 if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0); 2412 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 2413 } 2414 2415 /* Enable all available capabilities by default. */ 2416 if_setcapenable(ifp, if_getcapabilities(ifp)); 2417 2418 /* 2419 * Disable IPv6 TSO and TXCSUM by default, they still can 2420 * be enabled through SIOCSIFCAP. 2421 */ 2422 if_setcapenablebit(ifp, 0, (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6)); 2423 if_sethwassistbits(ifp, 0, (HN_CSUM_IP6_MASK | CSUM_IP6_TSO)); 2424 2425 if (if_getcapabilities(ifp) & (IFCAP_TSO6 | IFCAP_TSO4)) { 2426 /* 2427 * Lock hn_set_tso_maxsize() to simplify its 2428 * internal logic. 2429 */ 2430 HN_LOCK(sc); 2431 hn_set_tso_maxsize(sc, hn_tso_maxlen, ETHERMTU); 2432 HN_UNLOCK(sc); 2433 if_sethwtsomaxsegcount(ifp, HN_TX_DATA_SEGCNT_MAX); 2434 if_sethwtsomaxsegsize(ifp, PAGE_SIZE); 2435 } 2436 2437 ether_ifattach(ifp, eaddr); 2438 2439 if ((if_getcapabilities(ifp) & (IFCAP_TSO6 | IFCAP_TSO4)) && bootverbose) { 2440 if_printf(ifp, "TSO segcnt %u segsz %u\n", 2441 if_gethwtsomaxsegcount(ifp), if_gethwtsomaxsegsize(ifp)); 2442 } 2443 if (mtu < ETHERMTU) { 2444 2445 if_setmtu(ifp, mtu); 2446 } 2447 2448 /* Inform the upper layer about the long frame support. */ 2449 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2450 2451 /* 2452 * Kick off link status check. 2453 */ 2454 sc->hn_mgmt_taskq = sc->hn_mgmt_taskq0; 2455 hn_update_link_status(sc); 2456 2457 if (!hn_xpnt_vf) { 2458 sc->hn_ifnet_evthand = EVENTHANDLER_REGISTER(ifnet_event, 2459 hn_ifnet_event, sc, EVENTHANDLER_PRI_ANY); 2460 sc->hn_ifaddr_evthand = EVENTHANDLER_REGISTER(ifaddr_event, 2461 hn_ifaddr_event, sc, EVENTHANDLER_PRI_ANY); 2462 } else { 2463 sc->hn_ifnet_lnkhand = EVENTHANDLER_REGISTER(ifnet_link_event, 2464 hn_ifnet_lnkevent, sc, EVENTHANDLER_PRI_ANY); 2465 } 2466 2467 /* 2468 * NOTE: 2469 * Subscribe ether_ifattach event, instead of ifnet_arrival event, 2470 * since interface's LLADDR is needed; interface LLADDR is not 2471 * available when ifnet_arrival event is triggered. 2472 */ 2473 sc->hn_ifnet_atthand = EVENTHANDLER_REGISTER(ether_ifattach_event, 2474 hn_ifnet_attevent, sc, EVENTHANDLER_PRI_ANY); 2475 sc->hn_ifnet_dethand = EVENTHANDLER_REGISTER(ifnet_departure_event, 2476 hn_ifnet_detevent, sc, EVENTHANDLER_PRI_ANY); 2477 2478 return (0); 2479 failed: 2480 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) 2481 hn_synth_detach(sc); 2482 hn_detach(dev); 2483 return (error); 2484 } 2485 2486 static int 2487 hn_detach(device_t dev) 2488 { 2489 struct hn_softc *sc = device_get_softc(dev); 2490 if_t ifp = sc->hn_ifp, vf_ifp; 2491 2492 if (sc->hn_xact != NULL && vmbus_chan_is_revoked(sc->hn_prichan)) { 2493 /* 2494 * In case that the vmbus missed the orphan handler 2495 * installation. 2496 */ 2497 vmbus_xact_ctx_orphan(sc->hn_xact); 2498 } 2499 2500 if (sc->hn_ifaddr_evthand != NULL) 2501 EVENTHANDLER_DEREGISTER(ifaddr_event, sc->hn_ifaddr_evthand); 2502 if (sc->hn_ifnet_evthand != NULL) 2503 EVENTHANDLER_DEREGISTER(ifnet_event, sc->hn_ifnet_evthand); 2504 if (sc->hn_ifnet_atthand != NULL) { 2505 EVENTHANDLER_DEREGISTER(ether_ifattach_event, 2506 sc->hn_ifnet_atthand); 2507 } 2508 if (sc->hn_ifnet_dethand != NULL) { 2509 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 2510 sc->hn_ifnet_dethand); 2511 } 2512 if (sc->hn_ifnet_lnkhand != NULL) 2513 EVENTHANDLER_DEREGISTER(ifnet_link_event, sc->hn_ifnet_lnkhand); 2514 2515 vf_ifp = sc->hn_vf_ifp; 2516 __compiler_membar(); 2517 if (vf_ifp != NULL) 2518 hn_ifnet_detevent(sc, vf_ifp); 2519 2520 if (device_is_attached(dev)) { 2521 HN_LOCK(sc); 2522 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 2523 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 2524 hn_stop(sc, true); 2525 /* 2526 * NOTE: 2527 * hn_stop() only suspends data, so management 2528 * stuffs have to be suspended manually here. 2529 */ 2530 hn_suspend_mgmt(sc); 2531 hn_synth_detach(sc); 2532 } 2533 HN_UNLOCK(sc); 2534 ether_ifdetach(ifp); 2535 } 2536 2537 ifmedia_removeall(&sc->hn_media); 2538 hn_destroy_rx_data(sc); 2539 hn_destroy_tx_data(sc); 2540 2541 if (sc->hn_tx_taskqs != NULL && sc->hn_tx_taskqs != hn_tx_taskque) { 2542 int i; 2543 2544 for (i = 0; i < hn_tx_taskq_cnt; ++i) 2545 taskqueue_free(sc->hn_tx_taskqs[i]); 2546 free(sc->hn_tx_taskqs, M_DEVBUF); 2547 } 2548 taskqueue_free(sc->hn_mgmt_taskq0); 2549 if (sc->hn_vf_taskq != NULL) 2550 taskqueue_free(sc->hn_vf_taskq); 2551 2552 if (sc->hn_xact != NULL) { 2553 /* 2554 * Uninstall the orphan handler _before_ the xact is 2555 * destructed. 2556 */ 2557 vmbus_chan_unset_orphan(sc->hn_prichan); 2558 vmbus_xact_ctx_destroy(sc->hn_xact); 2559 } 2560 2561 if_free(ifp); 2562 2563 HN_LOCK_DESTROY(sc); 2564 rm_destroy(&sc->hn_vf_lock); 2565 return (0); 2566 } 2567 2568 static int 2569 hn_shutdown(device_t dev) 2570 { 2571 2572 return (0); 2573 } 2574 2575 static void 2576 hn_link_status(struct hn_softc *sc) 2577 { 2578 uint32_t link_status; 2579 int error; 2580 2581 error = hn_rndis_get_linkstatus(sc, &link_status); 2582 if (error) { 2583 /* XXX what to do? */ 2584 return; 2585 } 2586 2587 if (link_status == NDIS_MEDIA_STATE_CONNECTED) 2588 sc->hn_link_flags |= HN_LINK_FLAG_LINKUP; 2589 else 2590 sc->hn_link_flags &= ~HN_LINK_FLAG_LINKUP; 2591 if_link_state_change(sc->hn_ifp, 2592 (sc->hn_link_flags & HN_LINK_FLAG_LINKUP) ? 2593 LINK_STATE_UP : LINK_STATE_DOWN); 2594 } 2595 2596 static void 2597 hn_link_taskfunc(void *xsc, int pending __unused) 2598 { 2599 struct hn_softc *sc = xsc; 2600 2601 if (sc->hn_link_flags & HN_LINK_FLAG_NETCHG) 2602 return; 2603 hn_link_status(sc); 2604 } 2605 2606 static void 2607 hn_netchg_init_taskfunc(void *xsc, int pending __unused) 2608 { 2609 struct hn_softc *sc = xsc; 2610 2611 /* Prevent any link status checks from running. */ 2612 sc->hn_link_flags |= HN_LINK_FLAG_NETCHG; 2613 2614 /* 2615 * Fake up a [link down --> link up] state change; 5 seconds 2616 * delay is used, which closely simulates miibus reaction 2617 * upon link down event. 2618 */ 2619 sc->hn_link_flags &= ~HN_LINK_FLAG_LINKUP; 2620 if_link_state_change(sc->hn_ifp, LINK_STATE_DOWN); 2621 taskqueue_enqueue_timeout(sc->hn_mgmt_taskq0, 2622 &sc->hn_netchg_status, 5 * hz); 2623 } 2624 2625 static void 2626 hn_netchg_status_taskfunc(void *xsc, int pending __unused) 2627 { 2628 struct hn_softc *sc = xsc; 2629 2630 /* Re-allow link status checks. */ 2631 sc->hn_link_flags &= ~HN_LINK_FLAG_NETCHG; 2632 hn_link_status(sc); 2633 } 2634 2635 static void 2636 hn_update_link_status(struct hn_softc *sc) 2637 { 2638 2639 if (sc->hn_mgmt_taskq != NULL) 2640 taskqueue_enqueue(sc->hn_mgmt_taskq, &sc->hn_link_task); 2641 } 2642 2643 static void 2644 hn_change_network(struct hn_softc *sc) 2645 { 2646 2647 if (sc->hn_mgmt_taskq != NULL) 2648 taskqueue_enqueue(sc->hn_mgmt_taskq, &sc->hn_netchg_init); 2649 } 2650 2651 static __inline int 2652 hn_txdesc_dmamap_load(struct hn_tx_ring *txr, struct hn_txdesc *txd, 2653 struct mbuf **m_head, bus_dma_segment_t *segs, int *nsegs) 2654 { 2655 struct mbuf *m = *m_head; 2656 int error; 2657 2658 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID, ("txd uses chim")); 2659 2660 error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, txd->data_dmap, 2661 m, segs, nsegs, BUS_DMA_NOWAIT); 2662 if (error == EFBIG) { 2663 struct mbuf *m_new; 2664 2665 m_new = m_collapse(m, M_NOWAIT, HN_TX_DATA_SEGCNT_MAX); 2666 if (m_new == NULL) 2667 return ENOBUFS; 2668 else 2669 *m_head = m = m_new; 2670 txr->hn_tx_collapsed++; 2671 2672 error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, 2673 txd->data_dmap, m, segs, nsegs, BUS_DMA_NOWAIT); 2674 } 2675 if (!error) { 2676 bus_dmamap_sync(txr->hn_tx_data_dtag, txd->data_dmap, 2677 BUS_DMASYNC_PREWRITE); 2678 txd->flags |= HN_TXD_FLAG_DMAMAP; 2679 } 2680 return error; 2681 } 2682 2683 static __inline int 2684 hn_txdesc_put(struct hn_tx_ring *txr, struct hn_txdesc *txd) 2685 { 2686 2687 KASSERT((txd->flags & HN_TXD_FLAG_ONLIST) == 0, 2688 ("put an onlist txd %#x", txd->flags)); 2689 KASSERT((txd->flags & HN_TXD_FLAG_ONAGG) == 0, 2690 ("put an onagg txd %#x", txd->flags)); 2691 2692 KASSERT(txd->refs > 0, ("invalid txd refs %d", txd->refs)); 2693 if (atomic_fetchadd_int(&txd->refs, -1) != 1) 2694 return 0; 2695 2696 if (!STAILQ_EMPTY(&txd->agg_list)) { 2697 struct hn_txdesc *tmp_txd; 2698 2699 while ((tmp_txd = STAILQ_FIRST(&txd->agg_list)) != NULL) { 2700 int freed __diagused; 2701 2702 KASSERT(STAILQ_EMPTY(&tmp_txd->agg_list), 2703 ("resursive aggregation on aggregated txdesc")); 2704 KASSERT((tmp_txd->flags & HN_TXD_FLAG_ONAGG), 2705 ("not aggregated txdesc")); 2706 KASSERT((tmp_txd->flags & HN_TXD_FLAG_DMAMAP) == 0, 2707 ("aggregated txdesc uses dmamap")); 2708 KASSERT(tmp_txd->chim_index == HN_NVS_CHIM_IDX_INVALID, 2709 ("aggregated txdesc consumes " 2710 "chimney sending buffer")); 2711 KASSERT(tmp_txd->chim_size == 0, 2712 ("aggregated txdesc has non-zero " 2713 "chimney sending size")); 2714 2715 STAILQ_REMOVE_HEAD(&txd->agg_list, agg_link); 2716 tmp_txd->flags &= ~HN_TXD_FLAG_ONAGG; 2717 freed = hn_txdesc_put(txr, tmp_txd); 2718 KASSERT(freed, ("failed to free aggregated txdesc")); 2719 } 2720 } 2721 2722 if (txd->chim_index != HN_NVS_CHIM_IDX_INVALID) { 2723 KASSERT((txd->flags & HN_TXD_FLAG_DMAMAP) == 0, 2724 ("chim txd uses dmamap")); 2725 hn_chim_free(txr->hn_sc, txd->chim_index); 2726 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 2727 txd->chim_size = 0; 2728 } else if (txd->flags & HN_TXD_FLAG_DMAMAP) { 2729 bus_dmamap_sync(txr->hn_tx_data_dtag, 2730 txd->data_dmap, BUS_DMASYNC_POSTWRITE); 2731 bus_dmamap_unload(txr->hn_tx_data_dtag, 2732 txd->data_dmap); 2733 txd->flags &= ~HN_TXD_FLAG_DMAMAP; 2734 } 2735 2736 if (txd->m != NULL) { 2737 m_freem(txd->m); 2738 txd->m = NULL; 2739 } 2740 2741 txd->flags |= HN_TXD_FLAG_ONLIST; 2742 #ifndef HN_USE_TXDESC_BUFRING 2743 mtx_lock_spin(&txr->hn_txlist_spin); 2744 KASSERT(txr->hn_txdesc_avail >= 0 && 2745 txr->hn_txdesc_avail < txr->hn_txdesc_cnt, 2746 ("txdesc_put: invalid txd avail %d", txr->hn_txdesc_avail)); 2747 txr->hn_txdesc_avail++; 2748 SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); 2749 mtx_unlock_spin(&txr->hn_txlist_spin); 2750 #else /* HN_USE_TXDESC_BUFRING */ 2751 #ifdef HN_DEBUG 2752 atomic_add_int(&txr->hn_txdesc_avail, 1); 2753 #endif 2754 buf_ring_enqueue(txr->hn_txdesc_br, txd); 2755 #endif /* !HN_USE_TXDESC_BUFRING */ 2756 2757 return 1; 2758 } 2759 2760 static __inline struct hn_txdesc * 2761 hn_txdesc_get(struct hn_tx_ring *txr) 2762 { 2763 struct hn_txdesc *txd; 2764 2765 #ifndef HN_USE_TXDESC_BUFRING 2766 mtx_lock_spin(&txr->hn_txlist_spin); 2767 txd = SLIST_FIRST(&txr->hn_txlist); 2768 if (txd != NULL) { 2769 KASSERT(txr->hn_txdesc_avail > 0, 2770 ("txdesc_get: invalid txd avail %d", txr->hn_txdesc_avail)); 2771 txr->hn_txdesc_avail--; 2772 SLIST_REMOVE_HEAD(&txr->hn_txlist, link); 2773 } 2774 mtx_unlock_spin(&txr->hn_txlist_spin); 2775 #else 2776 txd = buf_ring_dequeue_sc(txr->hn_txdesc_br); 2777 #endif 2778 2779 if (txd != NULL) { 2780 #ifdef HN_USE_TXDESC_BUFRING 2781 #ifdef HN_DEBUG 2782 atomic_subtract_int(&txr->hn_txdesc_avail, 1); 2783 #endif 2784 #endif /* HN_USE_TXDESC_BUFRING */ 2785 KASSERT(txd->m == NULL && txd->refs == 0 && 2786 STAILQ_EMPTY(&txd->agg_list) && 2787 txd->chim_index == HN_NVS_CHIM_IDX_INVALID && 2788 txd->chim_size == 0 && 2789 (txd->flags & HN_TXD_FLAG_ONLIST) && 2790 (txd->flags & HN_TXD_FLAG_ONAGG) == 0 && 2791 (txd->flags & HN_TXD_FLAG_DMAMAP) == 0, ("invalid txd")); 2792 txd->flags &= ~HN_TXD_FLAG_ONLIST; 2793 txd->refs = 1; 2794 } 2795 return txd; 2796 } 2797 2798 static __inline void 2799 hn_txdesc_hold(struct hn_txdesc *txd) 2800 { 2801 2802 /* 0->1 transition will never work */ 2803 KASSERT(txd->refs > 0, ("invalid txd refs %d", txd->refs)); 2804 atomic_add_int(&txd->refs, 1); 2805 } 2806 2807 static __inline void 2808 hn_txdesc_agg(struct hn_txdesc *agg_txd, struct hn_txdesc *txd) 2809 { 2810 2811 KASSERT((agg_txd->flags & HN_TXD_FLAG_ONAGG) == 0, 2812 ("recursive aggregation on aggregating txdesc")); 2813 2814 KASSERT((txd->flags & HN_TXD_FLAG_ONAGG) == 0, 2815 ("already aggregated")); 2816 KASSERT(STAILQ_EMPTY(&txd->agg_list), 2817 ("recursive aggregation on to-be-aggregated txdesc")); 2818 2819 txd->flags |= HN_TXD_FLAG_ONAGG; 2820 STAILQ_INSERT_TAIL(&agg_txd->agg_list, txd, agg_link); 2821 } 2822 2823 static bool 2824 hn_tx_ring_pending(struct hn_tx_ring *txr) 2825 { 2826 bool pending = false; 2827 2828 #ifndef HN_USE_TXDESC_BUFRING 2829 mtx_lock_spin(&txr->hn_txlist_spin); 2830 if (txr->hn_txdesc_avail != txr->hn_txdesc_cnt) 2831 pending = true; 2832 mtx_unlock_spin(&txr->hn_txlist_spin); 2833 #else 2834 if (!buf_ring_full(txr->hn_txdesc_br)) 2835 pending = true; 2836 #endif 2837 return (pending); 2838 } 2839 2840 static __inline void 2841 hn_txeof(struct hn_tx_ring *txr) 2842 { 2843 txr->hn_has_txeof = 0; 2844 txr->hn_txeof(txr); 2845 } 2846 2847 static void 2848 hn_txpkt_done(struct hn_nvs_sendctx *sndc, struct hn_softc *sc, 2849 struct vmbus_channel *chan, const void *data __unused, int dlen __unused) 2850 { 2851 struct hn_txdesc *txd = sndc->hn_cbarg; 2852 struct hn_tx_ring *txr; 2853 2854 txr = txd->txr; 2855 KASSERT(txr->hn_chan == chan, 2856 ("channel mismatch, on chan%u, should be chan%u", 2857 vmbus_chan_id(chan), vmbus_chan_id(txr->hn_chan))); 2858 2859 txr->hn_has_txeof = 1; 2860 hn_txdesc_put(txr, txd); 2861 2862 ++txr->hn_txdone_cnt; 2863 if (txr->hn_txdone_cnt >= HN_EARLY_TXEOF_THRESH) { 2864 txr->hn_txdone_cnt = 0; 2865 if (txr->hn_oactive) 2866 hn_txeof(txr); 2867 } 2868 } 2869 2870 static void 2871 hn_chan_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr) 2872 { 2873 #if defined(INET) || defined(INET6) 2874 struct epoch_tracker et; 2875 2876 NET_EPOCH_ENTER(et); 2877 tcp_lro_flush_all(&rxr->hn_lro); 2878 NET_EPOCH_EXIT(et); 2879 #endif 2880 2881 /* 2882 * NOTE: 2883 * 'txr' could be NULL, if multiple channels and 2884 * ifnet.if_start method are enabled. 2885 */ 2886 if (txr == NULL || !txr->hn_has_txeof) 2887 return; 2888 2889 txr->hn_txdone_cnt = 0; 2890 hn_txeof(txr); 2891 } 2892 2893 static __inline uint32_t 2894 hn_rndis_pktmsg_offset(uint32_t ofs) 2895 { 2896 2897 KASSERT(ofs >= sizeof(struct rndis_packet_msg), 2898 ("invalid RNDIS packet msg offset %u", ofs)); 2899 return (ofs - __offsetof(struct rndis_packet_msg, rm_dataoffset)); 2900 } 2901 2902 static __inline void * 2903 hn_rndis_pktinfo_append(struct rndis_packet_msg *pkt, size_t pktsize, 2904 size_t pi_dlen, uint32_t pi_type) 2905 { 2906 const size_t pi_size = HN_RNDIS_PKTINFO_SIZE(pi_dlen); 2907 struct rndis_pktinfo *pi; 2908 2909 KASSERT((pi_size & RNDIS_PACKET_MSG_OFFSET_ALIGNMASK) == 0, 2910 ("unaligned pktinfo size %zu, pktinfo dlen %zu", pi_size, pi_dlen)); 2911 2912 /* 2913 * Per-packet-info does not move; it only grows. 2914 * 2915 * NOTE: 2916 * rm_pktinfooffset in this phase counts from the beginning 2917 * of rndis_packet_msg. 2918 */ 2919 KASSERT(pkt->rm_pktinfooffset + pkt->rm_pktinfolen + pi_size <= pktsize, 2920 ("%u pktinfo overflows RNDIS packet msg", pi_type)); 2921 pi = (struct rndis_pktinfo *)((uint8_t *)pkt + pkt->rm_pktinfooffset + 2922 pkt->rm_pktinfolen); 2923 pkt->rm_pktinfolen += pi_size; 2924 2925 pi->rm_size = pi_size; 2926 pi->rm_type = pi_type; 2927 pi->rm_internal = 0; 2928 pi->rm_pktinfooffset = RNDIS_PKTINFO_OFFSET; 2929 2930 return (pi->rm_data); 2931 } 2932 2933 static __inline int 2934 hn_flush_txagg(if_t ifp, struct hn_tx_ring *txr) 2935 { 2936 struct hn_txdesc *txd; 2937 struct mbuf *m; 2938 int error, pkts; 2939 2940 txd = txr->hn_agg_txd; 2941 KASSERT(txd != NULL, ("no aggregate txdesc")); 2942 2943 /* 2944 * Since hn_txpkt() will reset this temporary stat, save 2945 * it now, so that oerrors can be updated properly, if 2946 * hn_txpkt() ever fails. 2947 */ 2948 pkts = txr->hn_stat_pkts; 2949 2950 /* 2951 * Since txd's mbuf will _not_ be freed upon hn_txpkt() 2952 * failure, save it for later freeing, if hn_txpkt() ever 2953 * fails. 2954 */ 2955 m = txd->m; 2956 error = hn_txpkt(ifp, txr, txd); 2957 if (__predict_false(error)) { 2958 /* txd is freed, but m is not. */ 2959 m_freem(m); 2960 2961 txr->hn_flush_failed++; 2962 if_inc_counter(ifp, IFCOUNTER_OERRORS, pkts); 2963 } 2964 2965 /* Reset all aggregation states. */ 2966 txr->hn_agg_txd = NULL; 2967 txr->hn_agg_szleft = 0; 2968 txr->hn_agg_pktleft = 0; 2969 txr->hn_agg_prevpkt = NULL; 2970 2971 return (error); 2972 } 2973 2974 static void * 2975 hn_try_txagg(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd, 2976 int pktsize) 2977 { 2978 void *chim; 2979 2980 if (txr->hn_agg_txd != NULL) { 2981 if (txr->hn_agg_pktleft >= 1 && txr->hn_agg_szleft > pktsize) { 2982 struct hn_txdesc *agg_txd = txr->hn_agg_txd; 2983 struct rndis_packet_msg *pkt = txr->hn_agg_prevpkt; 2984 int olen; 2985 2986 /* 2987 * Update the previous RNDIS packet's total length, 2988 * it can be increased due to the mandatory alignment 2989 * padding for this RNDIS packet. And update the 2990 * aggregating txdesc's chimney sending buffer size 2991 * accordingly. 2992 * 2993 * XXX 2994 * Zero-out the padding, as required by the RNDIS spec. 2995 */ 2996 olen = pkt->rm_len; 2997 pkt->rm_len = roundup2(olen, txr->hn_agg_align); 2998 agg_txd->chim_size += pkt->rm_len - olen; 2999 3000 /* Link this txdesc to the parent. */ 3001 hn_txdesc_agg(agg_txd, txd); 3002 3003 chim = (uint8_t *)pkt + pkt->rm_len; 3004 /* Save the current packet for later fixup. */ 3005 txr->hn_agg_prevpkt = chim; 3006 3007 txr->hn_agg_pktleft--; 3008 txr->hn_agg_szleft -= pktsize; 3009 if (txr->hn_agg_szleft <= 3010 HN_PKTSIZE_MIN(txr->hn_agg_align)) { 3011 /* 3012 * Probably can't aggregate more packets, 3013 * flush this aggregating txdesc proactively. 3014 */ 3015 txr->hn_agg_pktleft = 0; 3016 } 3017 /* Done! */ 3018 return (chim); 3019 } 3020 hn_flush_txagg(ifp, txr); 3021 } 3022 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 3023 3024 txr->hn_tx_chimney_tried++; 3025 txd->chim_index = hn_chim_alloc(txr->hn_sc); 3026 if (txd->chim_index == HN_NVS_CHIM_IDX_INVALID) 3027 return (NULL); 3028 txr->hn_tx_chimney++; 3029 3030 chim = txr->hn_sc->hn_chim + 3031 (txd->chim_index * txr->hn_sc->hn_chim_szmax); 3032 3033 if (txr->hn_agg_pktmax > 1 && 3034 txr->hn_agg_szmax > pktsize + HN_PKTSIZE_MIN(txr->hn_agg_align)) { 3035 txr->hn_agg_txd = txd; 3036 txr->hn_agg_pktleft = txr->hn_agg_pktmax - 1; 3037 txr->hn_agg_szleft = txr->hn_agg_szmax - pktsize; 3038 txr->hn_agg_prevpkt = chim; 3039 } 3040 return (chim); 3041 } 3042 3043 /* 3044 * NOTE: 3045 * If this function fails, then both txd and m_head0 will be freed. 3046 */ 3047 static int 3048 hn_encap(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd, 3049 struct mbuf **m_head0) 3050 { 3051 bus_dma_segment_t segs[HN_TX_DATA_SEGCNT_MAX]; 3052 int error, nsegs, i; 3053 struct mbuf *m_head = *m_head0; 3054 struct rndis_packet_msg *pkt; 3055 uint32_t *pi_data; 3056 void *chim = NULL; 3057 int pkt_hlen, pkt_size; 3058 3059 pkt = txd->rndis_pkt; 3060 pkt_size = HN_PKTSIZE(m_head, txr->hn_agg_align); 3061 if (pkt_size < txr->hn_chim_size) { 3062 chim = hn_try_txagg(ifp, txr, txd, pkt_size); 3063 if (chim != NULL) 3064 pkt = chim; 3065 } else { 3066 if (txr->hn_agg_txd != NULL) 3067 hn_flush_txagg(ifp, txr); 3068 } 3069 3070 pkt->rm_type = REMOTE_NDIS_PACKET_MSG; 3071 pkt->rm_len = m_head->m_pkthdr.len; 3072 pkt->rm_dataoffset = 0; 3073 pkt->rm_datalen = m_head->m_pkthdr.len; 3074 pkt->rm_oobdataoffset = 0; 3075 pkt->rm_oobdatalen = 0; 3076 pkt->rm_oobdataelements = 0; 3077 pkt->rm_pktinfooffset = sizeof(*pkt); 3078 pkt->rm_pktinfolen = 0; 3079 pkt->rm_vchandle = 0; 3080 pkt->rm_reserved = 0; 3081 3082 if (txr->hn_tx_flags & HN_TX_FLAG_HASHVAL) { 3083 /* 3084 * Set the hash value for this packet. 3085 */ 3086 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3087 HN_NDIS_HASH_VALUE_SIZE, HN_NDIS_PKTINFO_TYPE_HASHVAL); 3088 3089 if (M_HASHTYPE_ISHASH(m_head)) 3090 /* 3091 * The flowid field contains the hash value host 3092 * set in the rx queue if it is a ip forwarding pkt. 3093 * Set the same hash value so host can send on the 3094 * cpu it was received. 3095 */ 3096 *pi_data = m_head->m_pkthdr.flowid; 3097 else 3098 /* 3099 * Otherwise just put the tx queue index. 3100 */ 3101 *pi_data = txr->hn_tx_idx; 3102 } 3103 3104 if (m_head->m_flags & M_VLANTAG) { 3105 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3106 NDIS_VLAN_INFO_SIZE, NDIS_PKTINFO_TYPE_VLAN); 3107 *pi_data = NDIS_VLAN_INFO_MAKE( 3108 EVL_VLANOFTAG(m_head->m_pkthdr.ether_vtag), 3109 EVL_PRIOFTAG(m_head->m_pkthdr.ether_vtag), 3110 EVL_CFIOFTAG(m_head->m_pkthdr.ether_vtag)); 3111 } 3112 3113 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 3114 #if defined(INET6) || defined(INET) 3115 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3116 NDIS_LSO2_INFO_SIZE, NDIS_PKTINFO_TYPE_LSO); 3117 #ifdef INET 3118 if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { 3119 *pi_data = NDIS_LSO2_INFO_MAKEIPV4( 3120 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen, 3121 m_head->m_pkthdr.tso_segsz); 3122 } 3123 #endif 3124 #if defined(INET6) && defined(INET) 3125 else 3126 #endif 3127 #ifdef INET6 3128 { 3129 *pi_data = NDIS_LSO2_INFO_MAKEIPV6( 3130 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen, 3131 m_head->m_pkthdr.tso_segsz); 3132 } 3133 #endif 3134 #endif /* INET6 || INET */ 3135 } else if (m_head->m_pkthdr.csum_flags & txr->hn_csum_assist) { 3136 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3137 NDIS_TXCSUM_INFO_SIZE, NDIS_PKTINFO_TYPE_CSUM); 3138 if (m_head->m_pkthdr.csum_flags & 3139 (CSUM_IP6_TCP | CSUM_IP6_UDP)) { 3140 *pi_data = NDIS_TXCSUM_INFO_IPV6; 3141 } else { 3142 *pi_data = NDIS_TXCSUM_INFO_IPV4; 3143 if (m_head->m_pkthdr.csum_flags & CSUM_IP) 3144 *pi_data |= NDIS_TXCSUM_INFO_IPCS; 3145 } 3146 3147 if (m_head->m_pkthdr.csum_flags & 3148 (CSUM_IP_TCP | CSUM_IP6_TCP)) { 3149 *pi_data |= NDIS_TXCSUM_INFO_MKTCPCS( 3150 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen); 3151 } else if (m_head->m_pkthdr.csum_flags & 3152 (CSUM_IP_UDP | CSUM_IP6_UDP)) { 3153 *pi_data |= NDIS_TXCSUM_INFO_MKUDPCS( 3154 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen); 3155 } 3156 } 3157 3158 pkt_hlen = pkt->rm_pktinfooffset + pkt->rm_pktinfolen; 3159 /* Fixup RNDIS packet message total length */ 3160 pkt->rm_len += pkt_hlen; 3161 /* Convert RNDIS packet message offsets */ 3162 pkt->rm_dataoffset = hn_rndis_pktmsg_offset(pkt_hlen); 3163 pkt->rm_pktinfooffset = hn_rndis_pktmsg_offset(pkt->rm_pktinfooffset); 3164 3165 /* 3166 * Fast path: Chimney sending. 3167 */ 3168 if (chim != NULL) { 3169 struct hn_txdesc *tgt_txd = txd; 3170 3171 if (txr->hn_agg_txd != NULL) { 3172 tgt_txd = txr->hn_agg_txd; 3173 #ifdef INVARIANTS 3174 *m_head0 = NULL; 3175 #endif 3176 } 3177 3178 KASSERT(pkt == chim, 3179 ("RNDIS pkt not in chimney sending buffer")); 3180 KASSERT(tgt_txd->chim_index != HN_NVS_CHIM_IDX_INVALID, 3181 ("chimney sending buffer is not used")); 3182 tgt_txd->chim_size += pkt->rm_len; 3183 3184 m_copydata(m_head, 0, m_head->m_pkthdr.len, 3185 ((uint8_t *)chim) + pkt_hlen); 3186 3187 txr->hn_gpa_cnt = 0; 3188 txr->hn_sendpkt = hn_txpkt_chim; 3189 goto done; 3190 } 3191 3192 KASSERT(txr->hn_agg_txd == NULL, ("aggregating sglist txdesc")); 3193 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID, 3194 ("chimney buffer is used")); 3195 KASSERT(pkt == txd->rndis_pkt, ("RNDIS pkt not in txdesc")); 3196 3197 error = hn_txdesc_dmamap_load(txr, txd, &m_head, segs, &nsegs); 3198 if (__predict_false(error)) { 3199 int freed __diagused; 3200 3201 /* 3202 * This mbuf is not linked w/ the txd yet, so free it now. 3203 */ 3204 m_freem(m_head); 3205 *m_head0 = NULL; 3206 3207 freed = hn_txdesc_put(txr, txd); 3208 KASSERT(freed != 0, 3209 ("fail to free txd upon txdma error")); 3210 3211 txr->hn_txdma_failed++; 3212 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 3213 return error; 3214 } 3215 *m_head0 = m_head; 3216 3217 /* +1 RNDIS packet message */ 3218 txr->hn_gpa_cnt = nsegs + 1; 3219 3220 /* send packet with page buffer */ 3221 txr->hn_gpa[0].gpa_page = atop(txd->rndis_pkt_paddr); 3222 txr->hn_gpa[0].gpa_ofs = txd->rndis_pkt_paddr & PAGE_MASK; 3223 txr->hn_gpa[0].gpa_len = pkt_hlen; 3224 3225 /* 3226 * Fill the page buffers with mbuf info after the page 3227 * buffer for RNDIS packet message. 3228 */ 3229 for (i = 0; i < nsegs; ++i) { 3230 struct vmbus_gpa *gpa = &txr->hn_gpa[i + 1]; 3231 3232 gpa->gpa_page = atop(segs[i].ds_addr); 3233 gpa->gpa_ofs = segs[i].ds_addr & PAGE_MASK; 3234 gpa->gpa_len = segs[i].ds_len; 3235 } 3236 3237 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 3238 txd->chim_size = 0; 3239 txr->hn_sendpkt = hn_txpkt_sglist; 3240 done: 3241 txd->m = m_head; 3242 3243 /* Set the completion routine */ 3244 hn_nvs_sendctx_init(&txd->send_ctx, hn_txpkt_done, txd); 3245 3246 /* Update temporary stats for later use. */ 3247 txr->hn_stat_pkts++; 3248 txr->hn_stat_size += m_head->m_pkthdr.len; 3249 if (m_head->m_flags & M_MCAST) 3250 txr->hn_stat_mcasts++; 3251 3252 return 0; 3253 } 3254 3255 /* 3256 * NOTE: 3257 * If this function fails, then txd will be freed, but the mbuf 3258 * associated w/ the txd will _not_ be freed. 3259 */ 3260 static int 3261 hn_txpkt(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd) 3262 { 3263 int error, send_failed = 0, has_bpf; 3264 3265 again: 3266 has_bpf = bpf_peers_present_if(ifp); 3267 if (has_bpf) { 3268 /* 3269 * Make sure that this txd and any aggregated txds are not 3270 * freed before ETHER_BPF_MTAP. 3271 */ 3272 hn_txdesc_hold(txd); 3273 } 3274 error = txr->hn_sendpkt(txr, txd); 3275 if (!error) { 3276 if (has_bpf) { 3277 const struct hn_txdesc *tmp_txd; 3278 3279 ETHER_BPF_MTAP(ifp, txd->m); 3280 STAILQ_FOREACH(tmp_txd, &txd->agg_list, agg_link) 3281 ETHER_BPF_MTAP(ifp, tmp_txd->m); 3282 } 3283 3284 if_inc_counter(ifp, IFCOUNTER_OPACKETS, txr->hn_stat_pkts); 3285 #ifdef HN_IFSTART_SUPPORT 3286 if (!hn_use_if_start) 3287 #endif 3288 { 3289 if_inc_counter(ifp, IFCOUNTER_OBYTES, 3290 txr->hn_stat_size); 3291 if (txr->hn_stat_mcasts != 0) { 3292 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 3293 txr->hn_stat_mcasts); 3294 } 3295 } 3296 txr->hn_pkts += txr->hn_stat_pkts; 3297 txr->hn_sends++; 3298 } 3299 if (has_bpf) 3300 hn_txdesc_put(txr, txd); 3301 3302 if (__predict_false(error)) { 3303 int freed __diagused; 3304 3305 /* 3306 * This should "really rarely" happen. 3307 * 3308 * XXX Too many RX to be acked or too many sideband 3309 * commands to run? Ask netvsc_channel_rollup() 3310 * to kick start later. 3311 */ 3312 txr->hn_has_txeof = 1; 3313 if (!send_failed) { 3314 txr->hn_send_failed++; 3315 send_failed = 1; 3316 /* 3317 * Try sending again after set hn_has_txeof; 3318 * in case that we missed the last 3319 * netvsc_channel_rollup(). 3320 */ 3321 goto again; 3322 } 3323 if_printf(ifp, "send failed\n"); 3324 3325 /* 3326 * Caller will perform further processing on the 3327 * associated mbuf, so don't free it in hn_txdesc_put(); 3328 * only unload it from the DMA map in hn_txdesc_put(), 3329 * if it was loaded. 3330 */ 3331 txd->m = NULL; 3332 freed = hn_txdesc_put(txr, txd); 3333 KASSERT(freed != 0, 3334 ("fail to free txd upon send error")); 3335 3336 txr->hn_send_failed++; 3337 } 3338 3339 /* Reset temporary stats, after this sending is done. */ 3340 txr->hn_stat_size = 0; 3341 txr->hn_stat_pkts = 0; 3342 txr->hn_stat_mcasts = 0; 3343 3344 return (error); 3345 } 3346 3347 /* 3348 * Append the specified data to the indicated mbuf chain, 3349 * Extend the mbuf chain if the new data does not fit in 3350 * existing space. 3351 * 3352 * This is a minor rewrite of m_append() from sys/kern/uipc_mbuf.c. 3353 * There should be an equivalent in the kernel mbuf code, 3354 * but there does not appear to be one yet. 3355 * 3356 * Differs from m_append() in that additional mbufs are 3357 * allocated with cluster size MJUMPAGESIZE, and filled 3358 * accordingly. 3359 * 3360 * Return the last mbuf in the chain or NULL if failed to 3361 * allocate new mbuf. 3362 */ 3363 static struct mbuf * 3364 hv_m_append(struct mbuf *m0, int len, c_caddr_t cp) 3365 { 3366 struct mbuf *m, *n; 3367 int remainder, space; 3368 3369 for (m = m0; m->m_next != NULL; m = m->m_next) 3370 ; 3371 remainder = len; 3372 space = M_TRAILINGSPACE(m); 3373 if (space > 0) { 3374 /* 3375 * Copy into available space. 3376 */ 3377 if (space > remainder) 3378 space = remainder; 3379 bcopy(cp, mtod(m, caddr_t) + m->m_len, space); 3380 m->m_len += space; 3381 cp += space; 3382 remainder -= space; 3383 } 3384 while (remainder > 0) { 3385 /* 3386 * Allocate a new mbuf; could check space 3387 * and allocate a cluster instead. 3388 */ 3389 n = m_getjcl(M_NOWAIT, m->m_type, 0, MJUMPAGESIZE); 3390 if (n == NULL) 3391 return NULL; 3392 n->m_len = min(MJUMPAGESIZE, remainder); 3393 bcopy(cp, mtod(n, caddr_t), n->m_len); 3394 cp += n->m_len; 3395 remainder -= n->m_len; 3396 m->m_next = n; 3397 m = n; 3398 } 3399 3400 return m; 3401 } 3402 3403 #if defined(INET) || defined(INET6) 3404 static __inline int 3405 hn_lro_rx(struct lro_ctrl *lc, struct mbuf *m) 3406 { 3407 if (hn_lro_mbufq_depth) { 3408 tcp_lro_queue_mbuf(lc, m); 3409 return 0; 3410 } 3411 return tcp_lro_rx(lc, m, 0); 3412 } 3413 #endif 3414 3415 static int 3416 hn_rxpkt(struct hn_rx_ring *rxr) 3417 { 3418 if_t ifp, hn_ifp = rxr->hn_ifp; 3419 struct mbuf *m_new, *n; 3420 int size, do_lro = 0, do_csum = 1, is_vf = 0; 3421 int hash_type = M_HASHTYPE_NONE; 3422 int l3proto = ETHERTYPE_MAX, l4proto = IPPROTO_DONE; 3423 int i; 3424 3425 ifp = hn_ifp; 3426 if (rxr->hn_rxvf_ifp != NULL) { 3427 /* 3428 * Non-transparent mode VF; pretend this packet is from 3429 * the VF. 3430 */ 3431 ifp = rxr->hn_rxvf_ifp; 3432 is_vf = 1; 3433 } else if (rxr->hn_rx_flags & HN_RX_FLAG_XPNT_VF) { 3434 /* Transparent mode VF. */ 3435 is_vf = 1; 3436 } 3437 3438 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 3439 /* 3440 * NOTE: 3441 * See the NOTE of hn_rndis_init_fixat(). This 3442 * function can be reached, immediately after the 3443 * RNDIS is initialized but before the ifnet is 3444 * setup on the hn_attach() path; drop the unexpected 3445 * packets. 3446 */ 3447 return (0); 3448 } 3449 3450 if (__predict_false(rxr->rsc.pktlen < ETHER_HDR_LEN)) { 3451 if_inc_counter(hn_ifp, IFCOUNTER_IERRORS, 1); 3452 return (0); 3453 } 3454 3455 if (rxr->rsc.cnt == 1 && rxr->rsc.pktlen <= MHLEN) { 3456 m_new = m_gethdr(M_NOWAIT, MT_DATA); 3457 if (m_new == NULL) { 3458 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3459 return (0); 3460 } 3461 memcpy(mtod(m_new, void *), rxr->rsc.frag_data[0], 3462 rxr->rsc.frag_len[0]); 3463 m_new->m_pkthdr.len = m_new->m_len = rxr->rsc.frag_len[0]; 3464 } else { 3465 /* 3466 * Get an mbuf with a cluster. For packets 2K or less, 3467 * get a standard 2K cluster. For anything larger, get a 3468 * 4K cluster. Any buffers larger than 4K can cause problems 3469 * if looped around to the Hyper-V TX channel, so avoid them. 3470 */ 3471 size = MCLBYTES; 3472 if (rxr->rsc.pktlen > MCLBYTES) { 3473 /* 4096 */ 3474 size = MJUMPAGESIZE; 3475 } 3476 3477 m_new = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size); 3478 if (m_new == NULL) { 3479 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3480 return (0); 3481 } 3482 3483 n = m_new; 3484 for (i = 0; i < rxr->rsc.cnt; i++) { 3485 n = hv_m_append(n, rxr->rsc.frag_len[i], 3486 rxr->rsc.frag_data[i]); 3487 if (n == NULL) { 3488 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3489 return (0); 3490 } else { 3491 m_new->m_pkthdr.len += rxr->rsc.frag_len[i]; 3492 } 3493 } 3494 } 3495 if (rxr->rsc.pktlen <= MHLEN) 3496 rxr->hn_small_pkts++; 3497 3498 m_new->m_pkthdr.rcvif = ifp; 3499 3500 if (__predict_false((if_getcapenable(hn_ifp) & IFCAP_RXCSUM) == 0)) 3501 do_csum = 0; 3502 3503 /* receive side checksum offload */ 3504 if (rxr->rsc.csum_info != NULL) { 3505 /* IP csum offload */ 3506 if ((*(rxr->rsc.csum_info) & NDIS_RXCSUM_INFO_IPCS_OK) && do_csum) { 3507 m_new->m_pkthdr.csum_flags |= 3508 (CSUM_IP_CHECKED | CSUM_IP_VALID); 3509 rxr->hn_csum_ip++; 3510 } 3511 3512 /* TCP/UDP csum offload */ 3513 if ((*(rxr->rsc.csum_info) & (NDIS_RXCSUM_INFO_UDPCS_OK | 3514 NDIS_RXCSUM_INFO_TCPCS_OK)) && do_csum) { 3515 m_new->m_pkthdr.csum_flags |= 3516 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3517 m_new->m_pkthdr.csum_data = 0xffff; 3518 if (*(rxr->rsc.csum_info) & NDIS_RXCSUM_INFO_TCPCS_OK) 3519 rxr->hn_csum_tcp++; 3520 else 3521 rxr->hn_csum_udp++; 3522 } 3523 3524 /* 3525 * XXX 3526 * As of this write (Oct 28th, 2016), host side will turn 3527 * on only TCPCS_OK and IPCS_OK even for UDP datagrams, so 3528 * the do_lro setting here is actually _not_ accurate. We 3529 * depend on the RSS hash type check to reset do_lro. 3530 */ 3531 if ((*(rxr->rsc.csum_info) & 3532 (NDIS_RXCSUM_INFO_TCPCS_OK | NDIS_RXCSUM_INFO_IPCS_OK)) == 3533 (NDIS_RXCSUM_INFO_TCPCS_OK | NDIS_RXCSUM_INFO_IPCS_OK)) 3534 do_lro = 1; 3535 } else { 3536 hn_rxpkt_proto(m_new, &l3proto, &l4proto); 3537 if (l3proto == ETHERTYPE_IP) { 3538 if (l4proto == IPPROTO_TCP) { 3539 if (do_csum && 3540 (rxr->hn_trust_hcsum & 3541 HN_TRUST_HCSUM_TCP)) { 3542 rxr->hn_csum_trusted++; 3543 m_new->m_pkthdr.csum_flags |= 3544 (CSUM_IP_CHECKED | CSUM_IP_VALID | 3545 CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3546 m_new->m_pkthdr.csum_data = 0xffff; 3547 } 3548 do_lro = 1; 3549 } else if (l4proto == IPPROTO_UDP) { 3550 if (do_csum && 3551 (rxr->hn_trust_hcsum & 3552 HN_TRUST_HCSUM_UDP)) { 3553 rxr->hn_csum_trusted++; 3554 m_new->m_pkthdr.csum_flags |= 3555 (CSUM_IP_CHECKED | CSUM_IP_VALID | 3556 CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3557 m_new->m_pkthdr.csum_data = 0xffff; 3558 } 3559 } else if (l4proto != IPPROTO_DONE && do_csum && 3560 (rxr->hn_trust_hcsum & HN_TRUST_HCSUM_IP)) { 3561 rxr->hn_csum_trusted++; 3562 m_new->m_pkthdr.csum_flags |= 3563 (CSUM_IP_CHECKED | CSUM_IP_VALID); 3564 } 3565 } 3566 } 3567 3568 if (rxr->rsc.vlan_info != NULL) { 3569 m_new->m_pkthdr.ether_vtag = EVL_MAKETAG( 3570 NDIS_VLAN_INFO_ID(*(rxr->rsc.vlan_info)), 3571 NDIS_VLAN_INFO_PRI(*(rxr->rsc.vlan_info)), 3572 NDIS_VLAN_INFO_CFI(*(rxr->rsc.vlan_info))); 3573 m_new->m_flags |= M_VLANTAG; 3574 } 3575 3576 /* 3577 * If VF is activated (tranparent/non-transparent mode does not 3578 * matter here). 3579 * 3580 * - Disable LRO 3581 * 3582 * hn(4) will only receive broadcast packets, multicast packets, 3583 * TCP SYN and SYN|ACK (in Azure), LRO is useless for these 3584 * packet types. 3585 * 3586 * For non-transparent, we definitely _cannot_ enable LRO at 3587 * all, since the LRO flush will use hn(4) as the receiving 3588 * interface; i.e. hn_ifp->if_input(hn_ifp, m). 3589 */ 3590 if (is_vf) 3591 do_lro = 0; 3592 3593 /* 3594 * If VF is activated (tranparent/non-transparent mode does not 3595 * matter here), do _not_ mess with unsupported hash types or 3596 * functions. 3597 */ 3598 if (rxr->rsc.hash_info != NULL) { 3599 rxr->hn_rss_pkts++; 3600 m_new->m_pkthdr.flowid = *(rxr->rsc.hash_value); 3601 if (!is_vf) 3602 hash_type = M_HASHTYPE_OPAQUE_HASH; 3603 if ((*(rxr->rsc.hash_info) & NDIS_HASH_FUNCTION_MASK) == 3604 NDIS_HASH_FUNCTION_TOEPLITZ) { 3605 uint32_t type = (*(rxr->rsc.hash_info) & NDIS_HASH_TYPE_MASK & 3606 rxr->hn_mbuf_hash); 3607 3608 /* 3609 * NOTE: 3610 * do_lro is resetted, if the hash types are not TCP 3611 * related. See the comment in the above csum_flags 3612 * setup section. 3613 */ 3614 switch (type) { 3615 case NDIS_HASH_IPV4: 3616 hash_type = M_HASHTYPE_RSS_IPV4; 3617 do_lro = 0; 3618 break; 3619 3620 case NDIS_HASH_TCP_IPV4: 3621 hash_type = M_HASHTYPE_RSS_TCP_IPV4; 3622 if (rxr->hn_rx_flags & HN_RX_FLAG_UDP_HASH) { 3623 int def_htype = M_HASHTYPE_OPAQUE_HASH; 3624 3625 if (is_vf) 3626 def_htype = M_HASHTYPE_NONE; 3627 3628 /* 3629 * UDP 4-tuple hash is delivered as 3630 * TCP 4-tuple hash. 3631 */ 3632 if (l3proto == ETHERTYPE_MAX) { 3633 hn_rxpkt_proto(m_new, 3634 &l3proto, &l4proto); 3635 } 3636 if (l3proto == ETHERTYPE_IP) { 3637 if (l4proto == IPPROTO_UDP && 3638 (rxr->hn_mbuf_hash & 3639 NDIS_HASH_UDP_IPV4_X)) { 3640 hash_type = 3641 M_HASHTYPE_RSS_UDP_IPV4; 3642 do_lro = 0; 3643 } else if (l4proto != 3644 IPPROTO_TCP) { 3645 hash_type = def_htype; 3646 do_lro = 0; 3647 } 3648 } else { 3649 hash_type = def_htype; 3650 do_lro = 0; 3651 } 3652 } 3653 break; 3654 3655 case NDIS_HASH_IPV6: 3656 hash_type = M_HASHTYPE_RSS_IPV6; 3657 do_lro = 0; 3658 break; 3659 3660 case NDIS_HASH_IPV6_EX: 3661 hash_type = M_HASHTYPE_RSS_IPV6_EX; 3662 do_lro = 0; 3663 break; 3664 3665 case NDIS_HASH_TCP_IPV6: 3666 hash_type = M_HASHTYPE_RSS_TCP_IPV6; 3667 break; 3668 3669 case NDIS_HASH_TCP_IPV6_EX: 3670 hash_type = M_HASHTYPE_RSS_TCP_IPV6_EX; 3671 break; 3672 } 3673 } 3674 } else if (!is_vf) { 3675 m_new->m_pkthdr.flowid = rxr->hn_rx_idx; 3676 hash_type = M_HASHTYPE_OPAQUE; 3677 } 3678 M_HASHTYPE_SET(m_new, hash_type); 3679 3680 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 3681 if (hn_ifp != ifp) { 3682 const struct ether_header *eh; 3683 3684 /* 3685 * Non-transparent mode VF is activated. 3686 */ 3687 3688 /* 3689 * Allow tapping on hn(4). 3690 */ 3691 ETHER_BPF_MTAP(hn_ifp, m_new); 3692 3693 /* 3694 * Update hn(4)'s stats. 3695 */ 3696 if_inc_counter(hn_ifp, IFCOUNTER_IPACKETS, 1); 3697 if_inc_counter(hn_ifp, IFCOUNTER_IBYTES, m_new->m_pkthdr.len); 3698 /* Checked at the beginning of this function. */ 3699 KASSERT(m_new->m_len >= ETHER_HDR_LEN, ("not ethernet frame")); 3700 eh = mtod(m_new, struct ether_header *); 3701 if (ETHER_IS_MULTICAST(eh->ether_dhost)) 3702 if_inc_counter(hn_ifp, IFCOUNTER_IMCASTS, 1); 3703 } 3704 rxr->hn_pkts++; 3705 3706 if ((if_getcapenable(hn_ifp) & IFCAP_LRO) && do_lro) { 3707 #if defined(INET) || defined(INET6) 3708 struct lro_ctrl *lro = &rxr->hn_lro; 3709 3710 if (lro->lro_cnt) { 3711 rxr->hn_lro_tried++; 3712 if (hn_lro_rx(lro, m_new) == 0) { 3713 /* DONE! */ 3714 return 0; 3715 } 3716 } 3717 #endif 3718 } 3719 if_input(ifp, m_new); 3720 3721 return (0); 3722 } 3723 3724 static int 3725 hn_ioctl(if_t ifp, u_long cmd, caddr_t data) 3726 { 3727 struct hn_softc *sc = if_getsoftc(ifp); 3728 struct ifreq *ifr = (struct ifreq *)data, ifr_vf; 3729 if_t vf_ifp; 3730 int mask, error = 0; 3731 struct ifrsskey *ifrk; 3732 struct ifrsshash *ifrh; 3733 uint32_t mtu; 3734 3735 switch (cmd) { 3736 case SIOCSIFMTU: 3737 if (ifr->ifr_mtu > HN_MTU_MAX) { 3738 error = EINVAL; 3739 break; 3740 } 3741 3742 HN_LOCK(sc); 3743 3744 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 3745 HN_UNLOCK(sc); 3746 break; 3747 } 3748 3749 if ((sc->hn_caps & HN_CAP_MTU) == 0) { 3750 /* Can't change MTU */ 3751 HN_UNLOCK(sc); 3752 error = EOPNOTSUPP; 3753 break; 3754 } 3755 3756 if (if_getmtu(ifp) == ifr->ifr_mtu) { 3757 HN_UNLOCK(sc); 3758 break; 3759 } 3760 3761 if (hn_xpnt_vf_isready(sc)) { 3762 vf_ifp = sc->hn_vf_ifp; 3763 ifr_vf = *ifr; 3764 strlcpy(ifr_vf.ifr_name, if_name(vf_ifp), 3765 sizeof(ifr_vf.ifr_name)); 3766 error = ifhwioctl(SIOCSIFMTU,vf_ifp, 3767 (caddr_t)&ifr_vf, curthread); 3768 if (error) { 3769 HN_UNLOCK(sc); 3770 if_printf(ifp, "%s SIOCSIFMTU %d failed: %d\n", 3771 if_name(vf_ifp), ifr->ifr_mtu, error); 3772 break; 3773 } 3774 } 3775 3776 /* 3777 * Suspend this interface before the synthetic parts 3778 * are ripped. 3779 */ 3780 hn_suspend(sc); 3781 3782 /* 3783 * Detach the synthetics parts, i.e. NVS and RNDIS. 3784 */ 3785 hn_synth_detach(sc); 3786 3787 /* 3788 * Reattach the synthetic parts, i.e. NVS and RNDIS, 3789 * with the new MTU setting. 3790 */ 3791 error = hn_synth_attach(sc, ifr->ifr_mtu); 3792 if (error) { 3793 HN_UNLOCK(sc); 3794 break; 3795 } 3796 3797 error = hn_rndis_get_mtu(sc, &mtu); 3798 if (error) 3799 mtu = ifr->ifr_mtu; 3800 else if (bootverbose) 3801 if_printf(ifp, "RNDIS mtu %u\n", mtu); 3802 3803 /* 3804 * Commit the requested MTU, after the synthetic parts 3805 * have been successfully attached. 3806 */ 3807 if (mtu >= ifr->ifr_mtu) { 3808 mtu = ifr->ifr_mtu; 3809 } else { 3810 if_printf(ifp, "fixup mtu %d -> %u\n", 3811 ifr->ifr_mtu, mtu); 3812 } 3813 if_setmtu(ifp, mtu); 3814 3815 /* 3816 * Synthetic parts' reattach may change the chimney 3817 * sending size; update it. 3818 */ 3819 if (sc->hn_tx_ring[0].hn_chim_size > sc->hn_chim_szmax) 3820 hn_set_chim_size(sc, sc->hn_chim_szmax); 3821 3822 /* 3823 * Make sure that various parameters based on MTU are 3824 * still valid, after the MTU change. 3825 */ 3826 hn_mtu_change_fixup(sc); 3827 3828 /* 3829 * All done! Resume the interface now. 3830 */ 3831 hn_resume(sc); 3832 3833 if ((sc->hn_flags & HN_FLAG_RXVF) || 3834 (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 3835 /* 3836 * Since we have reattached the NVS part, 3837 * change the datapath to VF again; in case 3838 * that it is lost, after the NVS was detached. 3839 */ 3840 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_VF); 3841 } 3842 3843 HN_UNLOCK(sc); 3844 break; 3845 3846 case SIOCSIFFLAGS: 3847 HN_LOCK(sc); 3848 3849 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 3850 HN_UNLOCK(sc); 3851 break; 3852 } 3853 3854 if (hn_xpnt_vf_isready(sc)) 3855 hn_xpnt_vf_saveifflags(sc); 3856 3857 if (if_getflags(ifp) & IFF_UP) { 3858 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 3859 /* 3860 * Caller meight hold mutex, e.g. 3861 * bpf; use busy-wait for the RNDIS 3862 * reply. 3863 */ 3864 HN_NO_SLEEPING(sc); 3865 hn_rxfilter_config(sc); 3866 HN_SLEEPING_OK(sc); 3867 3868 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 3869 error = hn_xpnt_vf_iocsetflags(sc); 3870 } else { 3871 hn_init_locked(sc); 3872 } 3873 } else { 3874 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 3875 hn_stop(sc, false); 3876 } 3877 sc->hn_if_flags = if_getflags(ifp); 3878 3879 HN_UNLOCK(sc); 3880 break; 3881 3882 case SIOCSIFCAP: 3883 HN_LOCK(sc); 3884 3885 if (hn_xpnt_vf_isready(sc)) { 3886 ifr_vf = *ifr; 3887 strlcpy(ifr_vf.ifr_name, if_name(sc->hn_vf_ifp), 3888 sizeof(ifr_vf.ifr_name)); 3889 error = hn_xpnt_vf_iocsetcaps(sc, &ifr_vf); 3890 HN_UNLOCK(sc); 3891 break; 3892 } 3893 3894 /* 3895 * Fix up requested capabilities w/ supported capabilities, 3896 * since the supported capabilities could have been changed. 3897 */ 3898 mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ 3899 if_getcapenable(ifp); 3900 3901 if (mask & IFCAP_TXCSUM) { 3902 if_togglecapenable(ifp, IFCAP_TXCSUM); 3903 if (if_getcapenable(ifp) & IFCAP_TXCSUM) 3904 if_sethwassistbits(ifp, HN_CSUM_IP_HWASSIST(sc), 0); 3905 else 3906 if_sethwassistbits(ifp, 0, HN_CSUM_IP_HWASSIST(sc)); 3907 } 3908 if (mask & IFCAP_TXCSUM_IPV6) { 3909 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 3910 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) 3911 if_sethwassistbits(ifp, HN_CSUM_IP6_HWASSIST(sc), 0); 3912 else 3913 if_sethwassistbits(ifp, 0, HN_CSUM_IP6_HWASSIST(sc)); 3914 } 3915 3916 /* TODO: flip RNDIS offload parameters for RXCSUM. */ 3917 if (mask & IFCAP_RXCSUM) 3918 if_togglecapenable(ifp, IFCAP_RXCSUM); 3919 #ifdef foo 3920 /* We can't diff IPv6 packets from IPv4 packets on RX path. */ 3921 if (mask & IFCAP_RXCSUM_IPV6) 3922 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 3923 #endif 3924 3925 if (mask & IFCAP_LRO) 3926 if_togglecapenable(ifp, IFCAP_LRO); 3927 3928 if (mask & IFCAP_TSO4) { 3929 if_togglecapenable(ifp, IFCAP_TSO4); 3930 if (if_getcapenable(ifp) & IFCAP_TSO4) 3931 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 3932 else 3933 if_sethwassistbits(ifp, 0, CSUM_IP_TSO); 3934 } 3935 if (mask & IFCAP_TSO6) { 3936 if_togglecapenable(ifp, IFCAP_TSO6); 3937 if (if_getcapenable(ifp) & IFCAP_TSO6) 3938 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 3939 else 3940 if_sethwassistbits(ifp, 0, CSUM_IP6_TSO); 3941 } 3942 3943 HN_UNLOCK(sc); 3944 break; 3945 3946 case SIOCADDMULTI: 3947 case SIOCDELMULTI: 3948 HN_LOCK(sc); 3949 3950 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 3951 HN_UNLOCK(sc); 3952 break; 3953 } 3954 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 3955 /* 3956 * Multicast uses mutex; use busy-wait for 3957 * the RNDIS reply. 3958 */ 3959 HN_NO_SLEEPING(sc); 3960 hn_rxfilter_config(sc); 3961 HN_SLEEPING_OK(sc); 3962 } 3963 3964 /* XXX vlan(4) style mcast addr maintenance */ 3965 if (hn_xpnt_vf_isready(sc)) { 3966 int old_if_flags; 3967 3968 old_if_flags = if_getflags(sc->hn_vf_ifp); 3969 hn_xpnt_vf_saveifflags(sc); 3970 3971 if ((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) && 3972 ((old_if_flags ^ if_getflags(sc->hn_vf_ifp)) & 3973 IFF_ALLMULTI)) 3974 error = hn_xpnt_vf_iocsetflags(sc); 3975 } 3976 3977 HN_UNLOCK(sc); 3978 break; 3979 3980 case SIOCSIFMEDIA: 3981 case SIOCGIFMEDIA: 3982 HN_LOCK(sc); 3983 if (hn_xpnt_vf_isready(sc)) { 3984 /* 3985 * SIOCGIFMEDIA expects ifmediareq, so don't 3986 * create and pass ifr_vf to the VF here; just 3987 * replace the ifr_name. 3988 */ 3989 vf_ifp = sc->hn_vf_ifp; 3990 strlcpy(ifr->ifr_name, if_name(vf_ifp), 3991 sizeof(ifr->ifr_name)); 3992 error = ifhwioctl(cmd, vf_ifp, data, curthread); 3993 /* Restore the ifr_name. */ 3994 strlcpy(ifr->ifr_name, if_name(ifp), 3995 sizeof(ifr->ifr_name)); 3996 HN_UNLOCK(sc); 3997 break; 3998 } 3999 HN_UNLOCK(sc); 4000 error = ifmedia_ioctl(ifp, ifr, &sc->hn_media, cmd); 4001 break; 4002 4003 case SIOCGIFRSSHASH: 4004 ifrh = (struct ifrsshash *)data; 4005 HN_LOCK(sc); 4006 if (sc->hn_rx_ring_inuse == 1) { 4007 HN_UNLOCK(sc); 4008 ifrh->ifrh_func = RSS_FUNC_NONE; 4009 ifrh->ifrh_types = 0; 4010 break; 4011 } 4012 4013 if (sc->hn_rss_hash & NDIS_HASH_FUNCTION_TOEPLITZ) 4014 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ; 4015 else 4016 ifrh->ifrh_func = RSS_FUNC_PRIVATE; 4017 ifrh->ifrh_types = hn_rss_type_fromndis(sc->hn_rss_hash); 4018 HN_UNLOCK(sc); 4019 break; 4020 4021 case SIOCGIFRSSKEY: 4022 ifrk = (struct ifrsskey *)data; 4023 HN_LOCK(sc); 4024 if (sc->hn_rx_ring_inuse == 1) { 4025 HN_UNLOCK(sc); 4026 ifrk->ifrk_func = RSS_FUNC_NONE; 4027 ifrk->ifrk_keylen = 0; 4028 break; 4029 } 4030 if (sc->hn_rss_hash & NDIS_HASH_FUNCTION_TOEPLITZ) 4031 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ; 4032 else 4033 ifrk->ifrk_func = RSS_FUNC_PRIVATE; 4034 ifrk->ifrk_keylen = NDIS_HASH_KEYSIZE_TOEPLITZ; 4035 memcpy(ifrk->ifrk_key, sc->hn_rss.rss_key, 4036 NDIS_HASH_KEYSIZE_TOEPLITZ); 4037 HN_UNLOCK(sc); 4038 break; 4039 4040 default: 4041 error = ether_ioctl(ifp, cmd, data); 4042 break; 4043 } 4044 return (error); 4045 } 4046 4047 static void 4048 hn_stop(struct hn_softc *sc, bool detaching) 4049 { 4050 if_t ifp = sc->hn_ifp; 4051 int i; 4052 4053 HN_LOCK_ASSERT(sc); 4054 4055 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 4056 ("synthetic parts were not attached")); 4057 4058 /* Clear RUNNING bit ASAP. */ 4059 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 4060 4061 /* Disable polling. */ 4062 hn_polling(sc, 0); 4063 4064 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) { 4065 KASSERT(sc->hn_vf_ifp != NULL, 4066 ("%s: VF is not attached", if_name(ifp))); 4067 4068 /* Mark transparent mode VF as disabled. */ 4069 hn_xpnt_vf_setdisable(sc, false /* keep hn_vf_ifp */); 4070 4071 /* 4072 * NOTE: 4073 * Datapath setting must happen _before_ bringing 4074 * the VF down. 4075 */ 4076 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_SYNTH); 4077 4078 /* 4079 * Bring the VF down. 4080 */ 4081 hn_xpnt_vf_saveifflags(sc); 4082 if_setflagbits(ifp, 0, IFF_UP); 4083 hn_xpnt_vf_iocsetflags(sc); 4084 } 4085 4086 /* Suspend data transfers. */ 4087 hn_suspend_data(sc); 4088 4089 /* Clear OACTIVE bit. */ 4090 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 4091 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 4092 sc->hn_tx_ring[i].hn_oactive = 0; 4093 4094 /* 4095 * If the non-transparent mode VF is active, make sure 4096 * that the RX filter still allows packet reception. 4097 */ 4098 if (!detaching && (sc->hn_flags & HN_FLAG_RXVF)) 4099 hn_rxfilter_config(sc); 4100 } 4101 4102 static void 4103 hn_init_locked(struct hn_softc *sc) 4104 { 4105 if_t ifp = sc->hn_ifp; 4106 int i; 4107 4108 HN_LOCK_ASSERT(sc); 4109 4110 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 4111 return; 4112 4113 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 4114 return; 4115 4116 /* Configure RX filter */ 4117 hn_rxfilter_config(sc); 4118 4119 /* Clear OACTIVE bit. */ 4120 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 4121 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 4122 sc->hn_tx_ring[i].hn_oactive = 0; 4123 4124 /* Clear TX 'suspended' bit. */ 4125 hn_resume_tx(sc, sc->hn_tx_ring_inuse); 4126 4127 if (hn_xpnt_vf_isready(sc)) { 4128 /* Initialize transparent VF. */ 4129 hn_xpnt_vf_init(sc); 4130 } 4131 4132 /* Everything is ready; unleash! */ 4133 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 4134 4135 /* Re-enable polling if requested. */ 4136 if (sc->hn_pollhz > 0) 4137 hn_polling(sc, sc->hn_pollhz); 4138 } 4139 4140 static void 4141 hn_init(void *xsc) 4142 { 4143 struct hn_softc *sc = xsc; 4144 4145 HN_LOCK(sc); 4146 hn_init_locked(sc); 4147 HN_UNLOCK(sc); 4148 } 4149 4150 static int 4151 hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS) 4152 { 4153 struct hn_softc *sc = arg1; 4154 unsigned int lenlim; 4155 int error; 4156 4157 lenlim = sc->hn_rx_ring[0].hn_lro.lro_length_lim; 4158 error = sysctl_handle_int(oidp, &lenlim, 0, req); 4159 if (error || req->newptr == NULL) 4160 return error; 4161 4162 HN_LOCK(sc); 4163 if (lenlim < HN_LRO_LENLIM_MIN(sc->hn_ifp) || 4164 lenlim > TCP_LRO_LENGTH_MAX) { 4165 HN_UNLOCK(sc); 4166 return EINVAL; 4167 } 4168 hn_set_lro_lenlim(sc, lenlim); 4169 HN_UNLOCK(sc); 4170 4171 return 0; 4172 } 4173 4174 static int 4175 hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS) 4176 { 4177 struct hn_softc *sc = arg1; 4178 int ackcnt, error, i; 4179 4180 /* 4181 * lro_ackcnt_lim is append count limit, 4182 * +1 to turn it into aggregation limit. 4183 */ 4184 ackcnt = sc->hn_rx_ring[0].hn_lro.lro_ackcnt_lim + 1; 4185 error = sysctl_handle_int(oidp, &ackcnt, 0, req); 4186 if (error || req->newptr == NULL) 4187 return error; 4188 4189 if (ackcnt < 2 || ackcnt > (TCP_LRO_ACKCNT_MAX + 1)) 4190 return EINVAL; 4191 4192 /* 4193 * Convert aggregation limit back to append 4194 * count limit. 4195 */ 4196 --ackcnt; 4197 HN_LOCK(sc); 4198 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 4199 sc->hn_rx_ring[i].hn_lro.lro_ackcnt_lim = ackcnt; 4200 HN_UNLOCK(sc); 4201 return 0; 4202 } 4203 4204 static int 4205 hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS) 4206 { 4207 struct hn_softc *sc = arg1; 4208 int hcsum = arg2; 4209 int on, error, i; 4210 4211 on = 0; 4212 if (sc->hn_rx_ring[0].hn_trust_hcsum & hcsum) 4213 on = 1; 4214 4215 error = sysctl_handle_int(oidp, &on, 0, req); 4216 if (error || req->newptr == NULL) 4217 return error; 4218 4219 HN_LOCK(sc); 4220 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4221 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 4222 4223 if (on) 4224 rxr->hn_trust_hcsum |= hcsum; 4225 else 4226 rxr->hn_trust_hcsum &= ~hcsum; 4227 } 4228 HN_UNLOCK(sc); 4229 return 0; 4230 } 4231 4232 static int 4233 hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS) 4234 { 4235 struct hn_softc *sc = arg1; 4236 int chim_size, error; 4237 4238 chim_size = sc->hn_tx_ring[0].hn_chim_size; 4239 error = sysctl_handle_int(oidp, &chim_size, 0, req); 4240 if (error || req->newptr == NULL) 4241 return error; 4242 4243 if (chim_size > sc->hn_chim_szmax || chim_size <= 0) 4244 return EINVAL; 4245 4246 HN_LOCK(sc); 4247 hn_set_chim_size(sc, chim_size); 4248 HN_UNLOCK(sc); 4249 return 0; 4250 } 4251 4252 static int 4253 hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS) 4254 { 4255 struct hn_softc *sc = arg1; 4256 int ofs = arg2, i, error; 4257 struct hn_rx_ring *rxr; 4258 uint64_t stat; 4259 4260 stat = 0; 4261 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4262 rxr = &sc->hn_rx_ring[i]; 4263 stat += *((uint64_t *)((uint8_t *)rxr + ofs)); 4264 } 4265 4266 error = sysctl_handle_64(oidp, &stat, 0, req); 4267 if (error || req->newptr == NULL) 4268 return error; 4269 4270 /* Zero out this stat. */ 4271 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4272 rxr = &sc->hn_rx_ring[i]; 4273 *((uint64_t *)((uint8_t *)rxr + ofs)) = 0; 4274 } 4275 return 0; 4276 } 4277 4278 static int 4279 hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) 4280 { 4281 struct hn_softc *sc = arg1; 4282 int ofs = arg2, i, error; 4283 struct hn_rx_ring *rxr; 4284 u_long stat; 4285 4286 stat = 0; 4287 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4288 rxr = &sc->hn_rx_ring[i]; 4289 stat += *((u_long *)((uint8_t *)rxr + ofs)); 4290 } 4291 4292 error = sysctl_handle_long(oidp, &stat, 0, req); 4293 if (error || req->newptr == NULL) 4294 return error; 4295 4296 /* Zero out this stat. */ 4297 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4298 rxr = &sc->hn_rx_ring[i]; 4299 *((u_long *)((uint8_t *)rxr + ofs)) = 0; 4300 } 4301 return 0; 4302 } 4303 4304 static int 4305 hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) 4306 { 4307 struct hn_softc *sc = arg1; 4308 int ofs = arg2, i, error; 4309 struct hn_tx_ring *txr; 4310 u_long stat; 4311 4312 stat = 0; 4313 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4314 txr = &sc->hn_tx_ring[i]; 4315 stat += *((u_long *)((uint8_t *)txr + ofs)); 4316 } 4317 4318 error = sysctl_handle_long(oidp, &stat, 0, req); 4319 if (error || req->newptr == NULL) 4320 return error; 4321 4322 /* Zero out this stat. */ 4323 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4324 txr = &sc->hn_tx_ring[i]; 4325 *((u_long *)((uint8_t *)txr + ofs)) = 0; 4326 } 4327 return 0; 4328 } 4329 4330 static int 4331 hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS) 4332 { 4333 struct hn_softc *sc = arg1; 4334 int ofs = arg2, i, error, conf; 4335 struct hn_tx_ring *txr; 4336 4337 txr = &sc->hn_tx_ring[0]; 4338 conf = *((int *)((uint8_t *)txr + ofs)); 4339 4340 error = sysctl_handle_int(oidp, &conf, 0, req); 4341 if (error || req->newptr == NULL) 4342 return error; 4343 4344 HN_LOCK(sc); 4345 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4346 txr = &sc->hn_tx_ring[i]; 4347 *((int *)((uint8_t *)txr + ofs)) = conf; 4348 } 4349 HN_UNLOCK(sc); 4350 4351 return 0; 4352 } 4353 4354 static int 4355 hn_txagg_size_sysctl(SYSCTL_HANDLER_ARGS) 4356 { 4357 struct hn_softc *sc = arg1; 4358 int error, size; 4359 4360 size = sc->hn_agg_size; 4361 error = sysctl_handle_int(oidp, &size, 0, req); 4362 if (error || req->newptr == NULL) 4363 return (error); 4364 4365 HN_LOCK(sc); 4366 sc->hn_agg_size = size; 4367 hn_set_txagg(sc); 4368 HN_UNLOCK(sc); 4369 4370 return (0); 4371 } 4372 4373 static int 4374 hn_txagg_pkts_sysctl(SYSCTL_HANDLER_ARGS) 4375 { 4376 struct hn_softc *sc = arg1; 4377 int error, pkts; 4378 4379 pkts = sc->hn_agg_pkts; 4380 error = sysctl_handle_int(oidp, &pkts, 0, req); 4381 if (error || req->newptr == NULL) 4382 return (error); 4383 4384 HN_LOCK(sc); 4385 sc->hn_agg_pkts = pkts; 4386 hn_set_txagg(sc); 4387 HN_UNLOCK(sc); 4388 4389 return (0); 4390 } 4391 4392 static int 4393 hn_txagg_pktmax_sysctl(SYSCTL_HANDLER_ARGS) 4394 { 4395 struct hn_softc *sc = arg1; 4396 int pkts; 4397 4398 pkts = sc->hn_tx_ring[0].hn_agg_pktmax; 4399 return (sysctl_handle_int(oidp, &pkts, 0, req)); 4400 } 4401 4402 static int 4403 hn_txagg_align_sysctl(SYSCTL_HANDLER_ARGS) 4404 { 4405 struct hn_softc *sc = arg1; 4406 int align; 4407 4408 align = sc->hn_tx_ring[0].hn_agg_align; 4409 return (sysctl_handle_int(oidp, &align, 0, req)); 4410 } 4411 4412 static void 4413 hn_chan_polling(struct vmbus_channel *chan, u_int pollhz) 4414 { 4415 if (pollhz == 0) 4416 vmbus_chan_poll_disable(chan); 4417 else 4418 vmbus_chan_poll_enable(chan, pollhz); 4419 } 4420 4421 static void 4422 hn_polling(struct hn_softc *sc, u_int pollhz) 4423 { 4424 int nsubch = sc->hn_rx_ring_inuse - 1; 4425 4426 HN_LOCK_ASSERT(sc); 4427 4428 if (nsubch > 0) { 4429 struct vmbus_channel **subch; 4430 int i; 4431 4432 subch = vmbus_subchan_get(sc->hn_prichan, nsubch); 4433 for (i = 0; i < nsubch; ++i) 4434 hn_chan_polling(subch[i], pollhz); 4435 vmbus_subchan_rel(subch, nsubch); 4436 } 4437 hn_chan_polling(sc->hn_prichan, pollhz); 4438 } 4439 4440 static int 4441 hn_polling_sysctl(SYSCTL_HANDLER_ARGS) 4442 { 4443 struct hn_softc *sc = arg1; 4444 int pollhz, error; 4445 4446 pollhz = sc->hn_pollhz; 4447 error = sysctl_handle_int(oidp, &pollhz, 0, req); 4448 if (error || req->newptr == NULL) 4449 return (error); 4450 4451 if (pollhz != 0 && 4452 (pollhz < VMBUS_CHAN_POLLHZ_MIN || pollhz > VMBUS_CHAN_POLLHZ_MAX)) 4453 return (EINVAL); 4454 4455 HN_LOCK(sc); 4456 if (sc->hn_pollhz != pollhz) { 4457 sc->hn_pollhz = pollhz; 4458 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) && 4459 (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 4460 hn_polling(sc, sc->hn_pollhz); 4461 } 4462 HN_UNLOCK(sc); 4463 4464 return (0); 4465 } 4466 4467 static int 4468 hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS) 4469 { 4470 struct hn_softc *sc = arg1; 4471 char verstr[16]; 4472 4473 snprintf(verstr, sizeof(verstr), "%u.%u", 4474 HN_NDIS_VERSION_MAJOR(sc->hn_ndis_ver), 4475 HN_NDIS_VERSION_MINOR(sc->hn_ndis_ver)); 4476 return sysctl_handle_string(oidp, verstr, sizeof(verstr), req); 4477 } 4478 4479 static int 4480 hn_caps_sysctl(SYSCTL_HANDLER_ARGS) 4481 { 4482 struct hn_softc *sc = arg1; 4483 char caps_str[128]; 4484 uint32_t caps; 4485 4486 HN_LOCK(sc); 4487 caps = sc->hn_caps; 4488 HN_UNLOCK(sc); 4489 snprintf(caps_str, sizeof(caps_str), "%b", caps, HN_CAP_BITS); 4490 return sysctl_handle_string(oidp, caps_str, sizeof(caps_str), req); 4491 } 4492 4493 static int 4494 hn_hwassist_sysctl(SYSCTL_HANDLER_ARGS) 4495 { 4496 struct hn_softc *sc = arg1; 4497 char assist_str[128]; 4498 uint32_t hwassist; 4499 4500 HN_LOCK(sc); 4501 hwassist = if_gethwassist(sc->hn_ifp); 4502 HN_UNLOCK(sc); 4503 snprintf(assist_str, sizeof(assist_str), "%b", hwassist, CSUM_BITS); 4504 return sysctl_handle_string(oidp, assist_str, sizeof(assist_str), req); 4505 } 4506 4507 static int 4508 hn_rxfilter_sysctl(SYSCTL_HANDLER_ARGS) 4509 { 4510 struct hn_softc *sc = arg1; 4511 char filter_str[128]; 4512 uint32_t filter; 4513 4514 HN_LOCK(sc); 4515 filter = sc->hn_rx_filter; 4516 HN_UNLOCK(sc); 4517 snprintf(filter_str, sizeof(filter_str), "%b", filter, 4518 NDIS_PACKET_TYPES); 4519 return sysctl_handle_string(oidp, filter_str, sizeof(filter_str), req); 4520 } 4521 4522 static int 4523 hn_rsc_sysctl(SYSCTL_HANDLER_ARGS) 4524 { 4525 struct hn_softc *sc = arg1; 4526 uint32_t mtu; 4527 int error; 4528 HN_LOCK(sc); 4529 error = hn_rndis_get_mtu(sc, &mtu); 4530 if (error) { 4531 if_printf(sc->hn_ifp, "failed to get mtu\n"); 4532 goto back; 4533 } 4534 error = SYSCTL_OUT(req, &(sc->hn_rsc_ctrl), sizeof(sc->hn_rsc_ctrl)); 4535 if (error || req->newptr == NULL) 4536 goto back; 4537 4538 error = SYSCTL_IN(req, &(sc->hn_rsc_ctrl), sizeof(sc->hn_rsc_ctrl)); 4539 if (error) 4540 goto back; 4541 error = hn_rndis_reconf_offload(sc, mtu); 4542 back: 4543 HN_UNLOCK(sc); 4544 return (error); 4545 } 4546 #ifndef RSS 4547 4548 static int 4549 hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS) 4550 { 4551 struct hn_softc *sc = arg1; 4552 int error; 4553 4554 HN_LOCK(sc); 4555 4556 error = SYSCTL_OUT(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key)); 4557 if (error || req->newptr == NULL) 4558 goto back; 4559 4560 if ((sc->hn_flags & HN_FLAG_RXVF) || 4561 (hn_xpnt_vf && sc->hn_vf_ifp != NULL)) { 4562 /* 4563 * RSS key is synchronized w/ VF's, don't allow users 4564 * to change it. 4565 */ 4566 error = EBUSY; 4567 goto back; 4568 } 4569 4570 error = SYSCTL_IN(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key)); 4571 if (error) 4572 goto back; 4573 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 4574 4575 if (sc->hn_rx_ring_inuse > 1) { 4576 error = hn_rss_reconfig(sc); 4577 } else { 4578 /* Not RSS capable, at least for now; just save the RSS key. */ 4579 error = 0; 4580 } 4581 back: 4582 HN_UNLOCK(sc); 4583 return (error); 4584 } 4585 4586 static int 4587 hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS) 4588 { 4589 struct hn_softc *sc = arg1; 4590 int error; 4591 4592 HN_LOCK(sc); 4593 4594 error = SYSCTL_OUT(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind)); 4595 if (error || req->newptr == NULL) 4596 goto back; 4597 4598 /* 4599 * Don't allow RSS indirect table change, if this interface is not 4600 * RSS capable currently. 4601 */ 4602 if (sc->hn_rx_ring_inuse == 1) { 4603 error = EOPNOTSUPP; 4604 goto back; 4605 } 4606 4607 error = SYSCTL_IN(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind)); 4608 if (error) 4609 goto back; 4610 sc->hn_flags |= HN_FLAG_HAS_RSSIND; 4611 4612 hn_rss_ind_fixup(sc); 4613 error = hn_rss_reconfig(sc); 4614 back: 4615 HN_UNLOCK(sc); 4616 return (error); 4617 } 4618 4619 #endif /* !RSS */ 4620 4621 static int 4622 hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS) 4623 { 4624 struct hn_softc *sc = arg1; 4625 char hash_str[128]; 4626 uint32_t hash; 4627 4628 HN_LOCK(sc); 4629 hash = sc->hn_rss_hash; 4630 HN_UNLOCK(sc); 4631 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4632 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4633 } 4634 4635 static int 4636 hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS) 4637 { 4638 struct hn_softc *sc = arg1; 4639 char hash_str[128]; 4640 uint32_t hash; 4641 4642 HN_LOCK(sc); 4643 hash = sc->hn_rss_hcap; 4644 HN_UNLOCK(sc); 4645 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4646 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4647 } 4648 4649 static int 4650 hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS) 4651 { 4652 struct hn_softc *sc = arg1; 4653 char hash_str[128]; 4654 uint32_t hash; 4655 4656 HN_LOCK(sc); 4657 hash = sc->hn_rx_ring[0].hn_mbuf_hash; 4658 HN_UNLOCK(sc); 4659 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4660 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4661 } 4662 4663 static int 4664 hn_vf_sysctl(SYSCTL_HANDLER_ARGS) 4665 { 4666 struct hn_softc *sc = arg1; 4667 char vf_name[IFNAMSIZ + 1]; 4668 if_t vf_ifp; 4669 4670 HN_LOCK(sc); 4671 vf_name[0] = '\0'; 4672 vf_ifp = sc->hn_vf_ifp; 4673 if (vf_ifp != NULL) 4674 snprintf(vf_name, sizeof(vf_name), "%s", if_name(vf_ifp)); 4675 HN_UNLOCK(sc); 4676 return sysctl_handle_string(oidp, vf_name, sizeof(vf_name), req); 4677 } 4678 4679 static int 4680 hn_rxvf_sysctl(SYSCTL_HANDLER_ARGS) 4681 { 4682 struct hn_softc *sc = arg1; 4683 char vf_name[IFNAMSIZ + 1]; 4684 if_t vf_ifp; 4685 4686 HN_LOCK(sc); 4687 vf_name[0] = '\0'; 4688 vf_ifp = sc->hn_rx_ring[0].hn_rxvf_ifp; 4689 if (vf_ifp != NULL) 4690 snprintf(vf_name, sizeof(vf_name), "%s", if_name(vf_ifp)); 4691 HN_UNLOCK(sc); 4692 return sysctl_handle_string(oidp, vf_name, sizeof(vf_name), req); 4693 } 4694 4695 static int 4696 hn_vflist_sysctl(SYSCTL_HANDLER_ARGS) 4697 { 4698 struct rm_priotracker pt; 4699 struct sbuf *sb; 4700 int error, i; 4701 bool first; 4702 4703 error = sysctl_wire_old_buffer(req, 0); 4704 if (error != 0) 4705 return (error); 4706 4707 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4708 if (sb == NULL) 4709 return (ENOMEM); 4710 4711 rm_rlock(&hn_vfmap_lock, &pt); 4712 4713 first = true; 4714 for (i = 0; i < hn_vfmap_size; ++i) { 4715 struct epoch_tracker et; 4716 if_t ifp; 4717 4718 if (hn_vfmap[i] == NULL) 4719 continue; 4720 4721 NET_EPOCH_ENTER(et); 4722 ifp = ifnet_byindex(i); 4723 if (ifp != NULL) { 4724 if (first) 4725 sbuf_printf(sb, "%s", if_name(ifp)); 4726 else 4727 sbuf_printf(sb, " %s", if_name(ifp)); 4728 first = false; 4729 } 4730 NET_EPOCH_EXIT(et); 4731 } 4732 4733 rm_runlock(&hn_vfmap_lock, &pt); 4734 4735 error = sbuf_finish(sb); 4736 sbuf_delete(sb); 4737 return (error); 4738 } 4739 4740 static int 4741 hn_vfmap_sysctl(SYSCTL_HANDLER_ARGS) 4742 { 4743 struct rm_priotracker pt; 4744 struct sbuf *sb; 4745 int error, i; 4746 bool first; 4747 4748 error = sysctl_wire_old_buffer(req, 0); 4749 if (error != 0) 4750 return (error); 4751 4752 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4753 if (sb == NULL) 4754 return (ENOMEM); 4755 4756 rm_rlock(&hn_vfmap_lock, &pt); 4757 4758 first = true; 4759 for (i = 0; i < hn_vfmap_size; ++i) { 4760 struct epoch_tracker et; 4761 if_t ifp, hn_ifp; 4762 4763 hn_ifp = hn_vfmap[i]; 4764 if (hn_ifp == NULL) 4765 continue; 4766 4767 NET_EPOCH_ENTER(et); 4768 ifp = ifnet_byindex(i); 4769 if (ifp != NULL) { 4770 if (first) { 4771 sbuf_printf(sb, "%s:%s", if_name(ifp), 4772 if_name(hn_ifp)); 4773 } else { 4774 sbuf_printf(sb, " %s:%s", if_name(ifp), 4775 if_name(hn_ifp)); 4776 } 4777 first = false; 4778 } 4779 NET_EPOCH_EXIT(et); 4780 } 4781 4782 rm_runlock(&hn_vfmap_lock, &pt); 4783 4784 error = sbuf_finish(sb); 4785 sbuf_delete(sb); 4786 return (error); 4787 } 4788 4789 static int 4790 hn_xpnt_vf_accbpf_sysctl(SYSCTL_HANDLER_ARGS) 4791 { 4792 struct hn_softc *sc = arg1; 4793 int error, onoff = 0; 4794 4795 if (sc->hn_xvf_flags & HN_XVFFLAG_ACCBPF) 4796 onoff = 1; 4797 error = sysctl_handle_int(oidp, &onoff, 0, req); 4798 if (error || req->newptr == NULL) 4799 return (error); 4800 4801 HN_LOCK(sc); 4802 /* NOTE: hn_vf_lock for hn_transmit() */ 4803 rm_wlock(&sc->hn_vf_lock); 4804 if (onoff) 4805 sc->hn_xvf_flags |= HN_XVFFLAG_ACCBPF; 4806 else 4807 sc->hn_xvf_flags &= ~HN_XVFFLAG_ACCBPF; 4808 rm_wunlock(&sc->hn_vf_lock); 4809 HN_UNLOCK(sc); 4810 4811 return (0); 4812 } 4813 4814 static int 4815 hn_xpnt_vf_enabled_sysctl(SYSCTL_HANDLER_ARGS) 4816 { 4817 struct hn_softc *sc = arg1; 4818 int enabled = 0; 4819 4820 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 4821 enabled = 1; 4822 return (sysctl_handle_int(oidp, &enabled, 0, req)); 4823 } 4824 4825 static int 4826 hn_check_iplen(const struct mbuf *m, int hoff) 4827 { 4828 const struct ip *ip; 4829 int len, iphlen, iplen; 4830 const struct tcphdr *th; 4831 int thoff; /* TCP data offset */ 4832 4833 len = hoff + sizeof(struct ip); 4834 4835 /* The packet must be at least the size of an IP header. */ 4836 if (m->m_pkthdr.len < len) 4837 return IPPROTO_DONE; 4838 4839 /* The fixed IP header must reside completely in the first mbuf. */ 4840 if (m->m_len < len) 4841 return IPPROTO_DONE; 4842 4843 ip = mtodo(m, hoff); 4844 4845 /* Bound check the packet's stated IP header length. */ 4846 iphlen = ip->ip_hl << 2; 4847 if (iphlen < sizeof(struct ip)) /* minimum header length */ 4848 return IPPROTO_DONE; 4849 4850 /* The full IP header must reside completely in the one mbuf. */ 4851 if (m->m_len < hoff + iphlen) 4852 return IPPROTO_DONE; 4853 4854 iplen = ntohs(ip->ip_len); 4855 4856 /* 4857 * Check that the amount of data in the buffers is as 4858 * at least much as the IP header would have us expect. 4859 */ 4860 if (m->m_pkthdr.len < hoff + iplen) 4861 return IPPROTO_DONE; 4862 4863 /* 4864 * Ignore IP fragments. 4865 */ 4866 if (ntohs(ip->ip_off) & (IP_OFFMASK | IP_MF)) 4867 return IPPROTO_DONE; 4868 4869 /* 4870 * The TCP/IP or UDP/IP header must be entirely contained within 4871 * the first fragment of a packet. 4872 */ 4873 switch (ip->ip_p) { 4874 case IPPROTO_TCP: 4875 if (iplen < iphlen + sizeof(struct tcphdr)) 4876 return IPPROTO_DONE; 4877 if (m->m_len < hoff + iphlen + sizeof(struct tcphdr)) 4878 return IPPROTO_DONE; 4879 th = (const struct tcphdr *)((const uint8_t *)ip + iphlen); 4880 thoff = th->th_off << 2; 4881 if (thoff < sizeof(struct tcphdr) || thoff + iphlen > iplen) 4882 return IPPROTO_DONE; 4883 if (m->m_len < hoff + iphlen + thoff) 4884 return IPPROTO_DONE; 4885 break; 4886 case IPPROTO_UDP: 4887 if (iplen < iphlen + sizeof(struct udphdr)) 4888 return IPPROTO_DONE; 4889 if (m->m_len < hoff + iphlen + sizeof(struct udphdr)) 4890 return IPPROTO_DONE; 4891 break; 4892 default: 4893 if (iplen < iphlen) 4894 return IPPROTO_DONE; 4895 break; 4896 } 4897 return ip->ip_p; 4898 } 4899 4900 static void 4901 hn_rxpkt_proto(const struct mbuf *m_new, int *l3proto, int *l4proto) 4902 { 4903 const struct ether_header *eh; 4904 uint16_t etype; 4905 int hoff; 4906 4907 hoff = sizeof(*eh); 4908 /* Checked at the beginning of this function. */ 4909 KASSERT(m_new->m_len >= hoff, ("not ethernet frame")); 4910 4911 eh = mtod(m_new, const struct ether_header *); 4912 etype = ntohs(eh->ether_type); 4913 if (etype == ETHERTYPE_VLAN) { 4914 const struct ether_vlan_header *evl; 4915 4916 hoff = sizeof(*evl); 4917 if (m_new->m_len < hoff) 4918 return; 4919 evl = mtod(m_new, const struct ether_vlan_header *); 4920 etype = ntohs(evl->evl_proto); 4921 } 4922 *l3proto = etype; 4923 4924 if (etype == ETHERTYPE_IP) 4925 *l4proto = hn_check_iplen(m_new, hoff); 4926 else 4927 *l4proto = IPPROTO_DONE; 4928 } 4929 4930 static int 4931 hn_create_rx_data(struct hn_softc *sc, int ring_cnt) 4932 { 4933 struct sysctl_oid_list *child; 4934 struct sysctl_ctx_list *ctx; 4935 device_t dev = sc->hn_dev; 4936 #if defined(INET) || defined(INET6) 4937 int lroent_cnt; 4938 #endif 4939 int i; 4940 4941 /* 4942 * Create RXBUF for reception. 4943 * 4944 * NOTE: 4945 * - It is shared by all channels. 4946 * - A large enough buffer is allocated, certain version of NVSes 4947 * may further limit the usable space. 4948 */ 4949 sc->hn_rxbuf = contigmalloc(HN_RXBUF_SIZE, M_DEVBUF, M_WAITOK | M_ZERO, 4950 0ul, ~0ul, PAGE_SIZE, 0); 4951 if (sc->hn_rxbuf == NULL) { 4952 device_printf(sc->hn_dev, "allocate rxbuf failed\n"); 4953 return (ENOMEM); 4954 } 4955 4956 sc->hn_rx_ring_cnt = ring_cnt; 4957 sc->hn_rx_ring_inuse = sc->hn_rx_ring_cnt; 4958 4959 sc->hn_rx_ring = malloc(sizeof(struct hn_rx_ring) * sc->hn_rx_ring_cnt, 4960 M_DEVBUF, M_WAITOK | M_ZERO); 4961 4962 #if defined(INET) || defined(INET6) 4963 lroent_cnt = hn_lro_entry_count; 4964 if (lroent_cnt < TCP_LRO_ENTRIES) 4965 lroent_cnt = TCP_LRO_ENTRIES; 4966 if (bootverbose) 4967 device_printf(dev, "LRO: entry count %d\n", lroent_cnt); 4968 #endif /* INET || INET6 */ 4969 4970 ctx = device_get_sysctl_ctx(dev); 4971 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 4972 4973 /* Create dev.hn.UNIT.rx sysctl tree */ 4974 sc->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "rx", 4975 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 4976 4977 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4978 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 4979 4980 rxr->hn_br = contigmalloc(HN_TXBR_SIZE + HN_RXBR_SIZE, M_DEVBUF, 4981 M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0); 4982 if (rxr->hn_br == NULL) { 4983 device_printf(dev, "allocate bufring failed\n"); 4984 return (ENOMEM); 4985 } 4986 4987 if (hn_trust_hosttcp) 4988 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_TCP; 4989 if (hn_trust_hostudp) 4990 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_UDP; 4991 if (hn_trust_hostip) 4992 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_IP; 4993 rxr->hn_mbuf_hash = NDIS_HASH_ALL; 4994 rxr->hn_ifp = sc->hn_ifp; 4995 if (i < sc->hn_tx_ring_cnt) 4996 rxr->hn_txr = &sc->hn_tx_ring[i]; 4997 rxr->hn_pktbuf_len = HN_PKTBUF_LEN_DEF; 4998 rxr->hn_pktbuf = malloc(rxr->hn_pktbuf_len, M_DEVBUF, M_WAITOK); 4999 rxr->hn_rx_idx = i; 5000 rxr->hn_rxbuf = sc->hn_rxbuf; 5001 5002 /* 5003 * Initialize LRO. 5004 */ 5005 #if defined(INET) || defined(INET6) 5006 tcp_lro_init_args(&rxr->hn_lro, sc->hn_ifp, lroent_cnt, 5007 hn_lro_mbufq_depth); 5008 rxr->hn_lro.lro_length_lim = HN_LRO_LENLIM_DEF; 5009 rxr->hn_lro.lro_ackcnt_lim = HN_LRO_ACKCNT_DEF; 5010 #endif /* INET || INET6 */ 5011 5012 if (sc->hn_rx_sysctl_tree != NULL) { 5013 char name[16]; 5014 5015 /* 5016 * Create per RX ring sysctl tree: 5017 * dev.hn.UNIT.rx.RINGID 5018 */ 5019 snprintf(name, sizeof(name), "%d", i); 5020 rxr->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, 5021 SYSCTL_CHILDREN(sc->hn_rx_sysctl_tree), 5022 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5023 5024 if (rxr->hn_rx_sysctl_tree != NULL) { 5025 SYSCTL_ADD_ULONG(ctx, 5026 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5027 OID_AUTO, "packets", 5028 CTLFLAG_RW | CTLFLAG_STATS, &rxr->hn_pkts, 5029 "# of packets received"); 5030 SYSCTL_ADD_ULONG(ctx, 5031 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5032 OID_AUTO, "rss_pkts", 5033 CTLFLAG_RW | CTLFLAG_STATS, 5034 &rxr->hn_rss_pkts, 5035 "# of packets w/ RSS info received"); 5036 SYSCTL_ADD_ULONG(ctx, 5037 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5038 OID_AUTO, "rsc_pkts", 5039 CTLFLAG_RW | CTLFLAG_STATS, 5040 &rxr->hn_rsc_pkts, 5041 "# of RSC packets received"); 5042 SYSCTL_ADD_ULONG(ctx, 5043 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5044 OID_AUTO, "rsc_drop", 5045 CTLFLAG_RW | CTLFLAG_STATS, 5046 &rxr->hn_rsc_drop, 5047 "# of RSC fragments dropped"); 5048 SYSCTL_ADD_INT(ctx, 5049 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5050 OID_AUTO, "pktbuf_len", CTLFLAG_RD, 5051 &rxr->hn_pktbuf_len, 0, 5052 "Temporary channel packet buffer length"); 5053 } 5054 } 5055 } 5056 5057 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_queued", 5058 CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5059 __offsetof(struct hn_rx_ring, hn_lro.lro_queued), 5060 hn_rx_stat_u64_sysctl, 5061 "LU", "LRO queued"); 5062 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_flushed", 5063 CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5064 __offsetof(struct hn_rx_ring, hn_lro.lro_flushed), 5065 hn_rx_stat_u64_sysctl, 5066 "LU", "LRO flushed"); 5067 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_tried", 5068 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5069 __offsetof(struct hn_rx_ring, hn_lro_tried), 5070 hn_rx_stat_ulong_sysctl, "LU", "# of LRO tries"); 5071 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_length_lim", 5072 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5073 hn_lro_lenlim_sysctl, "IU", 5074 "Max # of data bytes to be aggregated by LRO"); 5075 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_ackcnt_lim", 5076 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5077 hn_lro_ackcnt_sysctl, "I", 5078 "Max # of ACKs to be aggregated by LRO"); 5079 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hosttcp", 5080 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_TCP, 5081 hn_trust_hcsum_sysctl, "I", 5082 "Trust tcp segment verification on host side, " 5083 "when csum info is missing"); 5084 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostudp", 5085 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_UDP, 5086 hn_trust_hcsum_sysctl, "I", 5087 "Trust udp datagram verification on host side, " 5088 "when csum info is missing"); 5089 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostip", 5090 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_IP, 5091 hn_trust_hcsum_sysctl, "I", 5092 "Trust ip packet verification on host side, " 5093 "when csum info is missing"); 5094 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_ip", 5095 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5096 __offsetof(struct hn_rx_ring, hn_csum_ip), 5097 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM IP"); 5098 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_tcp", 5099 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5100 __offsetof(struct hn_rx_ring, hn_csum_tcp), 5101 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM TCP"); 5102 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_udp", 5103 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5104 __offsetof(struct hn_rx_ring, hn_csum_udp), 5105 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM UDP"); 5106 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_trusted", 5107 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5108 __offsetof(struct hn_rx_ring, hn_csum_trusted), 5109 hn_rx_stat_ulong_sysctl, "LU", 5110 "# of packets that we trust host's csum verification"); 5111 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "small_pkts", 5112 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5113 __offsetof(struct hn_rx_ring, hn_small_pkts), 5114 hn_rx_stat_ulong_sysctl, "LU", "# of small packets received"); 5115 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_ack_failed", 5116 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5117 __offsetof(struct hn_rx_ring, hn_ack_failed), 5118 hn_rx_stat_ulong_sysctl, "LU", "# of RXBUF ack failures"); 5119 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_cnt", 5120 CTLFLAG_RD, &sc->hn_rx_ring_cnt, 0, "# created RX rings"); 5121 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_inuse", 5122 CTLFLAG_RD, &sc->hn_rx_ring_inuse, 0, "# used RX rings"); 5123 5124 return (0); 5125 } 5126 5127 static void 5128 hn_destroy_rx_data(struct hn_softc *sc) 5129 { 5130 int i; 5131 5132 if (sc->hn_rxbuf != NULL) { 5133 if ((sc->hn_flags & HN_FLAG_RXBUF_REF) == 0) 5134 free(sc->hn_rxbuf, M_DEVBUF); 5135 else 5136 device_printf(sc->hn_dev, "RXBUF is referenced\n"); 5137 sc->hn_rxbuf = NULL; 5138 } 5139 5140 if (sc->hn_rx_ring_cnt == 0) 5141 return; 5142 5143 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 5144 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 5145 5146 if (rxr->hn_br == NULL) 5147 continue; 5148 if ((rxr->hn_rx_flags & HN_RX_FLAG_BR_REF) == 0) { 5149 free(rxr->hn_br, M_DEVBUF); 5150 } else { 5151 device_printf(sc->hn_dev, 5152 "%dth channel bufring is referenced", i); 5153 } 5154 rxr->hn_br = NULL; 5155 5156 #if defined(INET) || defined(INET6) 5157 tcp_lro_free(&rxr->hn_lro); 5158 #endif 5159 free(rxr->hn_pktbuf, M_DEVBUF); 5160 } 5161 free(sc->hn_rx_ring, M_DEVBUF); 5162 sc->hn_rx_ring = NULL; 5163 5164 sc->hn_rx_ring_cnt = 0; 5165 sc->hn_rx_ring_inuse = 0; 5166 } 5167 5168 static int 5169 hn_tx_ring_create(struct hn_softc *sc, int id) 5170 { 5171 struct hn_tx_ring *txr = &sc->hn_tx_ring[id]; 5172 device_t dev = sc->hn_dev; 5173 bus_dma_tag_t parent_dtag; 5174 int error, i; 5175 5176 txr->hn_sc = sc; 5177 txr->hn_tx_idx = id; 5178 5179 #ifndef HN_USE_TXDESC_BUFRING 5180 mtx_init(&txr->hn_txlist_spin, "hn txlist", NULL, MTX_SPIN); 5181 #endif 5182 mtx_init(&txr->hn_tx_lock, "hn tx", NULL, MTX_DEF); 5183 5184 txr->hn_txdesc_cnt = HN_TX_DESC_CNT; 5185 txr->hn_txdesc = malloc(sizeof(struct hn_txdesc) * txr->hn_txdesc_cnt, 5186 M_DEVBUF, M_WAITOK | M_ZERO); 5187 #ifndef HN_USE_TXDESC_BUFRING 5188 SLIST_INIT(&txr->hn_txlist); 5189 #else 5190 txr->hn_txdesc_br = buf_ring_alloc(txr->hn_txdesc_cnt, M_DEVBUF, 5191 M_WAITOK, &txr->hn_tx_lock); 5192 #endif 5193 5194 if (hn_tx_taskq_mode == HN_TX_TASKQ_M_EVTTQ) { 5195 txr->hn_tx_taskq = VMBUS_GET_EVENT_TASKQ( 5196 device_get_parent(dev), dev, HN_RING_IDX2CPU(sc, id)); 5197 } else { 5198 txr->hn_tx_taskq = sc->hn_tx_taskqs[id % hn_tx_taskq_cnt]; 5199 } 5200 5201 #ifdef HN_IFSTART_SUPPORT 5202 if (hn_use_if_start) { 5203 txr->hn_txeof = hn_start_txeof; 5204 TASK_INIT(&txr->hn_tx_task, 0, hn_start_taskfunc, txr); 5205 TASK_INIT(&txr->hn_txeof_task, 0, hn_start_txeof_taskfunc, txr); 5206 } else 5207 #endif 5208 { 5209 int br_depth; 5210 5211 txr->hn_txeof = hn_xmit_txeof; 5212 TASK_INIT(&txr->hn_tx_task, 0, hn_xmit_taskfunc, txr); 5213 TASK_INIT(&txr->hn_txeof_task, 0, hn_xmit_txeof_taskfunc, txr); 5214 5215 br_depth = hn_get_txswq_depth(txr); 5216 txr->hn_mbuf_br = buf_ring_alloc(br_depth, M_DEVBUF, 5217 M_WAITOK, &txr->hn_tx_lock); 5218 } 5219 5220 txr->hn_direct_tx_size = hn_direct_tx_size; 5221 5222 /* 5223 * Always schedule transmission instead of trying to do direct 5224 * transmission. This one gives the best performance so far. 5225 */ 5226 txr->hn_sched_tx = 1; 5227 5228 parent_dtag = bus_get_dma_tag(dev); 5229 5230 /* DMA tag for RNDIS packet messages. */ 5231 error = bus_dma_tag_create(parent_dtag, /* parent */ 5232 HN_RNDIS_PKT_ALIGN, /* alignment */ 5233 HN_RNDIS_PKT_BOUNDARY, /* boundary */ 5234 BUS_SPACE_MAXADDR, /* lowaddr */ 5235 BUS_SPACE_MAXADDR, /* highaddr */ 5236 NULL, NULL, /* filter, filterarg */ 5237 HN_RNDIS_PKT_LEN, /* maxsize */ 5238 1, /* nsegments */ 5239 HN_RNDIS_PKT_LEN, /* maxsegsize */ 5240 0, /* flags */ 5241 NULL, /* lockfunc */ 5242 NULL, /* lockfuncarg */ 5243 &txr->hn_tx_rndis_dtag); 5244 if (error) { 5245 device_printf(dev, "failed to create rndis dmatag\n"); 5246 return error; 5247 } 5248 5249 /* DMA tag for data. */ 5250 error = bus_dma_tag_create(parent_dtag, /* parent */ 5251 1, /* alignment */ 5252 HN_TX_DATA_BOUNDARY, /* boundary */ 5253 BUS_SPACE_MAXADDR, /* lowaddr */ 5254 BUS_SPACE_MAXADDR, /* highaddr */ 5255 NULL, NULL, /* filter, filterarg */ 5256 HN_TX_DATA_MAXSIZE, /* maxsize */ 5257 HN_TX_DATA_SEGCNT_MAX, /* nsegments */ 5258 HN_TX_DATA_SEGSIZE, /* maxsegsize */ 5259 0, /* flags */ 5260 NULL, /* lockfunc */ 5261 NULL, /* lockfuncarg */ 5262 &txr->hn_tx_data_dtag); 5263 if (error) { 5264 device_printf(dev, "failed to create data dmatag\n"); 5265 return error; 5266 } 5267 5268 for (i = 0; i < txr->hn_txdesc_cnt; ++i) { 5269 struct hn_txdesc *txd = &txr->hn_txdesc[i]; 5270 5271 txd->txr = txr; 5272 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 5273 STAILQ_INIT(&txd->agg_list); 5274 5275 /* 5276 * Allocate and load RNDIS packet message. 5277 */ 5278 error = bus_dmamem_alloc(txr->hn_tx_rndis_dtag, 5279 (void **)&txd->rndis_pkt, 5280 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, 5281 &txd->rndis_pkt_dmap); 5282 if (error) { 5283 device_printf(dev, 5284 "failed to allocate rndis_packet_msg, %d\n", i); 5285 return error; 5286 } 5287 5288 error = bus_dmamap_load(txr->hn_tx_rndis_dtag, 5289 txd->rndis_pkt_dmap, 5290 txd->rndis_pkt, HN_RNDIS_PKT_LEN, 5291 hyperv_dma_map_paddr, &txd->rndis_pkt_paddr, 5292 BUS_DMA_NOWAIT); 5293 if (error) { 5294 device_printf(dev, 5295 "failed to load rndis_packet_msg, %d\n", i); 5296 bus_dmamem_free(txr->hn_tx_rndis_dtag, 5297 txd->rndis_pkt, txd->rndis_pkt_dmap); 5298 return error; 5299 } 5300 5301 /* DMA map for TX data. */ 5302 error = bus_dmamap_create(txr->hn_tx_data_dtag, 0, 5303 &txd->data_dmap); 5304 if (error) { 5305 device_printf(dev, 5306 "failed to allocate tx data dmamap\n"); 5307 bus_dmamap_unload(txr->hn_tx_rndis_dtag, 5308 txd->rndis_pkt_dmap); 5309 bus_dmamem_free(txr->hn_tx_rndis_dtag, 5310 txd->rndis_pkt, txd->rndis_pkt_dmap); 5311 return error; 5312 } 5313 5314 /* All set, put it to list */ 5315 txd->flags |= HN_TXD_FLAG_ONLIST; 5316 #ifndef HN_USE_TXDESC_BUFRING 5317 SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); 5318 #else 5319 buf_ring_enqueue(txr->hn_txdesc_br, txd); 5320 #endif 5321 } 5322 txr->hn_txdesc_avail = txr->hn_txdesc_cnt; 5323 5324 if (sc->hn_tx_sysctl_tree != NULL) { 5325 struct sysctl_oid_list *child; 5326 struct sysctl_ctx_list *ctx; 5327 char name[16]; 5328 5329 /* 5330 * Create per TX ring sysctl tree: 5331 * dev.hn.UNIT.tx.RINGID 5332 */ 5333 ctx = device_get_sysctl_ctx(dev); 5334 child = SYSCTL_CHILDREN(sc->hn_tx_sysctl_tree); 5335 5336 snprintf(name, sizeof(name), "%d", id); 5337 txr->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, 5338 name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5339 5340 if (txr->hn_tx_sysctl_tree != NULL) { 5341 child = SYSCTL_CHILDREN(txr->hn_tx_sysctl_tree); 5342 5343 #ifdef HN_DEBUG 5344 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_avail", 5345 CTLFLAG_RD, &txr->hn_txdesc_avail, 0, 5346 "# of available TX descs"); 5347 #endif 5348 #ifdef HN_IFSTART_SUPPORT 5349 if (!hn_use_if_start) 5350 #endif 5351 { 5352 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "oactive", 5353 CTLFLAG_RD, &txr->hn_oactive, 0, 5354 "over active"); 5355 } 5356 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "packets", 5357 CTLFLAG_RW | CTLFLAG_STATS, &txr->hn_pkts, 5358 "# of packets transmitted"); 5359 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "sends", 5360 CTLFLAG_RW | CTLFLAG_STATS, &txr->hn_sends, 5361 "# of sends"); 5362 } 5363 } 5364 5365 return 0; 5366 } 5367 5368 static void 5369 hn_txdesc_dmamap_destroy(struct hn_txdesc *txd) 5370 { 5371 struct hn_tx_ring *txr = txd->txr; 5372 5373 KASSERT(txd->m == NULL, ("still has mbuf installed")); 5374 KASSERT((txd->flags & HN_TXD_FLAG_DMAMAP) == 0, ("still dma mapped")); 5375 5376 bus_dmamap_unload(txr->hn_tx_rndis_dtag, txd->rndis_pkt_dmap); 5377 bus_dmamem_free(txr->hn_tx_rndis_dtag, txd->rndis_pkt, 5378 txd->rndis_pkt_dmap); 5379 bus_dmamap_destroy(txr->hn_tx_data_dtag, txd->data_dmap); 5380 } 5381 5382 static void 5383 hn_txdesc_gc(struct hn_tx_ring *txr, struct hn_txdesc *txd) 5384 { 5385 5386 KASSERT(txd->refs == 0 || txd->refs == 1, 5387 ("invalid txd refs %d", txd->refs)); 5388 5389 /* Aggregated txds will be freed by their aggregating txd. */ 5390 if (txd->refs > 0 && (txd->flags & HN_TXD_FLAG_ONAGG) == 0) { 5391 int freed __diagused; 5392 5393 freed = hn_txdesc_put(txr, txd); 5394 KASSERT(freed, ("can't free txdesc")); 5395 } 5396 } 5397 5398 static void 5399 hn_tx_ring_destroy(struct hn_tx_ring *txr) 5400 { 5401 int i; 5402 5403 if (txr->hn_txdesc == NULL) 5404 return; 5405 5406 /* 5407 * NOTE: 5408 * Because the freeing of aggregated txds will be deferred 5409 * to the aggregating txd, two passes are used here: 5410 * - The first pass GCes any pending txds. This GC is necessary, 5411 * since if the channels are revoked, hypervisor will not 5412 * deliver send-done for all pending txds. 5413 * - The second pass frees the busdma stuffs, i.e. after all txds 5414 * were freed. 5415 */ 5416 for (i = 0; i < txr->hn_txdesc_cnt; ++i) 5417 hn_txdesc_gc(txr, &txr->hn_txdesc[i]); 5418 for (i = 0; i < txr->hn_txdesc_cnt; ++i) 5419 hn_txdesc_dmamap_destroy(&txr->hn_txdesc[i]); 5420 5421 if (txr->hn_tx_data_dtag != NULL) 5422 bus_dma_tag_destroy(txr->hn_tx_data_dtag); 5423 if (txr->hn_tx_rndis_dtag != NULL) 5424 bus_dma_tag_destroy(txr->hn_tx_rndis_dtag); 5425 5426 #ifdef HN_USE_TXDESC_BUFRING 5427 buf_ring_free(txr->hn_txdesc_br, M_DEVBUF); 5428 #endif 5429 5430 free(txr->hn_txdesc, M_DEVBUF); 5431 txr->hn_txdesc = NULL; 5432 5433 if (txr->hn_mbuf_br != NULL) 5434 buf_ring_free(txr->hn_mbuf_br, M_DEVBUF); 5435 5436 #ifndef HN_USE_TXDESC_BUFRING 5437 mtx_destroy(&txr->hn_txlist_spin); 5438 #endif 5439 mtx_destroy(&txr->hn_tx_lock); 5440 } 5441 5442 static int 5443 hn_create_tx_data(struct hn_softc *sc, int ring_cnt) 5444 { 5445 struct sysctl_oid_list *child; 5446 struct sysctl_ctx_list *ctx; 5447 int i; 5448 5449 /* 5450 * Create TXBUF for chimney sending. 5451 * 5452 * NOTE: It is shared by all channels. 5453 */ 5454 sc->hn_chim = contigmalloc(HN_CHIM_SIZE, M_DEVBUF, M_WAITOK | M_ZERO, 5455 0ul, ~0ul, PAGE_SIZE, 0); 5456 if (sc->hn_chim == NULL) { 5457 device_printf(sc->hn_dev, "allocate txbuf failed\n"); 5458 return (ENOMEM); 5459 } 5460 5461 sc->hn_tx_ring_cnt = ring_cnt; 5462 sc->hn_tx_ring_inuse = sc->hn_tx_ring_cnt; 5463 5464 sc->hn_tx_ring = malloc(sizeof(struct hn_tx_ring) * sc->hn_tx_ring_cnt, 5465 M_DEVBUF, M_WAITOK | M_ZERO); 5466 5467 ctx = device_get_sysctl_ctx(sc->hn_dev); 5468 child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->hn_dev)); 5469 5470 /* Create dev.hn.UNIT.tx sysctl tree */ 5471 sc->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "tx", 5472 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5473 5474 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 5475 int error; 5476 5477 error = hn_tx_ring_create(sc, i); 5478 if (error) 5479 return error; 5480 } 5481 5482 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "no_txdescs", 5483 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5484 __offsetof(struct hn_tx_ring, hn_no_txdescs), 5485 hn_tx_stat_ulong_sysctl, "LU", "# of times short of TX descs"); 5486 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "send_failed", 5487 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5488 __offsetof(struct hn_tx_ring, hn_send_failed), 5489 hn_tx_stat_ulong_sysctl, "LU", "# of hyper-v sending failure"); 5490 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txdma_failed", 5491 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5492 __offsetof(struct hn_tx_ring, hn_txdma_failed), 5493 hn_tx_stat_ulong_sysctl, "LU", "# of TX DMA failure"); 5494 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_flush_failed", 5495 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5496 __offsetof(struct hn_tx_ring, hn_flush_failed), 5497 hn_tx_stat_ulong_sysctl, "LU", 5498 "# of packet transmission aggregation flush failure"); 5499 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_collapsed", 5500 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5501 __offsetof(struct hn_tx_ring, hn_tx_collapsed), 5502 hn_tx_stat_ulong_sysctl, "LU", "# of TX mbuf collapsed"); 5503 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney", 5504 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5505 __offsetof(struct hn_tx_ring, hn_tx_chimney), 5506 hn_tx_stat_ulong_sysctl, "LU", "# of chimney send"); 5507 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_tried", 5508 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5509 __offsetof(struct hn_tx_ring, hn_tx_chimney_tried), 5510 hn_tx_stat_ulong_sysctl, "LU", "# of chimney send tries"); 5511 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_cnt", 5512 CTLFLAG_RD, &sc->hn_tx_ring[0].hn_txdesc_cnt, 0, 5513 "# of total TX descs"); 5514 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_chimney_max", 5515 CTLFLAG_RD, &sc->hn_chim_szmax, 0, 5516 "Chimney send packet size upper boundary"); 5517 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_size", 5518 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5519 hn_chim_size_sysctl, "I", "Chimney send packet size limit"); 5520 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "direct_tx_size", 5521 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5522 __offsetof(struct hn_tx_ring, hn_direct_tx_size), 5523 hn_tx_conf_int_sysctl, "I", 5524 "Size of the packet for direct transmission"); 5525 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "sched_tx", 5526 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5527 __offsetof(struct hn_tx_ring, hn_sched_tx), 5528 hn_tx_conf_int_sysctl, "I", 5529 "Always schedule transmission " 5530 "instead of doing direct transmission"); 5531 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_cnt", 5532 CTLFLAG_RD, &sc->hn_tx_ring_cnt, 0, "# created TX rings"); 5533 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_inuse", 5534 CTLFLAG_RD, &sc->hn_tx_ring_inuse, 0, "# used TX rings"); 5535 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "agg_szmax", 5536 CTLFLAG_RD, &sc->hn_tx_ring[0].hn_agg_szmax, 0, 5537 "Applied packet transmission aggregation size"); 5538 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_pktmax", 5539 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 5540 hn_txagg_pktmax_sysctl, "I", 5541 "Applied packet transmission aggregation packets"); 5542 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_align", 5543 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 5544 hn_txagg_align_sysctl, "I", 5545 "Applied packet transmission aggregation alignment"); 5546 5547 return 0; 5548 } 5549 5550 static void 5551 hn_set_chim_size(struct hn_softc *sc, int chim_size) 5552 { 5553 int i; 5554 5555 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5556 sc->hn_tx_ring[i].hn_chim_size = chim_size; 5557 } 5558 5559 static void 5560 hn_set_tso_maxsize(struct hn_softc *sc, int tso_maxlen, int mtu) 5561 { 5562 if_t ifp = sc->hn_ifp; 5563 u_int hw_tsomax; 5564 int tso_minlen; 5565 5566 HN_LOCK_ASSERT(sc); 5567 5568 if ((if_getcapabilities(ifp) & (IFCAP_TSO4 | IFCAP_TSO6)) == 0) 5569 return; 5570 5571 KASSERT(sc->hn_ndis_tso_sgmin >= 2, 5572 ("invalid NDIS tso sgmin %d", sc->hn_ndis_tso_sgmin)); 5573 tso_minlen = sc->hn_ndis_tso_sgmin * mtu; 5574 5575 KASSERT(sc->hn_ndis_tso_szmax >= tso_minlen && 5576 sc->hn_ndis_tso_szmax <= IP_MAXPACKET, 5577 ("invalid NDIS tso szmax %d", sc->hn_ndis_tso_szmax)); 5578 5579 if (tso_maxlen < tso_minlen) 5580 tso_maxlen = tso_minlen; 5581 else if (tso_maxlen > IP_MAXPACKET) 5582 tso_maxlen = IP_MAXPACKET; 5583 if (tso_maxlen > sc->hn_ndis_tso_szmax) 5584 tso_maxlen = sc->hn_ndis_tso_szmax; 5585 hw_tsomax = tso_maxlen - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 5586 5587 if (hn_xpnt_vf_isready(sc)) { 5588 if (hw_tsomax > if_gethwtsomax(sc->hn_vf_ifp)) 5589 hw_tsomax = if_gethwtsomax(sc->hn_vf_ifp); 5590 } 5591 if_sethwtsomax(ifp, hw_tsomax); 5592 if (bootverbose) 5593 if_printf(ifp, "TSO size max %u\n", if_gethwtsomax(ifp)); 5594 } 5595 5596 static void 5597 hn_fixup_tx_data(struct hn_softc *sc) 5598 { 5599 uint64_t csum_assist; 5600 int i; 5601 5602 hn_set_chim_size(sc, sc->hn_chim_szmax); 5603 if (hn_tx_chimney_size > 0 && 5604 hn_tx_chimney_size < sc->hn_chim_szmax) 5605 hn_set_chim_size(sc, hn_tx_chimney_size); 5606 5607 csum_assist = 0; 5608 if (sc->hn_caps & HN_CAP_IPCS) 5609 csum_assist |= CSUM_IP; 5610 if (sc->hn_caps & HN_CAP_TCP4CS) 5611 csum_assist |= CSUM_IP_TCP; 5612 if ((sc->hn_caps & HN_CAP_UDP4CS) && hn_enable_udp4cs) 5613 csum_assist |= CSUM_IP_UDP; 5614 if (sc->hn_caps & HN_CAP_TCP6CS) 5615 csum_assist |= CSUM_IP6_TCP; 5616 if ((sc->hn_caps & HN_CAP_UDP6CS) && hn_enable_udp6cs) 5617 csum_assist |= CSUM_IP6_UDP; 5618 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5619 sc->hn_tx_ring[i].hn_csum_assist = csum_assist; 5620 5621 if (sc->hn_caps & HN_CAP_HASHVAL) { 5622 /* 5623 * Support HASHVAL pktinfo on TX path. 5624 */ 5625 if (bootverbose) 5626 if_printf(sc->hn_ifp, "support HASHVAL pktinfo\n"); 5627 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5628 sc->hn_tx_ring[i].hn_tx_flags |= HN_TX_FLAG_HASHVAL; 5629 } 5630 } 5631 5632 static void 5633 hn_fixup_rx_data(struct hn_softc *sc) 5634 { 5635 5636 if (sc->hn_caps & HN_CAP_UDPHASH) { 5637 int i; 5638 5639 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 5640 sc->hn_rx_ring[i].hn_rx_flags |= HN_RX_FLAG_UDP_HASH; 5641 } 5642 } 5643 5644 static void 5645 hn_destroy_tx_data(struct hn_softc *sc) 5646 { 5647 int i; 5648 5649 if (sc->hn_chim != NULL) { 5650 if ((sc->hn_flags & HN_FLAG_CHIM_REF) == 0) { 5651 free(sc->hn_chim, M_DEVBUF); 5652 } else { 5653 device_printf(sc->hn_dev, 5654 "chimney sending buffer is referenced"); 5655 } 5656 sc->hn_chim = NULL; 5657 } 5658 5659 if (sc->hn_tx_ring_cnt == 0) 5660 return; 5661 5662 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5663 hn_tx_ring_destroy(&sc->hn_tx_ring[i]); 5664 5665 free(sc->hn_tx_ring, M_DEVBUF); 5666 sc->hn_tx_ring = NULL; 5667 5668 sc->hn_tx_ring_cnt = 0; 5669 sc->hn_tx_ring_inuse = 0; 5670 } 5671 5672 #ifdef HN_IFSTART_SUPPORT 5673 5674 static void 5675 hn_start_taskfunc(void *xtxr, int pending __unused) 5676 { 5677 struct hn_tx_ring *txr = xtxr; 5678 5679 mtx_lock(&txr->hn_tx_lock); 5680 hn_start_locked(txr, 0); 5681 mtx_unlock(&txr->hn_tx_lock); 5682 } 5683 5684 static int 5685 hn_start_locked(struct hn_tx_ring *txr, int len) 5686 { 5687 struct hn_softc *sc = txr->hn_sc; 5688 if_t ifp = sc->hn_ifp; 5689 int sched = 0; 5690 5691 KASSERT(hn_use_if_start, 5692 ("hn_start_locked is called, when if_start is disabled")); 5693 KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); 5694 mtx_assert(&txr->hn_tx_lock, MA_OWNED); 5695 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 5696 5697 if (__predict_false(txr->hn_suspended)) 5698 return (0); 5699 5700 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 5701 IFF_DRV_RUNNING) 5702 return (0); 5703 5704 while (!if_sendq_empty(ifp)) { 5705 struct hn_txdesc *txd; 5706 struct mbuf *m_head; 5707 int error; 5708 5709 m_head = if_dequeue(ifp); 5710 if (m_head == NULL) 5711 break; 5712 5713 if (len > 0 && m_head->m_pkthdr.len > len) { 5714 /* 5715 * This sending could be time consuming; let callers 5716 * dispatch this packet sending (and sending of any 5717 * following up packets) to tx taskqueue. 5718 */ 5719 if_sendq_prepend(ifp, m_head); 5720 sched = 1; 5721 break; 5722 } 5723 5724 #if defined(INET6) || defined(INET) 5725 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 5726 m_head = hn_tso_fixup(m_head); 5727 if (__predict_false(m_head == NULL)) { 5728 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 5729 continue; 5730 } 5731 } else if (m_head->m_pkthdr.csum_flags & 5732 (CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP6_UDP | CSUM_IP6_TCP)) { 5733 m_head = hn_set_hlen(m_head); 5734 if (__predict_false(m_head == NULL)) { 5735 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 5736 continue; 5737 } 5738 } 5739 #endif 5740 5741 txd = hn_txdesc_get(txr); 5742 if (txd == NULL) { 5743 txr->hn_no_txdescs++; 5744 if_sendq_prepend(ifp, m_head); 5745 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 5746 break; 5747 } 5748 5749 error = hn_encap(ifp, txr, txd, &m_head); 5750 if (error) { 5751 /* Both txd and m_head are freed */ 5752 KASSERT(txr->hn_agg_txd == NULL, 5753 ("encap failed w/ pending aggregating txdesc")); 5754 continue; 5755 } 5756 5757 if (txr->hn_agg_pktleft == 0) { 5758 if (txr->hn_agg_txd != NULL) { 5759 KASSERT(m_head == NULL, 5760 ("pending mbuf for aggregating txdesc")); 5761 error = hn_flush_txagg(ifp, txr); 5762 if (__predict_false(error)) { 5763 if_setdrvflagbits(ifp, 5764 IFF_DRV_OACTIVE, 0); 5765 break; 5766 } 5767 } else { 5768 KASSERT(m_head != NULL, ("mbuf was freed")); 5769 error = hn_txpkt(ifp, txr, txd); 5770 if (__predict_false(error)) { 5771 /* txd is freed, but m_head is not */ 5772 if_sendq_prepend(ifp, m_head); 5773 if_setdrvflagbits(ifp, 5774 IFF_DRV_OACTIVE, 0); 5775 break; 5776 } 5777 } 5778 } 5779 #ifdef INVARIANTS 5780 else { 5781 KASSERT(txr->hn_agg_txd != NULL, 5782 ("no aggregating txdesc")); 5783 KASSERT(m_head == NULL, 5784 ("pending mbuf for aggregating txdesc")); 5785 } 5786 #endif 5787 } 5788 5789 /* Flush pending aggerated transmission. */ 5790 if (txr->hn_agg_txd != NULL) 5791 hn_flush_txagg(ifp, txr); 5792 return (sched); 5793 } 5794 5795 static void 5796 hn_start(if_t ifp) 5797 { 5798 struct hn_softc *sc = if_getsoftc(ifp); 5799 struct hn_tx_ring *txr = &sc->hn_tx_ring[0]; 5800 5801 if (txr->hn_sched_tx) 5802 goto do_sched; 5803 5804 if (mtx_trylock(&txr->hn_tx_lock)) { 5805 int sched; 5806 5807 sched = hn_start_locked(txr, txr->hn_direct_tx_size); 5808 mtx_unlock(&txr->hn_tx_lock); 5809 if (!sched) 5810 return; 5811 } 5812 do_sched: 5813 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); 5814 } 5815 5816 static void 5817 hn_start_txeof_taskfunc(void *xtxr, int pending __unused) 5818 { 5819 struct hn_tx_ring *txr = xtxr; 5820 5821 mtx_lock(&txr->hn_tx_lock); 5822 if_setdrvflagbits(txr->hn_sc->hn_ifp, 0, IFF_DRV_OACTIVE); 5823 hn_start_locked(txr, 0); 5824 mtx_unlock(&txr->hn_tx_lock); 5825 } 5826 5827 static void 5828 hn_start_txeof(struct hn_tx_ring *txr) 5829 { 5830 struct hn_softc *sc = txr->hn_sc; 5831 if_t ifp = sc->hn_ifp; 5832 5833 KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); 5834 5835 if (txr->hn_sched_tx) 5836 goto do_sched; 5837 5838 if (mtx_trylock(&txr->hn_tx_lock)) { 5839 int sched; 5840 5841 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 5842 sched = hn_start_locked(txr, txr->hn_direct_tx_size); 5843 mtx_unlock(&txr->hn_tx_lock); 5844 if (sched) { 5845 taskqueue_enqueue(txr->hn_tx_taskq, 5846 &txr->hn_tx_task); 5847 } 5848 } else { 5849 do_sched: 5850 /* 5851 * Release the OACTIVE earlier, with the hope, that 5852 * others could catch up. The task will clear the 5853 * flag again with the hn_tx_lock to avoid possible 5854 * races. 5855 */ 5856 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 5857 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 5858 } 5859 } 5860 5861 #endif /* HN_IFSTART_SUPPORT */ 5862 5863 static int 5864 hn_xmit(struct hn_tx_ring *txr, int len) 5865 { 5866 struct hn_softc *sc = txr->hn_sc; 5867 if_t ifp = sc->hn_ifp; 5868 struct mbuf *m_head; 5869 int sched = 0; 5870 5871 mtx_assert(&txr->hn_tx_lock, MA_OWNED); 5872 #ifdef HN_IFSTART_SUPPORT 5873 KASSERT(hn_use_if_start == 0, 5874 ("hn_xmit is called, when if_start is enabled")); 5875 #endif 5876 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 5877 5878 if (__predict_false(txr->hn_suspended)) 5879 return (0); 5880 5881 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || txr->hn_oactive) 5882 return (0); 5883 5884 while ((m_head = drbr_peek(ifp, txr->hn_mbuf_br)) != NULL) { 5885 struct hn_txdesc *txd; 5886 int error; 5887 5888 if (len > 0 && m_head->m_pkthdr.len > len) { 5889 /* 5890 * This sending could be time consuming; let callers 5891 * dispatch this packet sending (and sending of any 5892 * following up packets) to tx taskqueue. 5893 */ 5894 drbr_putback(ifp, txr->hn_mbuf_br, m_head); 5895 sched = 1; 5896 break; 5897 } 5898 5899 txd = hn_txdesc_get(txr); 5900 if (txd == NULL) { 5901 txr->hn_no_txdescs++; 5902 drbr_putback(ifp, txr->hn_mbuf_br, m_head); 5903 txr->hn_oactive = 1; 5904 break; 5905 } 5906 5907 error = hn_encap(ifp, txr, txd, &m_head); 5908 if (error) { 5909 /* Both txd and m_head are freed; discard */ 5910 KASSERT(txr->hn_agg_txd == NULL, 5911 ("encap failed w/ pending aggregating txdesc")); 5912 drbr_advance(ifp, txr->hn_mbuf_br); 5913 continue; 5914 } 5915 5916 if (txr->hn_agg_pktleft == 0) { 5917 if (txr->hn_agg_txd != NULL) { 5918 KASSERT(m_head == NULL, 5919 ("pending mbuf for aggregating txdesc")); 5920 error = hn_flush_txagg(ifp, txr); 5921 if (__predict_false(error)) { 5922 txr->hn_oactive = 1; 5923 break; 5924 } 5925 } else { 5926 KASSERT(m_head != NULL, ("mbuf was freed")); 5927 error = hn_txpkt(ifp, txr, txd); 5928 if (__predict_false(error)) { 5929 /* txd is freed, but m_head is not */ 5930 drbr_putback(ifp, txr->hn_mbuf_br, 5931 m_head); 5932 txr->hn_oactive = 1; 5933 break; 5934 } 5935 } 5936 } 5937 #ifdef INVARIANTS 5938 else { 5939 KASSERT(txr->hn_agg_txd != NULL, 5940 ("no aggregating txdesc")); 5941 KASSERT(m_head == NULL, 5942 ("pending mbuf for aggregating txdesc")); 5943 } 5944 #endif 5945 5946 /* Sent */ 5947 drbr_advance(ifp, txr->hn_mbuf_br); 5948 } 5949 5950 /* Flush pending aggerated transmission. */ 5951 if (txr->hn_agg_txd != NULL) 5952 hn_flush_txagg(ifp, txr); 5953 return (sched); 5954 } 5955 5956 static int 5957 hn_transmit(if_t ifp, struct mbuf *m) 5958 { 5959 struct hn_softc *sc = if_getsoftc(ifp); 5960 struct hn_tx_ring *txr; 5961 int error, idx = 0; 5962 5963 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) { 5964 struct rm_priotracker pt; 5965 5966 rm_rlock(&sc->hn_vf_lock, &pt); 5967 if (__predict_true(sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 5968 struct mbuf *m_bpf = NULL; 5969 int obytes, omcast; 5970 5971 obytes = m->m_pkthdr.len; 5972 omcast = (m->m_flags & M_MCAST) != 0; 5973 5974 if (sc->hn_xvf_flags & HN_XVFFLAG_ACCBPF) { 5975 if (bpf_peers_present_if(ifp)) { 5976 m_bpf = m_copypacket(m, M_NOWAIT); 5977 if (m_bpf == NULL) { 5978 /* 5979 * Failed to grab a shallow 5980 * copy; tap now. 5981 */ 5982 ETHER_BPF_MTAP(ifp, m); 5983 } 5984 } 5985 } else { 5986 ETHER_BPF_MTAP(ifp, m); 5987 } 5988 5989 error = if_transmit(sc->hn_vf_ifp, m); 5990 rm_runlock(&sc->hn_vf_lock, &pt); 5991 5992 if (m_bpf != NULL) { 5993 if (!error) 5994 ETHER_BPF_MTAP(ifp, m_bpf); 5995 m_freem(m_bpf); 5996 } 5997 5998 if (error == ENOBUFS) { 5999 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6000 } else if (error) { 6001 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6002 } else { 6003 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 6004 if_inc_counter(ifp, IFCOUNTER_OBYTES, obytes); 6005 if (omcast) { 6006 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 6007 omcast); 6008 } 6009 } 6010 return (error); 6011 } 6012 rm_runlock(&sc->hn_vf_lock, &pt); 6013 } 6014 6015 #if defined(INET6) || defined(INET) 6016 /* 6017 * Perform TSO packet header fixup or get l2/l3 header length now, 6018 * since packet headers should be cache-hot. 6019 */ 6020 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 6021 m = hn_tso_fixup(m); 6022 if (__predict_false(m == NULL)) { 6023 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6024 return EIO; 6025 } 6026 } else if (m->m_pkthdr.csum_flags & 6027 (CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP6_UDP | CSUM_IP6_TCP)) { 6028 m = hn_set_hlen(m); 6029 if (__predict_false(m == NULL)) { 6030 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6031 return EIO; 6032 } 6033 } 6034 #endif 6035 6036 /* 6037 * Select the TX ring based on flowid 6038 */ 6039 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 6040 #ifdef RSS 6041 uint32_t bid; 6042 6043 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 6044 &bid) == 0) 6045 idx = bid % sc->hn_tx_ring_inuse; 6046 else 6047 #endif 6048 { 6049 #if defined(INET6) || defined(INET) 6050 int tcpsyn = 0; 6051 6052 if (m->m_pkthdr.len < 128 && 6053 (m->m_pkthdr.csum_flags & 6054 (CSUM_IP_TCP | CSUM_IP6_TCP)) && 6055 (m->m_pkthdr.csum_flags & CSUM_TSO) == 0) { 6056 m = hn_check_tcpsyn(m, &tcpsyn); 6057 if (__predict_false(m == NULL)) { 6058 if_inc_counter(ifp, 6059 IFCOUNTER_OERRORS, 1); 6060 return (EIO); 6061 } 6062 } 6063 #else 6064 const int tcpsyn = 0; 6065 #endif 6066 if (tcpsyn) 6067 idx = 0; 6068 else 6069 idx = m->m_pkthdr.flowid % sc->hn_tx_ring_inuse; 6070 } 6071 } 6072 txr = &sc->hn_tx_ring[idx]; 6073 6074 error = drbr_enqueue(ifp, txr->hn_mbuf_br, m); 6075 if (error) { 6076 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6077 return error; 6078 } 6079 6080 if (txr->hn_oactive) 6081 return 0; 6082 6083 if (txr->hn_sched_tx) 6084 goto do_sched; 6085 6086 if (mtx_trylock(&txr->hn_tx_lock)) { 6087 int sched; 6088 6089 sched = hn_xmit(txr, txr->hn_direct_tx_size); 6090 mtx_unlock(&txr->hn_tx_lock); 6091 if (!sched) 6092 return 0; 6093 } 6094 do_sched: 6095 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); 6096 return 0; 6097 } 6098 6099 static void 6100 hn_tx_ring_qflush(struct hn_tx_ring *txr) 6101 { 6102 struct mbuf *m; 6103 6104 mtx_lock(&txr->hn_tx_lock); 6105 while ((m = buf_ring_dequeue_sc(txr->hn_mbuf_br)) != NULL) 6106 m_freem(m); 6107 mtx_unlock(&txr->hn_tx_lock); 6108 } 6109 6110 static void 6111 hn_xmit_qflush(if_t ifp) 6112 { 6113 struct hn_softc *sc = if_getsoftc(ifp); 6114 struct rm_priotracker pt; 6115 int i; 6116 6117 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 6118 hn_tx_ring_qflush(&sc->hn_tx_ring[i]); 6119 if_qflush(ifp); 6120 6121 rm_rlock(&sc->hn_vf_lock, &pt); 6122 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 6123 if_qflush(sc->hn_vf_ifp); 6124 rm_runlock(&sc->hn_vf_lock, &pt); 6125 } 6126 6127 static void 6128 hn_xmit_txeof(struct hn_tx_ring *txr) 6129 { 6130 6131 if (txr->hn_sched_tx) 6132 goto do_sched; 6133 6134 if (mtx_trylock(&txr->hn_tx_lock)) { 6135 int sched; 6136 6137 txr->hn_oactive = 0; 6138 sched = hn_xmit(txr, txr->hn_direct_tx_size); 6139 mtx_unlock(&txr->hn_tx_lock); 6140 if (sched) { 6141 taskqueue_enqueue(txr->hn_tx_taskq, 6142 &txr->hn_tx_task); 6143 } 6144 } else { 6145 do_sched: 6146 /* 6147 * Release the oactive earlier, with the hope, that 6148 * others could catch up. The task will clear the 6149 * oactive again with the hn_tx_lock to avoid possible 6150 * races. 6151 */ 6152 txr->hn_oactive = 0; 6153 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 6154 } 6155 } 6156 6157 static void 6158 hn_xmit_taskfunc(void *xtxr, int pending __unused) 6159 { 6160 struct hn_tx_ring *txr = xtxr; 6161 6162 mtx_lock(&txr->hn_tx_lock); 6163 hn_xmit(txr, 0); 6164 mtx_unlock(&txr->hn_tx_lock); 6165 } 6166 6167 static void 6168 hn_xmit_txeof_taskfunc(void *xtxr, int pending __unused) 6169 { 6170 struct hn_tx_ring *txr = xtxr; 6171 6172 mtx_lock(&txr->hn_tx_lock); 6173 txr->hn_oactive = 0; 6174 hn_xmit(txr, 0); 6175 mtx_unlock(&txr->hn_tx_lock); 6176 } 6177 6178 static int 6179 hn_chan_attach(struct hn_softc *sc, struct vmbus_channel *chan) 6180 { 6181 struct vmbus_chan_br cbr; 6182 struct hn_rx_ring *rxr; 6183 struct hn_tx_ring *txr = NULL; 6184 int idx, error; 6185 6186 idx = vmbus_chan_subidx(chan); 6187 6188 /* 6189 * Link this channel to RX/TX ring. 6190 */ 6191 KASSERT(idx >= 0 && idx < sc->hn_rx_ring_inuse, 6192 ("invalid channel index %d, should > 0 && < %d", 6193 idx, sc->hn_rx_ring_inuse)); 6194 rxr = &sc->hn_rx_ring[idx]; 6195 KASSERT((rxr->hn_rx_flags & HN_RX_FLAG_ATTACHED) == 0, 6196 ("RX ring %d already attached", idx)); 6197 rxr->hn_rx_flags |= HN_RX_FLAG_ATTACHED; 6198 rxr->hn_chan = chan; 6199 6200 if (bootverbose) { 6201 if_printf(sc->hn_ifp, "link RX ring %d to chan%u\n", 6202 idx, vmbus_chan_id(chan)); 6203 } 6204 6205 if (idx < sc->hn_tx_ring_inuse) { 6206 txr = &sc->hn_tx_ring[idx]; 6207 KASSERT((txr->hn_tx_flags & HN_TX_FLAG_ATTACHED) == 0, 6208 ("TX ring %d already attached", idx)); 6209 txr->hn_tx_flags |= HN_TX_FLAG_ATTACHED; 6210 6211 txr->hn_chan = chan; 6212 if (bootverbose) { 6213 if_printf(sc->hn_ifp, "link TX ring %d to chan%u\n", 6214 idx, vmbus_chan_id(chan)); 6215 } 6216 } 6217 6218 /* Bind this channel to a proper CPU. */ 6219 vmbus_chan_cpu_set(chan, HN_RING_IDX2CPU(sc, idx)); 6220 6221 /* 6222 * Open this channel 6223 */ 6224 cbr.cbr = rxr->hn_br; 6225 cbr.cbr_paddr = pmap_kextract((vm_offset_t)rxr->hn_br); 6226 cbr.cbr_txsz = HN_TXBR_SIZE; 6227 cbr.cbr_rxsz = HN_RXBR_SIZE; 6228 error = vmbus_chan_open_br(chan, &cbr, NULL, 0, hn_chan_callback, rxr); 6229 if (error) { 6230 if (error == EISCONN) { 6231 if_printf(sc->hn_ifp, "bufring is connected after " 6232 "chan%u open failure\n", vmbus_chan_id(chan)); 6233 rxr->hn_rx_flags |= HN_RX_FLAG_BR_REF; 6234 } else { 6235 if_printf(sc->hn_ifp, "open chan%u failed: %d\n", 6236 vmbus_chan_id(chan), error); 6237 } 6238 } 6239 return (error); 6240 } 6241 6242 static void 6243 hn_chan_detach(struct hn_softc *sc, struct vmbus_channel *chan) 6244 { 6245 struct hn_rx_ring *rxr; 6246 int idx, error; 6247 6248 idx = vmbus_chan_subidx(chan); 6249 6250 /* 6251 * Link this channel to RX/TX ring. 6252 */ 6253 KASSERT(idx >= 0 && idx < sc->hn_rx_ring_inuse, 6254 ("invalid channel index %d, should > 0 && < %d", 6255 idx, sc->hn_rx_ring_inuse)); 6256 rxr = &sc->hn_rx_ring[idx]; 6257 KASSERT((rxr->hn_rx_flags & HN_RX_FLAG_ATTACHED), 6258 ("RX ring %d is not attached", idx)); 6259 rxr->hn_rx_flags &= ~HN_RX_FLAG_ATTACHED; 6260 6261 if (idx < sc->hn_tx_ring_inuse) { 6262 struct hn_tx_ring *txr = &sc->hn_tx_ring[idx]; 6263 6264 KASSERT((txr->hn_tx_flags & HN_TX_FLAG_ATTACHED), 6265 ("TX ring %d is not attached attached", idx)); 6266 txr->hn_tx_flags &= ~HN_TX_FLAG_ATTACHED; 6267 } 6268 6269 /* 6270 * Close this channel. 6271 * 6272 * NOTE: 6273 * Channel closing does _not_ destroy the target channel. 6274 */ 6275 error = vmbus_chan_close_direct(chan); 6276 if (error == EISCONN) { 6277 if_printf(sc->hn_ifp, "chan%u bufring is connected " 6278 "after being closed\n", vmbus_chan_id(chan)); 6279 rxr->hn_rx_flags |= HN_RX_FLAG_BR_REF; 6280 } else if (error) { 6281 if_printf(sc->hn_ifp, "chan%u close failed: %d\n", 6282 vmbus_chan_id(chan), error); 6283 } 6284 } 6285 6286 static int 6287 hn_attach_subchans(struct hn_softc *sc) 6288 { 6289 struct vmbus_channel **subchans; 6290 int subchan_cnt = sc->hn_rx_ring_inuse - 1; 6291 int i, error = 0; 6292 6293 KASSERT(subchan_cnt > 0, ("no sub-channels")); 6294 6295 /* Attach the sub-channels. */ 6296 subchans = vmbus_subchan_get(sc->hn_prichan, subchan_cnt); 6297 for (i = 0; i < subchan_cnt; ++i) { 6298 int error1; 6299 6300 error1 = hn_chan_attach(sc, subchans[i]); 6301 if (error1) { 6302 error = error1; 6303 /* Move on; all channels will be detached later. */ 6304 } 6305 } 6306 vmbus_subchan_rel(subchans, subchan_cnt); 6307 6308 if (error) { 6309 if_printf(sc->hn_ifp, "sub-channels attach failed: %d\n", error); 6310 } else { 6311 if (bootverbose) { 6312 if_printf(sc->hn_ifp, "%d sub-channels attached\n", 6313 subchan_cnt); 6314 } 6315 } 6316 return (error); 6317 } 6318 6319 static void 6320 hn_detach_allchans(struct hn_softc *sc) 6321 { 6322 struct vmbus_channel **subchans; 6323 int subchan_cnt = sc->hn_rx_ring_inuse - 1; 6324 int i; 6325 6326 if (subchan_cnt == 0) 6327 goto back; 6328 6329 /* Detach the sub-channels. */ 6330 subchans = vmbus_subchan_get(sc->hn_prichan, subchan_cnt); 6331 for (i = 0; i < subchan_cnt; ++i) 6332 hn_chan_detach(sc, subchans[i]); 6333 vmbus_subchan_rel(subchans, subchan_cnt); 6334 6335 back: 6336 /* 6337 * Detach the primary channel, _after_ all sub-channels 6338 * are detached. 6339 */ 6340 hn_chan_detach(sc, sc->hn_prichan); 6341 6342 /* Wait for sub-channels to be destroyed, if any. */ 6343 vmbus_subchan_drain(sc->hn_prichan); 6344 6345 #ifdef INVARIANTS 6346 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 6347 KASSERT((sc->hn_rx_ring[i].hn_rx_flags & 6348 HN_RX_FLAG_ATTACHED) == 0, 6349 ("%dth RX ring is still attached", i)); 6350 } 6351 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 6352 KASSERT((sc->hn_tx_ring[i].hn_tx_flags & 6353 HN_TX_FLAG_ATTACHED) == 0, 6354 ("%dth TX ring is still attached", i)); 6355 } 6356 #endif 6357 } 6358 6359 static int 6360 hn_synth_alloc_subchans(struct hn_softc *sc, int *nsubch) 6361 { 6362 struct vmbus_channel **subchans; 6363 int nchan, rxr_cnt, error; 6364 6365 nchan = *nsubch + 1; 6366 if (nchan == 1) { 6367 /* 6368 * Multiple RX/TX rings are not requested. 6369 */ 6370 *nsubch = 0; 6371 return (0); 6372 } 6373 6374 /* 6375 * Query RSS capabilities, e.g. # of RX rings, and # of indirect 6376 * table entries. 6377 */ 6378 error = hn_rndis_query_rsscaps(sc, &rxr_cnt); 6379 if (error) { 6380 /* No RSS; this is benign. */ 6381 *nsubch = 0; 6382 return (0); 6383 } 6384 if (bootverbose) { 6385 if_printf(sc->hn_ifp, "RX rings offered %u, requested %d\n", 6386 rxr_cnt, nchan); 6387 } 6388 6389 if (nchan > rxr_cnt) 6390 nchan = rxr_cnt; 6391 if (nchan == 1) { 6392 if_printf(sc->hn_ifp, "only 1 channel is supported, no vRSS\n"); 6393 *nsubch = 0; 6394 return (0); 6395 } 6396 6397 /* 6398 * Allocate sub-channels from NVS. 6399 */ 6400 *nsubch = nchan - 1; 6401 error = hn_nvs_alloc_subchans(sc, nsubch); 6402 if (error || *nsubch == 0) { 6403 /* Failed to allocate sub-channels. */ 6404 *nsubch = 0; 6405 return (0); 6406 } 6407 6408 /* 6409 * Wait for all sub-channels to become ready before moving on. 6410 */ 6411 subchans = vmbus_subchan_get(sc->hn_prichan, *nsubch); 6412 vmbus_subchan_rel(subchans, *nsubch); 6413 return (0); 6414 } 6415 6416 static bool 6417 hn_synth_attachable(const struct hn_softc *sc) 6418 { 6419 int i; 6420 6421 if (sc->hn_flags & HN_FLAG_ERRORS) 6422 return (false); 6423 6424 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 6425 const struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 6426 6427 if (rxr->hn_rx_flags & HN_RX_FLAG_BR_REF) 6428 return (false); 6429 } 6430 return (true); 6431 } 6432 6433 /* 6434 * Make sure that the RX filter is zero after the successful 6435 * RNDIS initialization. 6436 * 6437 * NOTE: 6438 * Under certain conditions on certain versions of Hyper-V, 6439 * the RNDIS rxfilter is _not_ zero on the hypervisor side 6440 * after the successful RNDIS initialization, which breaks 6441 * the assumption of any following code (well, it breaks the 6442 * RNDIS API contract actually). Clear the RNDIS rxfilter 6443 * explicitly, drain packets sneaking through, and drain the 6444 * interrupt taskqueues scheduled due to the stealth packets. 6445 */ 6446 static void 6447 hn_rndis_init_fixat(struct hn_softc *sc, int nchan) 6448 { 6449 6450 hn_disable_rx(sc); 6451 hn_drain_rxtx(sc, nchan); 6452 } 6453 6454 static int 6455 hn_synth_attach(struct hn_softc *sc, int mtu) 6456 { 6457 #define ATTACHED_NVS 0x0002 6458 #define ATTACHED_RNDIS 0x0004 6459 6460 struct ndis_rssprm_toeplitz *rss = &sc->hn_rss; 6461 int error, nsubch, nchan = 1, i, rndis_inited; 6462 uint32_t old_caps, attached = 0; 6463 6464 KASSERT((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0, 6465 ("synthetic parts were attached")); 6466 6467 if (!hn_synth_attachable(sc)) 6468 return (ENXIO); 6469 6470 /* Save capabilities for later verification. */ 6471 old_caps = sc->hn_caps; 6472 sc->hn_caps = 0; 6473 6474 /* Clear RSS stuffs. */ 6475 sc->hn_rss_ind_size = 0; 6476 sc->hn_rss_hash = 0; 6477 sc->hn_rss_hcap = 0; 6478 6479 /* 6480 * Attach the primary channel _before_ attaching NVS and RNDIS. 6481 */ 6482 error = hn_chan_attach(sc, sc->hn_prichan); 6483 if (error) 6484 goto failed; 6485 6486 /* 6487 * Attach NVS. 6488 */ 6489 error = hn_nvs_attach(sc, mtu); 6490 if (error) 6491 goto failed; 6492 attached |= ATTACHED_NVS; 6493 6494 /* 6495 * Attach RNDIS _after_ NVS is attached. 6496 */ 6497 error = hn_rndis_attach(sc, mtu, &rndis_inited); 6498 if (rndis_inited) 6499 attached |= ATTACHED_RNDIS; 6500 if (error) 6501 goto failed; 6502 6503 /* 6504 * Make sure capabilities are not changed. 6505 */ 6506 if (device_is_attached(sc->hn_dev) && old_caps != sc->hn_caps) { 6507 if_printf(sc->hn_ifp, "caps mismatch old 0x%08x, new 0x%08x\n", 6508 old_caps, sc->hn_caps); 6509 error = ENXIO; 6510 goto failed; 6511 } 6512 6513 /* 6514 * Allocate sub-channels for multi-TX/RX rings. 6515 * 6516 * NOTE: 6517 * The # of RX rings that can be used is equivalent to the # of 6518 * channels to be requested. 6519 */ 6520 nsubch = sc->hn_rx_ring_cnt - 1; 6521 error = hn_synth_alloc_subchans(sc, &nsubch); 6522 if (error) 6523 goto failed; 6524 /* NOTE: _Full_ synthetic parts detach is required now. */ 6525 sc->hn_flags |= HN_FLAG_SYNTH_ATTACHED; 6526 6527 /* 6528 * Set the # of TX/RX rings that could be used according to 6529 * the # of channels that NVS offered. 6530 */ 6531 nchan = nsubch + 1; 6532 hn_set_ring_inuse(sc, nchan); 6533 if (nchan == 1) { 6534 /* Only the primary channel can be used; done */ 6535 goto back; 6536 } 6537 6538 /* 6539 * Attach the sub-channels. 6540 * 6541 * NOTE: hn_set_ring_inuse() _must_ have been called. 6542 */ 6543 error = hn_attach_subchans(sc); 6544 if (error) 6545 goto failed; 6546 6547 /* 6548 * Configure RSS key and indirect table _after_ all sub-channels 6549 * are attached. 6550 */ 6551 if ((sc->hn_flags & HN_FLAG_HAS_RSSKEY) == 0) { 6552 /* 6553 * RSS key is not set yet; set it to the default RSS key. 6554 */ 6555 if (bootverbose) 6556 if_printf(sc->hn_ifp, "setup default RSS key\n"); 6557 #ifdef RSS 6558 rss_getkey(rss->rss_key); 6559 #else 6560 memcpy(rss->rss_key, hn_rss_key_default, sizeof(rss->rss_key)); 6561 #endif 6562 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 6563 } 6564 6565 if ((sc->hn_flags & HN_FLAG_HAS_RSSIND) == 0) { 6566 /* 6567 * RSS indirect table is not set yet; set it up in round- 6568 * robin fashion. 6569 */ 6570 if (bootverbose) { 6571 if_printf(sc->hn_ifp, "setup default RSS indirect " 6572 "table\n"); 6573 } 6574 for (i = 0; i < NDIS_HASH_INDCNT; ++i) { 6575 uint32_t subidx; 6576 6577 #ifdef RSS 6578 subidx = rss_get_indirection_to_bucket(i); 6579 #else 6580 subidx = i; 6581 #endif 6582 rss->rss_ind[i] = subidx % nchan; 6583 } 6584 sc->hn_flags |= HN_FLAG_HAS_RSSIND; 6585 } else { 6586 /* 6587 * # of usable channels may be changed, so we have to 6588 * make sure that all entries in RSS indirect table 6589 * are valid. 6590 * 6591 * NOTE: hn_set_ring_inuse() _must_ have been called. 6592 */ 6593 hn_rss_ind_fixup(sc); 6594 } 6595 6596 sc->hn_rss_hash = sc->hn_rss_hcap; 6597 if ((sc->hn_flags & HN_FLAG_RXVF) || 6598 (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 6599 /* NOTE: Don't reconfigure RSS; will do immediately. */ 6600 hn_vf_rss_fixup(sc, false); 6601 } 6602 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE); 6603 if (error) 6604 goto failed; 6605 back: 6606 /* 6607 * Fixup transmission aggregation setup. 6608 */ 6609 hn_set_txagg(sc); 6610 hn_rndis_init_fixat(sc, nchan); 6611 return (0); 6612 6613 failed: 6614 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 6615 hn_rndis_init_fixat(sc, nchan); 6616 hn_synth_detach(sc); 6617 } else { 6618 if (attached & ATTACHED_RNDIS) { 6619 hn_rndis_init_fixat(sc, nchan); 6620 hn_rndis_detach(sc); 6621 } 6622 if (attached & ATTACHED_NVS) 6623 hn_nvs_detach(sc); 6624 hn_chan_detach(sc, sc->hn_prichan); 6625 /* Restore old capabilities. */ 6626 sc->hn_caps = old_caps; 6627 } 6628 return (error); 6629 6630 #undef ATTACHED_RNDIS 6631 #undef ATTACHED_NVS 6632 } 6633 6634 /* 6635 * NOTE: 6636 * The interface must have been suspended though hn_suspend(), before 6637 * this function get called. 6638 */ 6639 static void 6640 hn_synth_detach(struct hn_softc *sc) 6641 { 6642 6643 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 6644 ("synthetic parts were not attached")); 6645 6646 /* Detach the RNDIS first. */ 6647 hn_rndis_detach(sc); 6648 6649 /* Detach NVS. */ 6650 hn_nvs_detach(sc); 6651 6652 /* Detach all of the channels. */ 6653 hn_detach_allchans(sc); 6654 6655 if (vmbus_current_version >= VMBUS_VERSION_WIN10 && sc->hn_rxbuf_gpadl != 0) { 6656 /* 6657 * Host is post-Win2016, disconnect RXBUF from primary channel here. 6658 */ 6659 int error; 6660 6661 error = vmbus_chan_gpadl_disconnect(sc->hn_prichan, 6662 sc->hn_rxbuf_gpadl); 6663 if (error) { 6664 if_printf(sc->hn_ifp, 6665 "rxbuf gpadl disconn failed: %d\n", error); 6666 sc->hn_flags |= HN_FLAG_RXBUF_REF; 6667 } 6668 sc->hn_rxbuf_gpadl = 0; 6669 } 6670 6671 if (vmbus_current_version >= VMBUS_VERSION_WIN10 && sc->hn_chim_gpadl != 0) { 6672 /* 6673 * Host is post-Win2016, disconnect chimney sending buffer from 6674 * primary channel here. 6675 */ 6676 int error; 6677 6678 error = vmbus_chan_gpadl_disconnect(sc->hn_prichan, 6679 sc->hn_chim_gpadl); 6680 if (error) { 6681 if_printf(sc->hn_ifp, 6682 "chim gpadl disconn failed: %d\n", error); 6683 sc->hn_flags |= HN_FLAG_CHIM_REF; 6684 } 6685 sc->hn_chim_gpadl = 0; 6686 } 6687 sc->hn_flags &= ~HN_FLAG_SYNTH_ATTACHED; 6688 } 6689 6690 static void 6691 hn_set_ring_inuse(struct hn_softc *sc, int ring_cnt) 6692 { 6693 KASSERT(ring_cnt > 0 && ring_cnt <= sc->hn_rx_ring_cnt, 6694 ("invalid ring count %d", ring_cnt)); 6695 6696 if (sc->hn_tx_ring_cnt > ring_cnt) 6697 sc->hn_tx_ring_inuse = ring_cnt; 6698 else 6699 sc->hn_tx_ring_inuse = sc->hn_tx_ring_cnt; 6700 sc->hn_rx_ring_inuse = ring_cnt; 6701 6702 #ifdef RSS 6703 if (sc->hn_rx_ring_inuse != rss_getnumbuckets()) { 6704 if_printf(sc->hn_ifp, "# of RX rings (%d) does not match " 6705 "# of RSS buckets (%d)\n", sc->hn_rx_ring_inuse, 6706 rss_getnumbuckets()); 6707 } 6708 #endif 6709 6710 if (bootverbose) { 6711 if_printf(sc->hn_ifp, "%d TX ring, %d RX ring\n", 6712 sc->hn_tx_ring_inuse, sc->hn_rx_ring_inuse); 6713 } 6714 } 6715 6716 static void 6717 hn_chan_drain(struct hn_softc *sc, struct vmbus_channel *chan) 6718 { 6719 6720 /* 6721 * NOTE: 6722 * The TX bufring will not be drained by the hypervisor, 6723 * if the primary channel is revoked. 6724 */ 6725 while (!vmbus_chan_rx_empty(chan) || 6726 (!vmbus_chan_is_revoked(sc->hn_prichan) && 6727 !vmbus_chan_tx_empty(chan))) 6728 pause("waitch", 1); 6729 vmbus_chan_intr_drain(chan); 6730 } 6731 6732 static void 6733 hn_disable_rx(struct hn_softc *sc) 6734 { 6735 6736 /* 6737 * Disable RX by clearing RX filter forcefully. 6738 */ 6739 sc->hn_rx_filter = NDIS_PACKET_TYPE_NONE; 6740 hn_rndis_set_rxfilter(sc, sc->hn_rx_filter); /* ignore error */ 6741 6742 /* 6743 * Give RNDIS enough time to flush all pending data packets. 6744 */ 6745 pause("waitrx", (200 * hz) / 1000); 6746 } 6747 6748 /* 6749 * NOTE: 6750 * RX/TX _must_ have been suspended/disabled, before this function 6751 * is called. 6752 */ 6753 static void 6754 hn_drain_rxtx(struct hn_softc *sc, int nchan) 6755 { 6756 struct vmbus_channel **subch = NULL; 6757 int nsubch; 6758 6759 /* 6760 * Drain RX/TX bufrings and interrupts. 6761 */ 6762 nsubch = nchan - 1; 6763 if (nsubch > 0) 6764 subch = vmbus_subchan_get(sc->hn_prichan, nsubch); 6765 6766 if (subch != NULL) { 6767 int i; 6768 6769 for (i = 0; i < nsubch; ++i) 6770 hn_chan_drain(sc, subch[i]); 6771 } 6772 hn_chan_drain(sc, sc->hn_prichan); 6773 6774 if (subch != NULL) 6775 vmbus_subchan_rel(subch, nsubch); 6776 } 6777 6778 static void 6779 hn_suspend_data(struct hn_softc *sc) 6780 { 6781 struct hn_tx_ring *txr; 6782 int i; 6783 6784 HN_LOCK_ASSERT(sc); 6785 6786 /* 6787 * Suspend TX. 6788 */ 6789 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 6790 txr = &sc->hn_tx_ring[i]; 6791 6792 mtx_lock(&txr->hn_tx_lock); 6793 txr->hn_suspended = 1; 6794 mtx_unlock(&txr->hn_tx_lock); 6795 /* No one is able send more packets now. */ 6796 6797 /* 6798 * Wait for all pending sends to finish. 6799 * 6800 * NOTE: 6801 * We will _not_ receive all pending send-done, if the 6802 * primary channel is revoked. 6803 */ 6804 while (hn_tx_ring_pending(txr) && 6805 !vmbus_chan_is_revoked(sc->hn_prichan)) 6806 pause("hnwtx", 1 /* 1 tick */); 6807 } 6808 6809 /* 6810 * Disable RX. 6811 */ 6812 hn_disable_rx(sc); 6813 6814 /* 6815 * Drain RX/TX. 6816 */ 6817 hn_drain_rxtx(sc, sc->hn_rx_ring_inuse); 6818 6819 /* 6820 * Drain any pending TX tasks. 6821 * 6822 * NOTE: 6823 * The above hn_drain_rxtx() can dispatch TX tasks, so the TX 6824 * tasks will have to be drained _after_ the above hn_drain_rxtx(). 6825 */ 6826 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 6827 txr = &sc->hn_tx_ring[i]; 6828 6829 taskqueue_drain(txr->hn_tx_taskq, &txr->hn_tx_task); 6830 taskqueue_drain(txr->hn_tx_taskq, &txr->hn_txeof_task); 6831 } 6832 } 6833 6834 static void 6835 hn_suspend_mgmt_taskfunc(void *xsc, int pending __unused) 6836 { 6837 6838 ((struct hn_softc *)xsc)->hn_mgmt_taskq = NULL; 6839 } 6840 6841 static void 6842 hn_suspend_mgmt(struct hn_softc *sc) 6843 { 6844 struct task task; 6845 6846 HN_LOCK_ASSERT(sc); 6847 6848 /* 6849 * Make sure that hn_mgmt_taskq0 can nolonger be accessed 6850 * through hn_mgmt_taskq. 6851 */ 6852 TASK_INIT(&task, 0, hn_suspend_mgmt_taskfunc, sc); 6853 vmbus_chan_run_task(sc->hn_prichan, &task); 6854 6855 /* 6856 * Make sure that all pending management tasks are completed. 6857 */ 6858 taskqueue_drain(sc->hn_mgmt_taskq0, &sc->hn_netchg_init); 6859 taskqueue_drain_timeout(sc->hn_mgmt_taskq0, &sc->hn_netchg_status); 6860 taskqueue_drain_all(sc->hn_mgmt_taskq0); 6861 } 6862 6863 static void 6864 hn_suspend(struct hn_softc *sc) 6865 { 6866 6867 /* Disable polling. */ 6868 hn_polling(sc, 0); 6869 6870 /* 6871 * If the non-transparent mode VF is activated, the synthetic 6872 * device is receiving packets, so the data path of the 6873 * synthetic device must be suspended. 6874 */ 6875 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) || 6876 (sc->hn_flags & HN_FLAG_RXVF)) 6877 hn_suspend_data(sc); 6878 hn_suspend_mgmt(sc); 6879 } 6880 6881 static void 6882 hn_resume_tx(struct hn_softc *sc, int tx_ring_cnt) 6883 { 6884 int i; 6885 6886 KASSERT(tx_ring_cnt <= sc->hn_tx_ring_cnt, 6887 ("invalid TX ring count %d", tx_ring_cnt)); 6888 6889 for (i = 0; i < tx_ring_cnt; ++i) { 6890 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 6891 6892 mtx_lock(&txr->hn_tx_lock); 6893 txr->hn_suspended = 0; 6894 mtx_unlock(&txr->hn_tx_lock); 6895 } 6896 } 6897 6898 static void 6899 hn_resume_data(struct hn_softc *sc) 6900 { 6901 int i; 6902 6903 HN_LOCK_ASSERT(sc); 6904 6905 /* 6906 * Re-enable RX. 6907 */ 6908 hn_rxfilter_config(sc); 6909 6910 /* 6911 * Make sure to clear suspend status on "all" TX rings, 6912 * since hn_tx_ring_inuse can be changed after 6913 * hn_suspend_data(). 6914 */ 6915 hn_resume_tx(sc, sc->hn_tx_ring_cnt); 6916 6917 #ifdef HN_IFSTART_SUPPORT 6918 if (!hn_use_if_start) 6919 #endif 6920 { 6921 /* 6922 * Flush unused drbrs, since hn_tx_ring_inuse may be 6923 * reduced. 6924 */ 6925 for (i = sc->hn_tx_ring_inuse; i < sc->hn_tx_ring_cnt; ++i) 6926 hn_tx_ring_qflush(&sc->hn_tx_ring[i]); 6927 } 6928 6929 /* 6930 * Kick start TX. 6931 */ 6932 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 6933 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 6934 6935 /* 6936 * Use txeof task, so that any pending oactive can be 6937 * cleared properly. 6938 */ 6939 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 6940 } 6941 } 6942 6943 static void 6944 hn_resume_mgmt(struct hn_softc *sc) 6945 { 6946 6947 sc->hn_mgmt_taskq = sc->hn_mgmt_taskq0; 6948 6949 /* 6950 * Kick off network change detection, if it was pending. 6951 * If no network change was pending, start link status 6952 * checks, which is more lightweight than network change 6953 * detection. 6954 */ 6955 if (sc->hn_link_flags & HN_LINK_FLAG_NETCHG) 6956 hn_change_network(sc); 6957 else 6958 hn_update_link_status(sc); 6959 } 6960 6961 static void 6962 hn_resume(struct hn_softc *sc) 6963 { 6964 6965 /* 6966 * If the non-transparent mode VF is activated, the synthetic 6967 * device have to receive packets, so the data path of the 6968 * synthetic device must be resumed. 6969 */ 6970 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) || 6971 (sc->hn_flags & HN_FLAG_RXVF)) 6972 hn_resume_data(sc); 6973 6974 /* 6975 * Don't resume link status change if VF is attached/activated. 6976 * - In the non-transparent VF mode, the synthetic device marks 6977 * link down until the VF is deactivated; i.e. VF is down. 6978 * - In transparent VF mode, VF's media status is used until 6979 * the VF is detached. 6980 */ 6981 if ((sc->hn_flags & HN_FLAG_RXVF) == 0 && 6982 !(hn_xpnt_vf && sc->hn_vf_ifp != NULL)) 6983 hn_resume_mgmt(sc); 6984 6985 /* 6986 * Re-enable polling if this interface is running and 6987 * the polling is requested. 6988 */ 6989 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) && sc->hn_pollhz > 0) 6990 hn_polling(sc, sc->hn_pollhz); 6991 } 6992 6993 static void 6994 hn_rndis_rx_status(struct hn_softc *sc, const void *data, int dlen) 6995 { 6996 const struct rndis_status_msg *msg; 6997 int ofs; 6998 6999 if (dlen < sizeof(*msg)) { 7000 if_printf(sc->hn_ifp, "invalid RNDIS status\n"); 7001 return; 7002 } 7003 msg = data; 7004 7005 switch (msg->rm_status) { 7006 case RNDIS_STATUS_MEDIA_CONNECT: 7007 case RNDIS_STATUS_MEDIA_DISCONNECT: 7008 hn_update_link_status(sc); 7009 break; 7010 7011 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 7012 case RNDIS_STATUS_LINK_SPEED_CHANGE: 7013 /* Not really useful; ignore. */ 7014 break; 7015 7016 case RNDIS_STATUS_NETWORK_CHANGE: 7017 ofs = RNDIS_STBUFOFFSET_ABS(msg->rm_stbufoffset); 7018 if (dlen < ofs + msg->rm_stbuflen || 7019 msg->rm_stbuflen < sizeof(uint32_t)) { 7020 if_printf(sc->hn_ifp, "network changed\n"); 7021 } else { 7022 uint32_t change; 7023 7024 memcpy(&change, ((const uint8_t *)msg) + ofs, 7025 sizeof(change)); 7026 if_printf(sc->hn_ifp, "network changed, change %u\n", 7027 change); 7028 } 7029 hn_change_network(sc); 7030 break; 7031 7032 default: 7033 if_printf(sc->hn_ifp, "unknown RNDIS status 0x%08x\n", 7034 msg->rm_status); 7035 break; 7036 } 7037 } 7038 7039 static int 7040 hn_rndis_rxinfo(const void *info_data, int info_dlen, struct hn_rxinfo *info) 7041 { 7042 const struct rndis_pktinfo *pi = info_data; 7043 uint32_t mask = 0; 7044 7045 while (info_dlen != 0) { 7046 const void *data; 7047 uint32_t dlen; 7048 7049 if (__predict_false(info_dlen < sizeof(*pi))) 7050 return (EINVAL); 7051 if (__predict_false(info_dlen < pi->rm_size)) 7052 return (EINVAL); 7053 info_dlen -= pi->rm_size; 7054 7055 if (__predict_false(pi->rm_size & RNDIS_PKTINFO_SIZE_ALIGNMASK)) 7056 return (EINVAL); 7057 if (__predict_false(pi->rm_size < pi->rm_pktinfooffset)) 7058 return (EINVAL); 7059 dlen = pi->rm_size - pi->rm_pktinfooffset; 7060 data = pi->rm_data; 7061 7062 if (pi->rm_internal == 1) { 7063 switch (pi->rm_type) { 7064 case NDIS_PKTINFO_IT_PKTINFO_ID: 7065 if (__predict_false(dlen < NDIS_PKTINFOID_SZ)) 7066 return (EINVAL); 7067 info->pktinfo_id = 7068 (const struct packet_info_id *)data; 7069 mask |= HN_RXINFO_PKTINFO_ID; 7070 break; 7071 7072 default: 7073 goto next; 7074 } 7075 } else { 7076 switch (pi->rm_type) { 7077 case NDIS_PKTINFO_TYPE_VLAN: 7078 if (__predict_false(dlen 7079 < NDIS_VLAN_INFO_SIZE)) 7080 return (EINVAL); 7081 info->vlan_info = (const uint32_t *)data; 7082 mask |= HN_RXINFO_VLAN; 7083 break; 7084 7085 case NDIS_PKTINFO_TYPE_CSUM: 7086 if (__predict_false(dlen 7087 < NDIS_RXCSUM_INFO_SIZE)) 7088 return (EINVAL); 7089 info->csum_info = (const uint32_t *)data; 7090 mask |= HN_RXINFO_CSUM; 7091 break; 7092 7093 case HN_NDIS_PKTINFO_TYPE_HASHVAL: 7094 if (__predict_false(dlen 7095 < HN_NDIS_HASH_VALUE_SIZE)) 7096 return (EINVAL); 7097 info->hash_value = (const uint32_t *)data; 7098 mask |= HN_RXINFO_HASHVAL; 7099 break; 7100 7101 case HN_NDIS_PKTINFO_TYPE_HASHINF: 7102 if (__predict_false(dlen 7103 < HN_NDIS_HASH_INFO_SIZE)) 7104 return (EINVAL); 7105 info->hash_info = (const uint32_t *)data; 7106 mask |= HN_RXINFO_HASHINF; 7107 break; 7108 7109 default: 7110 goto next; 7111 } 7112 } 7113 7114 if (mask == HN_RXINFO_ALL) { 7115 /* All found; done */ 7116 break; 7117 } 7118 next: 7119 pi = (const struct rndis_pktinfo *) 7120 ((const uint8_t *)pi + pi->rm_size); 7121 } 7122 7123 /* 7124 * Final fixup. 7125 * - If there is no hash value, invalidate the hash info. 7126 */ 7127 if ((mask & HN_RXINFO_HASHVAL) == 0) 7128 info->hash_info = NULL; 7129 return (0); 7130 } 7131 7132 static __inline bool 7133 hn_rndis_check_overlap(int off, int len, int check_off, int check_len) 7134 { 7135 7136 if (off < check_off) { 7137 if (__predict_true(off + len <= check_off)) 7138 return (false); 7139 } else if (off > check_off) { 7140 if (__predict_true(check_off + check_len <= off)) 7141 return (false); 7142 } 7143 return (true); 7144 } 7145 7146 static __inline void 7147 hn_rsc_add_data(struct hn_rx_ring *rxr, const void *data, 7148 uint32_t len, struct hn_rxinfo *info) 7149 { 7150 uint32_t cnt = rxr->rsc.cnt; 7151 7152 if (cnt) { 7153 rxr->rsc.pktlen += len; 7154 } else { 7155 rxr->rsc.vlan_info = info->vlan_info; 7156 rxr->rsc.csum_info = info->csum_info; 7157 rxr->rsc.hash_info = info->hash_info; 7158 rxr->rsc.hash_value = info->hash_value; 7159 rxr->rsc.pktlen = len; 7160 } 7161 7162 rxr->rsc.frag_data[cnt] = data; 7163 rxr->rsc.frag_len[cnt] = len; 7164 rxr->rsc.cnt++; 7165 } 7166 7167 static void 7168 hn_rndis_rx_data(struct hn_rx_ring *rxr, const void *data, int dlen) 7169 { 7170 const struct rndis_packet_msg *pkt; 7171 struct hn_rxinfo info; 7172 int data_off, pktinfo_off, data_len, pktinfo_len; 7173 bool rsc_more= false; 7174 7175 /* 7176 * Check length. 7177 */ 7178 if (__predict_false(dlen < sizeof(*pkt))) { 7179 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg\n"); 7180 return; 7181 } 7182 pkt = data; 7183 7184 if (__predict_false(dlen < pkt->rm_len)) { 7185 if_printf(rxr->hn_ifp, "truncated RNDIS packet msg, " 7186 "dlen %d, msglen %u\n", dlen, pkt->rm_len); 7187 return; 7188 } 7189 if (__predict_false(pkt->rm_len < 7190 pkt->rm_datalen + pkt->rm_oobdatalen + pkt->rm_pktinfolen)) { 7191 if_printf(rxr->hn_ifp, "invalid RNDIS packet msglen, " 7192 "msglen %u, data %u, oob %u, pktinfo %u\n", 7193 pkt->rm_len, pkt->rm_datalen, pkt->rm_oobdatalen, 7194 pkt->rm_pktinfolen); 7195 return; 7196 } 7197 if (__predict_false(pkt->rm_datalen == 0)) { 7198 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, no data\n"); 7199 return; 7200 } 7201 7202 /* 7203 * Check offests. 7204 */ 7205 #define IS_OFFSET_INVALID(ofs) \ 7206 ((ofs) < RNDIS_PACKET_MSG_OFFSET_MIN || \ 7207 ((ofs) & RNDIS_PACKET_MSG_OFFSET_ALIGNMASK)) 7208 7209 /* XXX Hyper-V does not meet data offset alignment requirement */ 7210 if (__predict_false(pkt->rm_dataoffset < RNDIS_PACKET_MSG_OFFSET_MIN)) { 7211 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7212 "data offset %u\n", pkt->rm_dataoffset); 7213 return; 7214 } 7215 if (__predict_false(pkt->rm_oobdataoffset > 0 && 7216 IS_OFFSET_INVALID(pkt->rm_oobdataoffset))) { 7217 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7218 "oob offset %u\n", pkt->rm_oobdataoffset); 7219 return; 7220 } 7221 if (__predict_true(pkt->rm_pktinfooffset > 0) && 7222 __predict_false(IS_OFFSET_INVALID(pkt->rm_pktinfooffset))) { 7223 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7224 "pktinfo offset %u\n", pkt->rm_pktinfooffset); 7225 return; 7226 } 7227 7228 #undef IS_OFFSET_INVALID 7229 7230 data_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_dataoffset); 7231 data_len = pkt->rm_datalen; 7232 pktinfo_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_pktinfooffset); 7233 pktinfo_len = pkt->rm_pktinfolen; 7234 7235 /* 7236 * Check OOB coverage. 7237 */ 7238 if (__predict_false(pkt->rm_oobdatalen != 0)) { 7239 int oob_off, oob_len; 7240 7241 if_printf(rxr->hn_ifp, "got oobdata\n"); 7242 oob_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_oobdataoffset); 7243 oob_len = pkt->rm_oobdatalen; 7244 7245 if (__predict_false(oob_off + oob_len > pkt->rm_len)) { 7246 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7247 "oob overflow, msglen %u, oob abs %d len %d\n", 7248 pkt->rm_len, oob_off, oob_len); 7249 return; 7250 } 7251 7252 /* 7253 * Check against data. 7254 */ 7255 if (hn_rndis_check_overlap(oob_off, oob_len, 7256 data_off, data_len)) { 7257 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7258 "oob overlaps data, oob abs %d len %d, " 7259 "data abs %d len %d\n", 7260 oob_off, oob_len, data_off, data_len); 7261 return; 7262 } 7263 7264 /* 7265 * Check against pktinfo. 7266 */ 7267 if (pktinfo_len != 0 && 7268 hn_rndis_check_overlap(oob_off, oob_len, 7269 pktinfo_off, pktinfo_len)) { 7270 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7271 "oob overlaps pktinfo, oob abs %d len %d, " 7272 "pktinfo abs %d len %d\n", 7273 oob_off, oob_len, pktinfo_off, pktinfo_len); 7274 return; 7275 } 7276 } 7277 7278 /* 7279 * Check per-packet-info coverage and find useful per-packet-info. 7280 */ 7281 info.vlan_info = NULL; 7282 info.csum_info = NULL; 7283 info.hash_info = NULL; 7284 info.pktinfo_id = NULL; 7285 7286 if (__predict_true(pktinfo_len != 0)) { 7287 bool overlap; 7288 int error; 7289 7290 if (__predict_false(pktinfo_off + pktinfo_len > pkt->rm_len)) { 7291 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7292 "pktinfo overflow, msglen %u, " 7293 "pktinfo abs %d len %d\n", 7294 pkt->rm_len, pktinfo_off, pktinfo_len); 7295 return; 7296 } 7297 7298 /* 7299 * Check packet info coverage. 7300 */ 7301 overlap = hn_rndis_check_overlap(pktinfo_off, pktinfo_len, 7302 data_off, data_len); 7303 if (__predict_false(overlap)) { 7304 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7305 "pktinfo overlap data, pktinfo abs %d len %d, " 7306 "data abs %d len %d\n", 7307 pktinfo_off, pktinfo_len, data_off, data_len); 7308 return; 7309 } 7310 7311 /* 7312 * Find useful per-packet-info. 7313 */ 7314 error = hn_rndis_rxinfo(((const uint8_t *)pkt) + pktinfo_off, 7315 pktinfo_len, &info); 7316 if (__predict_false(error)) { 7317 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg " 7318 "pktinfo\n"); 7319 return; 7320 } 7321 } 7322 7323 if (__predict_false(data_off + data_len > pkt->rm_len)) { 7324 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7325 "data overflow, msglen %u, data abs %d len %d\n", 7326 pkt->rm_len, data_off, data_len); 7327 return; 7328 } 7329 7330 /* Identify RSC fragments, drop invalid packets */ 7331 if ((info.pktinfo_id != NULL) && 7332 (info.pktinfo_id->flag & HN_NDIS_PKTINFO_SUBALLOC)) { 7333 if (info.pktinfo_id->flag & HN_NDIS_PKTINFO_1ST_FRAG) { 7334 rxr->rsc.cnt = 0; 7335 rxr->hn_rsc_pkts++; 7336 } else if (rxr->rsc.cnt == 0) 7337 goto drop; 7338 7339 rsc_more = true; 7340 7341 if (info.pktinfo_id->flag & HN_NDIS_PKTINFO_LAST_FRAG) 7342 rsc_more = false; 7343 7344 if (rsc_more && rxr->rsc.is_last) 7345 goto drop; 7346 } else { 7347 rxr->rsc.cnt = 0; 7348 } 7349 7350 if (__predict_false(rxr->rsc.cnt >= HN_NVS_RSC_MAX)) 7351 goto drop; 7352 7353 /* Store data in per rx ring structure */ 7354 hn_rsc_add_data(rxr,((const uint8_t *)pkt) + data_off, 7355 data_len, &info); 7356 7357 if (rsc_more) 7358 return; 7359 7360 hn_rxpkt(rxr); 7361 rxr->rsc.cnt = 0; 7362 return; 7363 drop: 7364 rxr->hn_rsc_drop++; 7365 return; 7366 } 7367 7368 static __inline void 7369 hn_rndis_rxpkt(struct hn_rx_ring *rxr, const void *data, int dlen) 7370 { 7371 const struct rndis_msghdr *hdr; 7372 7373 if (__predict_false(dlen < sizeof(*hdr))) { 7374 if_printf(rxr->hn_ifp, "invalid RNDIS msg\n"); 7375 return; 7376 } 7377 hdr = data; 7378 7379 if (__predict_true(hdr->rm_type == REMOTE_NDIS_PACKET_MSG)) { 7380 /* Hot data path. */ 7381 hn_rndis_rx_data(rxr, data, dlen); 7382 /* Done! */ 7383 return; 7384 } 7385 7386 if (hdr->rm_type == REMOTE_NDIS_INDICATE_STATUS_MSG) 7387 hn_rndis_rx_status(if_getsoftc(rxr->hn_ifp), data, dlen); 7388 else 7389 hn_rndis_rx_ctrl(if_getsoftc(rxr->hn_ifp), data, dlen); 7390 } 7391 7392 static void 7393 hn_nvs_handle_notify(struct hn_softc *sc, const struct vmbus_chanpkt_hdr *pkt) 7394 { 7395 const struct hn_nvs_hdr *hdr; 7396 7397 if (VMBUS_CHANPKT_DATALEN(pkt) < sizeof(*hdr)) { 7398 if_printf(sc->hn_ifp, "invalid nvs notify\n"); 7399 return; 7400 } 7401 hdr = VMBUS_CHANPKT_CONST_DATA(pkt); 7402 7403 if (hdr->nvs_type == HN_NVS_TYPE_TXTBL_NOTE) { 7404 /* Useless; ignore */ 7405 return; 7406 } 7407 if_printf(sc->hn_ifp, "got notify, nvs type %u\n", hdr->nvs_type); 7408 } 7409 7410 static void 7411 hn_nvs_handle_comp(struct hn_softc *sc, struct vmbus_channel *chan, 7412 const struct vmbus_chanpkt_hdr *pkt) 7413 { 7414 struct hn_nvs_sendctx *sndc; 7415 7416 sndc = (struct hn_nvs_sendctx *)(uintptr_t)pkt->cph_xactid; 7417 sndc->hn_cb(sndc, sc, chan, VMBUS_CHANPKT_CONST_DATA(pkt), 7418 VMBUS_CHANPKT_DATALEN(pkt)); 7419 /* 7420 * NOTE: 7421 * 'sndc' CAN NOT be accessed anymore, since it can be freed by 7422 * its callback. 7423 */ 7424 } 7425 7426 static void 7427 hn_nvs_handle_rxbuf(struct hn_rx_ring *rxr, struct vmbus_channel *chan, 7428 const struct vmbus_chanpkt_hdr *pkthdr) 7429 { 7430 struct epoch_tracker et; 7431 const struct vmbus_chanpkt_rxbuf *pkt; 7432 const struct hn_nvs_hdr *nvs_hdr; 7433 int count, i, hlen; 7434 7435 if (__predict_false(VMBUS_CHANPKT_DATALEN(pkthdr) < sizeof(*nvs_hdr))) { 7436 if_printf(rxr->hn_ifp, "invalid nvs RNDIS\n"); 7437 return; 7438 } 7439 nvs_hdr = VMBUS_CHANPKT_CONST_DATA(pkthdr); 7440 7441 /* Make sure that this is a RNDIS message. */ 7442 if (__predict_false(nvs_hdr->nvs_type != HN_NVS_TYPE_RNDIS)) { 7443 if_printf(rxr->hn_ifp, "nvs type %u, not RNDIS\n", 7444 nvs_hdr->nvs_type); 7445 return; 7446 } 7447 7448 hlen = VMBUS_CHANPKT_GETLEN(pkthdr->cph_hlen); 7449 if (__predict_false(hlen < sizeof(*pkt))) { 7450 if_printf(rxr->hn_ifp, "invalid rxbuf chanpkt\n"); 7451 return; 7452 } 7453 pkt = (const struct vmbus_chanpkt_rxbuf *)pkthdr; 7454 7455 if (__predict_false(pkt->cp_rxbuf_id != HN_NVS_RXBUF_SIG)) { 7456 if_printf(rxr->hn_ifp, "invalid rxbuf_id 0x%08x\n", 7457 pkt->cp_rxbuf_id); 7458 return; 7459 } 7460 7461 count = pkt->cp_rxbuf_cnt; 7462 if (__predict_false(hlen < 7463 __offsetof(struct vmbus_chanpkt_rxbuf, cp_rxbuf[count]))) { 7464 if_printf(rxr->hn_ifp, "invalid rxbuf_cnt %d\n", count); 7465 return; 7466 } 7467 7468 NET_EPOCH_ENTER(et); 7469 /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */ 7470 for (i = 0; i < count; ++i) { 7471 int ofs, len; 7472 7473 ofs = pkt->cp_rxbuf[i].rb_ofs; 7474 len = pkt->cp_rxbuf[i].rb_len; 7475 if (__predict_false(ofs + len > HN_RXBUF_SIZE)) { 7476 if_printf(rxr->hn_ifp, "%dth RNDIS msg overflow rxbuf, " 7477 "ofs %d, len %d\n", i, ofs, len); 7478 continue; 7479 } 7480 7481 rxr->rsc.is_last = (i == (count - 1)); 7482 hn_rndis_rxpkt(rxr, rxr->hn_rxbuf + ofs, len); 7483 } 7484 NET_EPOCH_EXIT(et); 7485 7486 /* 7487 * Ack the consumed RXBUF associated w/ this channel packet, 7488 * so that this RXBUF can be recycled by the hypervisor. 7489 */ 7490 hn_nvs_ack_rxbuf(rxr, chan, pkt->cp_hdr.cph_xactid); 7491 } 7492 7493 static void 7494 hn_nvs_ack_rxbuf(struct hn_rx_ring *rxr, struct vmbus_channel *chan, 7495 uint64_t tid) 7496 { 7497 struct hn_nvs_rndis_ack ack; 7498 int retries, error; 7499 7500 ack.nvs_type = HN_NVS_TYPE_RNDIS_ACK; 7501 ack.nvs_status = HN_NVS_STATUS_OK; 7502 7503 retries = 0; 7504 again: 7505 error = vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP, 7506 VMBUS_CHANPKT_FLAG_NONE, &ack, sizeof(ack), tid); 7507 if (__predict_false(error == EAGAIN)) { 7508 /* 7509 * NOTE: 7510 * This should _not_ happen in real world, since the 7511 * consumption of the TX bufring from the TX path is 7512 * controlled. 7513 */ 7514 if (rxr->hn_ack_failed == 0) 7515 if_printf(rxr->hn_ifp, "RXBUF ack retry\n"); 7516 rxr->hn_ack_failed++; 7517 retries++; 7518 if (retries < 10) { 7519 DELAY(100); 7520 goto again; 7521 } 7522 /* RXBUF leaks! */ 7523 if_printf(rxr->hn_ifp, "RXBUF ack failed\n"); 7524 } 7525 } 7526 7527 static void 7528 hn_chan_callback(struct vmbus_channel *chan, void *xrxr) 7529 { 7530 struct hn_rx_ring *rxr = xrxr; 7531 struct hn_softc *sc = if_getsoftc(rxr->hn_ifp); 7532 7533 for (;;) { 7534 struct vmbus_chanpkt_hdr *pkt = rxr->hn_pktbuf; 7535 int error, pktlen; 7536 7537 pktlen = rxr->hn_pktbuf_len; 7538 error = vmbus_chan_recv_pkt(chan, pkt, &pktlen); 7539 if (__predict_false(error == ENOBUFS)) { 7540 void *nbuf; 7541 int nlen; 7542 7543 /* 7544 * Expand channel packet buffer. 7545 * 7546 * XXX 7547 * Use M_WAITOK here, since allocation failure 7548 * is fatal. 7549 */ 7550 nlen = rxr->hn_pktbuf_len * 2; 7551 while (nlen < pktlen) 7552 nlen *= 2; 7553 nbuf = malloc(nlen, M_DEVBUF, M_WAITOK); 7554 7555 if_printf(rxr->hn_ifp, "expand pktbuf %d -> %d\n", 7556 rxr->hn_pktbuf_len, nlen); 7557 7558 free(rxr->hn_pktbuf, M_DEVBUF); 7559 rxr->hn_pktbuf = nbuf; 7560 rxr->hn_pktbuf_len = nlen; 7561 /* Retry! */ 7562 continue; 7563 } else if (__predict_false(error == EAGAIN)) { 7564 /* No more channel packets; done! */ 7565 break; 7566 } 7567 KASSERT(!error, ("vmbus_chan_recv_pkt failed: %d", error)); 7568 7569 switch (pkt->cph_type) { 7570 case VMBUS_CHANPKT_TYPE_COMP: 7571 hn_nvs_handle_comp(sc, chan, pkt); 7572 break; 7573 7574 case VMBUS_CHANPKT_TYPE_RXBUF: 7575 hn_nvs_handle_rxbuf(rxr, chan, pkt); 7576 break; 7577 7578 case VMBUS_CHANPKT_TYPE_INBAND: 7579 hn_nvs_handle_notify(sc, pkt); 7580 break; 7581 7582 default: 7583 if_printf(rxr->hn_ifp, "unknown chan pkt %u\n", 7584 pkt->cph_type); 7585 break; 7586 } 7587 } 7588 hn_chan_rollup(rxr, rxr->hn_txr); 7589 } 7590 7591 static void 7592 hn_sysinit(void *arg __unused) 7593 { 7594 int i; 7595 7596 hn_udpcs_fixup = counter_u64_alloc(M_WAITOK); 7597 7598 #ifdef HN_IFSTART_SUPPORT 7599 /* 7600 * Don't use ifnet.if_start if transparent VF mode is requested; 7601 * mainly due to the IFF_DRV_OACTIVE flag. 7602 */ 7603 if (hn_xpnt_vf && hn_use_if_start) { 7604 hn_use_if_start = 0; 7605 printf("hn: tranparent VF mode, if_transmit will be used, " 7606 "instead of if_start\n"); 7607 } 7608 #endif 7609 if (hn_xpnt_vf_attwait < HN_XPNT_VF_ATTWAIT_MIN) { 7610 printf("hn: invalid transparent VF attach routing " 7611 "wait timeout %d, reset to %d\n", 7612 hn_xpnt_vf_attwait, HN_XPNT_VF_ATTWAIT_MIN); 7613 hn_xpnt_vf_attwait = HN_XPNT_VF_ATTWAIT_MIN; 7614 } 7615 7616 /* 7617 * Initialize VF map. 7618 */ 7619 rm_init_flags(&hn_vfmap_lock, "hn_vfmap", RM_SLEEPABLE); 7620 hn_vfmap_size = HN_VFMAP_SIZE_DEF; 7621 hn_vfmap = malloc(sizeof(if_t) * hn_vfmap_size, M_DEVBUF, 7622 M_WAITOK | M_ZERO); 7623 7624 /* 7625 * Fix the # of TX taskqueues. 7626 */ 7627 if (hn_tx_taskq_cnt <= 0) 7628 hn_tx_taskq_cnt = 1; 7629 else if (hn_tx_taskq_cnt > mp_ncpus) 7630 hn_tx_taskq_cnt = mp_ncpus; 7631 7632 /* 7633 * Fix the TX taskqueue mode. 7634 */ 7635 switch (hn_tx_taskq_mode) { 7636 case HN_TX_TASKQ_M_INDEP: 7637 case HN_TX_TASKQ_M_GLOBAL: 7638 case HN_TX_TASKQ_M_EVTTQ: 7639 break; 7640 default: 7641 hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; 7642 break; 7643 } 7644 7645 if (vm_guest != VM_GUEST_HV) 7646 return; 7647 7648 if (hn_tx_taskq_mode != HN_TX_TASKQ_M_GLOBAL) 7649 return; 7650 7651 hn_tx_taskque = malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), 7652 M_DEVBUF, M_WAITOK); 7653 for (i = 0; i < hn_tx_taskq_cnt; ++i) { 7654 hn_tx_taskque[i] = taskqueue_create("hn_tx", M_WAITOK, 7655 taskqueue_thread_enqueue, &hn_tx_taskque[i]); 7656 taskqueue_start_threads(&hn_tx_taskque[i], 1, PI_NET, 7657 "hn tx%d", i); 7658 } 7659 } 7660 SYSINIT(hn_sysinit, SI_SUB_DRIVERS, SI_ORDER_SECOND, hn_sysinit, NULL); 7661 7662 static void 7663 hn_sysuninit(void *arg __unused) 7664 { 7665 7666 if (hn_tx_taskque != NULL) { 7667 int i; 7668 7669 for (i = 0; i < hn_tx_taskq_cnt; ++i) 7670 taskqueue_free(hn_tx_taskque[i]); 7671 free(hn_tx_taskque, M_DEVBUF); 7672 } 7673 7674 if (hn_vfmap != NULL) 7675 free(hn_vfmap, M_DEVBUF); 7676 rm_destroy(&hn_vfmap_lock); 7677 7678 counter_u64_free(hn_udpcs_fixup); 7679 } 7680 SYSUNINIT(hn_sysuninit, SI_SUB_DRIVERS, SI_ORDER_SECOND, hn_sysuninit, NULL); 7681