Lines Matching +full:pa +full:- +full:stats
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
418 if (--loaded == 0) { in vtnet_modevent()
446 sc->vtnet_dev = dev; in vtnet_attach()
450 callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); in vtnet_attach()
489 ether_ifdetach(sc->vtnet_ifp); in vtnet_attach()
512 ifp = sc->vtnet_ifp; in vtnet_detach()
519 callout_drain(&sc->vtnet_tick_ch); in vtnet_detach()
529 if (sc->vtnet_pfil != NULL) { in vtnet_detach()
530 pfil_head_unregister(sc->vtnet_pfil); in vtnet_detach()
531 sc->vtnet_pfil = NULL; in vtnet_detach()
536 if (sc->vtnet_vlan_attach != NULL) { in vtnet_detach()
537 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach); in vtnet_detach()
538 sc->vtnet_vlan_attach = NULL; in vtnet_detach()
540 if (sc->vtnet_vlan_detach != NULL) { in vtnet_detach()
541 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach); in vtnet_detach()
542 sc->vtnet_vlan_detach = NULL; in vtnet_detach()
545 ifmedia_removeall(&sc->vtnet_media); in vtnet_detach()
549 sc->vtnet_ifp = NULL; in vtnet_detach()
555 if (sc->vtnet_ctrl_vq != NULL) in vtnet_detach()
572 sc->vtnet_flags |= VTNET_FLAG_SUSPENDED; in vtnet_suspend()
585 ifp = sc->vtnet_ifp; in vtnet_resume()
590 sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED; in vtnet_resume()
629 if (sc->vtnet_link_active != 0) in vtnet_config_change()
643 dev = sc->vtnet_dev; in vtnet_negotiate_features()
713 sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG; in vtnet_negotiate_features()
716 sc->vtnet_features = negotiated_features; in vtnet_negotiate_features()
717 sc->vtnet_negotiated_features = negotiated_features; in vtnet_negotiate_features()
728 dev = sc->vtnet_dev; in vtnet_setup_features()
735 sc->vtnet_flags |= VTNET_FLAG_MODERN; in vtnet_setup_features()
737 sc->vtnet_flags |= VTNET_FLAG_INDIRECT; in vtnet_setup_features()
739 sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX; in vtnet_setup_features()
743 sc->vtnet_flags |= VTNET_FLAG_MAC; in vtnet_setup_features()
747 sc->vtnet_max_mtu = virtio_read_dev_config_2(dev, in vtnet_setup_features()
750 sc->vtnet_max_mtu = VTNET_MAX_MTU; in vtnet_setup_features()
753 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS; in vtnet_setup_features()
754 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); in vtnet_setup_features()
757 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_v1); in vtnet_setup_features()
759 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr); in vtnet_setup_features()
761 if (vtnet_modern(sc) || sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) in vtnet_setup_features()
762 sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_INLINE; in vtnet_setup_features()
763 else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) in vtnet_setup_features()
764 sc->vtnet_rx_nsegs = VTNET_RX_SEGS_LRO_NOMRG; in vtnet_setup_features()
766 sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE; in vtnet_setup_features()
774 sc->vtnet_flags |= VTNET_FLAG_SW_LRO; in vtnet_setup_features()
779 sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MAX; in vtnet_setup_features()
781 sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MIN; in vtnet_setup_features()
783 sc->vtnet_req_vq_pairs = 1; in vtnet_setup_features()
784 sc->vtnet_max_vq_pairs = 1; in vtnet_setup_features()
787 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ; in vtnet_setup_features()
790 sc->vtnet_flags |= VTNET_FLAG_CTRL_RX; in vtnet_setup_features()
792 sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER; in vtnet_setup_features()
794 sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC; in vtnet_setup_features()
797 sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev, in vtnet_setup_features()
803 if (sc->vtnet_max_vq_pairs > 1) { in vtnet_setup_features()
815 if (req > sc->vtnet_max_vq_pairs) in vtnet_setup_features()
816 req = sc->vtnet_max_vq_pairs; in vtnet_setup_features()
820 sc->vtnet_req_vq_pairs = req; in vtnet_setup_features()
821 sc->vtnet_flags |= VTNET_FLAG_MQ; in vtnet_setup_features()
833 rxq = &sc->vtnet_rxqs[id]; in vtnet_init_rxq()
835 snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d", in vtnet_init_rxq()
836 device_get_nameunit(sc->vtnet_dev), id); in vtnet_init_rxq()
837 mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF); in vtnet_init_rxq()
839 rxq->vtnrx_sc = sc; in vtnet_init_rxq()
840 rxq->vtnrx_id = id; in vtnet_init_rxq()
842 rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); in vtnet_init_rxq()
843 if (rxq->vtnrx_sg == NULL) in vtnet_init_rxq()
848 if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp, in vtnet_init_rxq()
849 sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0) in vtnet_init_rxq()
854 NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); in vtnet_init_rxq()
855 rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, in vtnet_init_rxq()
856 taskqueue_thread_enqueue, &rxq->vtnrx_tq); in vtnet_init_rxq()
858 return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); in vtnet_init_rxq()
866 txq = &sc->vtnet_txqs[id]; in vtnet_init_txq()
868 snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d", in vtnet_init_txq()
869 device_get_nameunit(sc->vtnet_dev), id); in vtnet_init_txq()
870 mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF); in vtnet_init_txq()
872 txq->vtntx_sc = sc; in vtnet_init_txq()
873 txq->vtntx_id = id; in vtnet_init_txq()
875 txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT); in vtnet_init_txq()
876 if (txq->vtntx_sg == NULL) in vtnet_init_txq()
880 txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF, in vtnet_init_txq()
881 M_NOWAIT, &txq->vtntx_mtx); in vtnet_init_txq()
882 if (txq->vtntx_br == NULL) in vtnet_init_txq()
885 TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq); in vtnet_init_txq()
887 TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq); in vtnet_init_txq()
888 txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT, in vtnet_init_txq()
889 taskqueue_thread_enqueue, &txq->vtntx_tq); in vtnet_init_txq()
890 if (txq->vtntx_tq == NULL) in vtnet_init_txq()
901 npairs = sc->vtnet_max_vq_pairs; in vtnet_alloc_rxtx_queues()
903 sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF, in vtnet_alloc_rxtx_queues()
905 sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF, in vtnet_alloc_rxtx_queues()
907 if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL) in vtnet_alloc_rxtx_queues()
929 rxq->vtnrx_sc = NULL; in vtnet_destroy_rxq()
930 rxq->vtnrx_id = -1; in vtnet_destroy_rxq()
933 tcp_lro_free(&rxq->vtnrx_lro); in vtnet_destroy_rxq()
936 if (rxq->vtnrx_sg != NULL) { in vtnet_destroy_rxq()
937 sglist_free(rxq->vtnrx_sg); in vtnet_destroy_rxq()
938 rxq->vtnrx_sg = NULL; in vtnet_destroy_rxq()
941 if (mtx_initialized(&rxq->vtnrx_mtx) != 0) in vtnet_destroy_rxq()
942 mtx_destroy(&rxq->vtnrx_mtx); in vtnet_destroy_rxq()
949 txq->vtntx_sc = NULL; in vtnet_destroy_txq()
950 txq->vtntx_id = -1; in vtnet_destroy_txq()
952 if (txq->vtntx_sg != NULL) { in vtnet_destroy_txq()
953 sglist_free(txq->vtntx_sg); in vtnet_destroy_txq()
954 txq->vtntx_sg = NULL; in vtnet_destroy_txq()
958 if (txq->vtntx_br != NULL) { in vtnet_destroy_txq()
959 buf_ring_free(txq->vtntx_br, M_DEVBUF); in vtnet_destroy_txq()
960 txq->vtntx_br = NULL; in vtnet_destroy_txq()
964 if (mtx_initialized(&txq->vtntx_mtx) != 0) in vtnet_destroy_txq()
965 mtx_destroy(&txq->vtntx_mtx); in vtnet_destroy_txq()
973 if (sc->vtnet_rxqs != NULL) { in vtnet_free_rxtx_queues()
974 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) in vtnet_free_rxtx_queues()
975 vtnet_destroy_rxq(&sc->vtnet_rxqs[i]); in vtnet_free_rxtx_queues()
976 free(sc->vtnet_rxqs, M_DEVBUF); in vtnet_free_rxtx_queues()
977 sc->vtnet_rxqs = NULL; in vtnet_free_rxtx_queues()
980 if (sc->vtnet_txqs != NULL) { in vtnet_free_rxtx_queues()
981 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) in vtnet_free_rxtx_queues()
982 vtnet_destroy_txq(&sc->vtnet_txqs[i]); in vtnet_free_rxtx_queues()
983 free(sc->vtnet_txqs, M_DEVBUF); in vtnet_free_rxtx_queues()
984 sc->vtnet_txqs = NULL; in vtnet_free_rxtx_queues()
992 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { in vtnet_alloc_rx_filters()
993 sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter), in vtnet_alloc_rx_filters()
995 if (sc->vtnet_mac_filter == NULL) in vtnet_alloc_rx_filters()
999 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { in vtnet_alloc_rx_filters()
1000 sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) * in vtnet_alloc_rx_filters()
1002 if (sc->vtnet_vlan_filter == NULL) in vtnet_alloc_rx_filters()
1013 if (sc->vtnet_mac_filter != NULL) { in vtnet_free_rx_filters()
1014 free(sc->vtnet_mac_filter, M_DEVBUF); in vtnet_free_rx_filters()
1015 sc->vtnet_mac_filter = NULL; in vtnet_free_rx_filters()
1018 if (sc->vtnet_vlan_filter != NULL) { in vtnet_free_rx_filters()
1019 free(sc->vtnet_vlan_filter, M_DEVBUF); in vtnet_free_rx_filters()
1020 sc->vtnet_vlan_filter = NULL; in vtnet_free_rx_filters()
1033 dev = sc->vtnet_dev; in vtnet_alloc_virtqueues()
1035 nvqs = sc->vtnet_max_vq_pairs * 2; in vtnet_alloc_virtqueues()
1036 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) in vtnet_alloc_virtqueues()
1043 for (i = 0, idx = 0; i < sc->vtnet_req_vq_pairs; i++, idx += 2) { in vtnet_alloc_virtqueues()
1044 rxq = &sc->vtnet_rxqs[i]; in vtnet_alloc_virtqueues()
1045 VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs, in vtnet_alloc_virtqueues()
1046 vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq, in vtnet_alloc_virtqueues()
1047 "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id); in vtnet_alloc_virtqueues()
1049 txq = &sc->vtnet_txqs[i]; in vtnet_alloc_virtqueues()
1050 VQ_ALLOC_INFO_INIT(&info[idx + 1], sc->vtnet_tx_nsegs, in vtnet_alloc_virtqueues()
1051 vtnet_tx_vq_intr, txq, &txq->vtntx_vq, in vtnet_alloc_virtqueues()
1052 "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id); in vtnet_alloc_virtqueues()
1056 for (; i < sc->vtnet_max_vq_pairs; i++, idx += 2) { in vtnet_alloc_virtqueues()
1057 rxq = &sc->vtnet_rxqs[i]; in vtnet_alloc_virtqueues()
1058 VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, rxq, &rxq->vtnrx_vq, in vtnet_alloc_virtqueues()
1059 "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id); in vtnet_alloc_virtqueues()
1061 txq = &sc->vtnet_txqs[i]; in vtnet_alloc_virtqueues()
1062 VQ_ALLOC_INFO_INIT(&info[idx + 1], 0, NULL, txq, &txq->vtntx_vq, in vtnet_alloc_virtqueues()
1063 "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id); in vtnet_alloc_virtqueues()
1066 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { in vtnet_alloc_virtqueues()
1068 &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev)); in vtnet_alloc_virtqueues()
1083 dev = sc->vtnet_dev; in vtnet_alloc_interface()
1086 sc->vtnet_ifp = ifp; in vtnet_alloc_interface()
1095 struct pfil_head_args pa; in vtnet_setup_interface() local
1098 dev = sc->vtnet_dev; in vtnet_setup_interface()
1099 ifp = sc->vtnet_ifp; in vtnet_setup_interface()
1111 struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq; in vtnet_setup_interface()
1113 if_setsendqlen(ifp, virtqueue_size(vq) - 1); in vtnet_setup_interface()
1122 ifmedia_init(&sc->vtnet_media, 0, vtnet_ifmedia_upd, vtnet_ifmedia_sts); in vtnet_setup_interface()
1123 ifmedia_add(&sc->vtnet_media, IFM_ETHER | IFM_AUTO, 0, NULL); in vtnet_setup_interface()
1124 ifmedia_set(&sc->vtnet_media, IFM_ETHER | IFM_AUTO); in vtnet_setup_interface()
1137 sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; in vtnet_setup_interface()
1146 if_sethwtsomax(ifp, tso_maxlen - in vtnet_setup_interface()
1148 if_sethwtsomaxsegcount(ifp, sc->vtnet_tx_nsegs - 1); in vtnet_setup_interface()
1162 sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM; in vtnet_setup_interface()
1178 if (sc->vtnet_max_mtu >= ETHERMTU_JUMBO) in vtnet_setup_interface()
1187 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { in vtnet_setup_interface()
1190 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, in vtnet_setup_interface()
1192 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, in vtnet_setup_interface()
1196 ether_ifattach(ifp, sc->vtnet_hwaddr); in vtnet_setup_interface()
1203 pa.pa_version = PFIL_VERSION; in vtnet_setup_interface()
1204 pa.pa_flags = PFIL_IN; in vtnet_setup_interface()
1205 pa.pa_type = PFIL_TYPE_ETHERNET; in vtnet_setup_interface()
1206 pa.pa_headname = if_name(ifp); in vtnet_setup_interface()
1207 sc->vtnet_pfil = pfil_head_register(&pa); in vtnet_setup_interface()
1217 if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) in vtnet_rx_cluster_size()
1219 else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) in vtnet_rx_cluster_size()
1228 MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr_v1)); in vtnet_rx_cluster_size()
1237 if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) { in vtnet_rx_cluster_size()
1258 ifp = sc->vtnet_ifp; in vtnet_ioctl_mtu()
1263 else if (mtu < ETHERMIN || mtu > sc->vtnet_max_mtu) in vtnet_ioctl_mtu()
1269 if (clustersz != sc->vtnet_rx_clustersz && in vtnet_ioctl_mtu()
1284 ifp = sc->vtnet_ifp; in vtnet_ioctl_ifflags()
1300 if ((if_getflags(ifp) ^ sc->vtnet_if_flags) & in vtnet_ioctl_ifflags()
1302 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) in vtnet_ioctl_ifflags()
1315 sc->vtnet_if_flags = if_getflags(ifp); in vtnet_ioctl_ifflags()
1324 ifp = sc->vtnet_ifp; in vtnet_ioctl_multi()
1328 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX && in vtnet_ioctl_multi()
1341 ifp = sc->vtnet_ifp; in vtnet_ioctl_ifcap()
1342 mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ if_getcapenable(ifp); in vtnet_ioctl_ifcap()
1361 if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) in vtnet_ioctl_ifcap()
1431 error = vtnet_ioctl_mtu(sc, ifr->ifr_mtu); in vtnet_ioctl()
1450 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd); in vtnet_ioctl()
1482 vq = rxq->vtnrx_vq; in vtnet_rxq_populate()
1512 struct netmap_kring *kring = netmap_kring_on(NA(rxq->vtnrx_sc->vtnet_ifp), in vtnet_rxq_free_mbufs()
1513 rxq->vtnrx_id, NR_RX); in vtnet_rxq_free_mbufs()
1518 vq = rxq->vtnrx_vq; in vtnet_rxq_free_mbufs()
1537 size = sc->vtnet_rx_clustersz; in vtnet_rx_alloc_buf()
1539 KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, in vtnet_rx_alloc_buf()
1545 sc->vtnet_stats.mbuf_alloc_failed++; in vtnet_rx_alloc_buf()
1550 m->m_len = size; in vtnet_rx_alloc_buf()
1555 if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) { in vtnet_rx_alloc_buf()
1559 m_tail->m_next = m; in vtnet_rx_alloc_buf()
1582 sc = rxq->vtnrx_sc; in vtnet_rxq_replace_lro_nomrg_buf()
1583 clustersz = sc->vtnet_rx_clustersz; in vtnet_rxq_replace_lro_nomrg_buf()
1588 if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) in vtnet_rxq_replace_lro_nomrg_buf()
1589 clustersz -= VTNET_ETHER_ALIGN; in vtnet_rxq_replace_lro_nomrg_buf()
1605 sc->vtnet_stats.rx_frame_too_large++; in vtnet_rxq_replace_lro_nomrg_buf()
1613 KASSERT(m->m_len == clustersz, in vtnet_rxq_replace_lro_nomrg_buf()
1615 m->m_len, clustersz)); in vtnet_rxq_replace_lro_nomrg_buf()
1617 m->m_len = MIN(m->m_len, len); in vtnet_rxq_replace_lro_nomrg_buf()
1618 len -= m->m_len; in vtnet_rxq_replace_lro_nomrg_buf()
1621 m = m->m_next; in vtnet_rxq_replace_lro_nomrg_buf()
1625 KASSERT(nreplace > 0 && nreplace <= sc->vtnet_rx_nmbufs, in vtnet_rxq_replace_lro_nomrg_buf()
1627 nreplace, sc->vtnet_rx_nmbufs)); in vtnet_rxq_replace_lro_nomrg_buf()
1631 m_prev->m_len = clustersz; in vtnet_rxq_replace_lro_nomrg_buf()
1639 if (m_prev->m_next != NULL) { in vtnet_rxq_replace_lro_nomrg_buf()
1640 m_tail->m_next = m_prev->m_next; in vtnet_rxq_replace_lro_nomrg_buf()
1641 m_prev->m_next = NULL; in vtnet_rxq_replace_lro_nomrg_buf()
1653 if (m_tail->m_next != NULL) { in vtnet_rxq_replace_lro_nomrg_buf()
1654 m_prev->m_next = m_tail->m_next; in vtnet_rxq_replace_lro_nomrg_buf()
1655 m_tail->m_next = NULL; in vtnet_rxq_replace_lro_nomrg_buf()
1657 m_prev->m_len = clustersz; in vtnet_rxq_replace_lro_nomrg_buf()
1658 sc->vtnet_stats.rx_enq_replacement_failed++; in vtnet_rxq_replace_lro_nomrg_buf()
1672 sc = rxq->vtnrx_sc; in vtnet_rxq_replace_buf()
1674 if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) in vtnet_rxq_replace_buf()
1677 MPASS(m->m_next == NULL); in vtnet_rxq_replace_buf()
1678 if (m->m_len < len) in vtnet_rxq_replace_buf()
1687 sc->vtnet_stats.rx_enq_replacement_failed++; in vtnet_rxq_replace_buf()
1690 m->m_len = len; in vtnet_rxq_replace_buf()
1702 sc = rxq->vtnrx_sc; in vtnet_rxq_enqueue_buf()
1703 sg = rxq->vtnrx_sg; in vtnet_rxq_enqueue_buf()
1705 KASSERT(m->m_next == NULL || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, in vtnet_rxq_enqueue_buf()
1711 (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) != 0; /* TODO: ANY_LAYOUT */ in vtnet_rxq_enqueue_buf()
1722 MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr)); in vtnet_rxq_enqueue_buf()
1725 error = sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size); in vtnet_rxq_enqueue_buf()
1729 m->m_len - sizeof(struct vtnet_rx_header)); in vtnet_rxq_enqueue_buf()
1733 if (m->m_next != NULL) in vtnet_rxq_enqueue_buf()
1734 error = sglist_append_mbuf(sg, m->m_next); in vtnet_rxq_enqueue_buf()
1740 return (virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg)); in vtnet_rxq_enqueue_buf()
1750 sc = rxq->vtnrx_sc; in vtnet_rxq_new_buf()
1752 m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL); in vtnet_rxq_new_buf()
1770 sc = rxq->vtnrx_sc; in vtnet_rxq_csum_needs_csum()
1781 * Default to receiving the packet as-is for performance reasons, but in vtnet_rxq_csum_needs_csum()
1789 if ((sc->vtnet_flags & VTNET_FLAG_FIXUP_NEEDS_CSUM) == 0) { in vtnet_rxq_csum_needs_csum()
1805 csum_off = hdr->csum_start + hdr->csum_offset; in vtnet_rxq_csum_needs_csum()
1809 if (m->m_len < csum_end || m->m_pkthdr.len < csum_end) in vtnet_rxq_csum_needs_csum()
1821 csum = in_cksum_skip(m, m->m_pkthdr.len, hdr->csum_start); in vtnet_rxq_csum_needs_csum()
1823 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; in vtnet_rxq_csum_needs_csum()
1824 m->m_pkthdr.csum_data = 0xFFFF; in vtnet_rxq_csum_needs_csum()
1829 sc->vtnet_stats.rx_csum_bad_ethtype++; in vtnet_rxq_csum_needs_csum()
1846 sc = rxq->vtnrx_sc; in vtnet_rxq_csum_data_valid()
1852 if (__predict_false(m->m_len < hoff + sizeof(struct ip))) in vtnet_rxq_csum_data_valid()
1855 struct ip *ip = (struct ip *)(m->m_data + hoff); in vtnet_rxq_csum_data_valid()
1856 protocol = ip->ip_p; in vtnet_rxq_csum_data_valid()
1862 if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr)) in vtnet_rxq_csum_data_valid()
1875 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; in vtnet_rxq_csum_data_valid()
1876 m->m_pkthdr.csum_data = 0xFFFF; in vtnet_rxq_csum_data_valid()
1881 * protocol. Let the stack re-verify the checksum later in vtnet_rxq_csum_data_valid()
1885 if_printf(sc->vtnet_ifp, in vtnet_rxq_csum_data_valid()
1888 __func__, etype, protocol, hdr->csum_start, in vtnet_rxq_csum_data_valid()
1889 hdr->csum_offset); in vtnet_rxq_csum_data_valid()
1906 etype = ntohs(eh->ether_type); in vtnet_rxq_csum()
1911 etype = ntohs(evh->evl_proto); in vtnet_rxq_csum()
1916 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) in vtnet_rxq_csum()
1927 while (--nbufs > 0) { in vtnet_rxq_discard_merged_bufs()
1928 m = virtqueue_dequeue(rxq->vtnrx_vq, NULL); in vtnet_rxq_discard_merged_bufs()
1956 sc = rxq->vtnrx_sc; in vtnet_rxq_merged_eof()
1957 vq = rxq->vtnrx_vq; in vtnet_rxq_merged_eof()
1960 while (--nbufs > 0) { in vtnet_rxq_merged_eof()
1966 rxq->vtnrx_stats.vrxs_ierrors++; in vtnet_rxq_merged_eof()
1971 rxq->vtnrx_stats.vrxs_iqdrops++; in vtnet_rxq_merged_eof()
1978 if (m->m_len < len) in vtnet_rxq_merged_eof()
1979 len = m->m_len; in vtnet_rxq_merged_eof()
1981 m->m_len = len; in vtnet_rxq_merged_eof()
1982 m->m_flags &= ~M_PKTHDR; in vtnet_rxq_merged_eof()
1984 m_head->m_pkthdr.len += len; in vtnet_rxq_merged_eof()
1985 m_tail->m_next = m; in vtnet_rxq_merged_eof()
1992 sc->vtnet_stats.rx_mergeable_failed++; in vtnet_rxq_merged_eof()
2004 lro = &rxq->vtnrx_lro; in vtnet_lro_rx()
2006 if (lro->lro_mbuf_max != 0) { in vtnet_lro_rx()
2022 sc = rxq->vtnrx_sc; in vtnet_rxq_input()
2023 ifp = sc->vtnet_ifp; in vtnet_rxq_input()
2027 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { in vtnet_rxq_input()
2033 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) in vtnet_rxq_input()
2034 hdr->csum_start -= ETHER_VLAN_ENCAP_LEN; in vtnet_rxq_input()
2038 m->m_pkthdr.flowid = rxq->vtnrx_id; in vtnet_rxq_input()
2041 if (hdr->flags & in vtnet_rxq_input()
2044 rxq->vtnrx_stats.vrxs_csum++; in vtnet_rxq_input()
2046 rxq->vtnrx_stats.vrxs_csum_failed++; in vtnet_rxq_input()
2049 if (hdr->gso_size != 0) { in vtnet_rxq_input()
2050 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { in vtnet_rxq_input()
2053 m->m_pkthdr.lro_nsegs = in vtnet_rxq_input()
2054 howmany(m->m_pkthdr.len, hdr->gso_size); in vtnet_rxq_input()
2055 rxq->vtnrx_stats.vrxs_host_lro++; in vtnet_rxq_input()
2060 rxq->vtnrx_stats.vrxs_ipackets++; in vtnet_rxq_input()
2061 rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; in vtnet_rxq_input()
2082 sc = rxq->vtnrx_sc; in vtnet_rxq_eof()
2083 vq = rxq->vtnrx_vq; in vtnet_rxq_eof()
2084 ifp = sc->vtnet_ifp; in vtnet_rxq_eof()
2086 count = sc->vtnet_rx_process_limit; in vtnet_rxq_eof()
2091 while (count-- > 0) { in vtnet_rxq_eof()
2100 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) { in vtnet_rxq_eof()
2101 rxq->vtnrx_stats.vrxs_ierrors++; in vtnet_rxq_eof()
2106 if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) { in vtnet_rxq_eof()
2110 nbufs = vtnet_htog16(sc, mhdr->num_buffers); in vtnet_rxq_eof()
2126 rxq->vtnrx_stats.vrxs_iqdrops++; in vtnet_rxq_eof()
2133 m->m_pkthdr.len = len; in vtnet_rxq_eof()
2134 m->m_pkthdr.rcvif = ifp; in vtnet_rxq_eof()
2135 m->m_pkthdr.csum_flags = 0; in vtnet_rxq_eof()
2152 lhdr.flags = hdr->flags; in vtnet_rxq_eof()
2153 lhdr.gso_type = hdr->gso_type; in vtnet_rxq_eof()
2154 lhdr.hdr_len = vtnet_htog16(sc, hdr->hdr_len); in vtnet_rxq_eof()
2155 lhdr.gso_size = vtnet_htog16(sc, hdr->gso_size); in vtnet_rxq_eof()
2156 lhdr.csum_start = vtnet_htog16(sc, hdr->csum_start); in vtnet_rxq_eof()
2157 lhdr.csum_offset = vtnet_htog16(sc, hdr->csum_offset); in vtnet_rxq_eof()
2160 if (PFIL_HOOKED_IN(sc->vtnet_pfil)) { in vtnet_rxq_eof()
2163 pfil = pfil_mbuf_in(sc->vtnet_pfil, &m, ifp, NULL); in vtnet_rxq_eof()
2180 tcp_lro_flush_all(&rxq->vtnrx_lro); in vtnet_rxq_eof()
2199 sc = rxq->vtnrx_sc; in vtnet_rx_vq_process()
2200 ifp = sc->vtnet_ifp; in vtnet_rx_vq_process()
2202 if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) { in vtnet_rx_vq_process()
2205 * or multiqueue without per-VQ MSIX so every queue needs to in vtnet_rx_vq_process()
2224 nmirq = netmap_rx_irq(ifp, rxq->vtnrx_id, &more); in vtnet_rx_vq_process()
2228 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); in vtnet_rx_vq_process()
2248 if (tries-- > 0) in vtnet_rx_vq_process()
2251 rxq->vtnrx_stats.vrxs_rescheduled++; in vtnet_rx_vq_process()
2253 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); in vtnet_rx_vq_process()
2282 sc = txq->vtntx_sc; in vtnet_txq_intr_threshold()
2292 threshold = virtqueue_size(txq->vtntx_vq) / 4; in vtnet_txq_intr_threshold()
2298 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 && in vtnet_txq_intr_threshold()
2299 threshold < sc->vtnet_tx_nsegs) in vtnet_txq_intr_threshold()
2300 threshold = sc->vtnet_tx_nsegs; in vtnet_txq_intr_threshold()
2310 vq = txq->vtntx_vq; in vtnet_txq_below_threshold()
2312 return (virtqueue_nfree(vq) <= txq->vtntx_intr_threshold); in vtnet_txq_below_threshold()
2320 vq = txq->vtntx_vq; in vtnet_txq_notify()
2322 txq->vtntx_watchdog = VTNET_TX_TIMEOUT; in vtnet_txq_notify()
2348 struct netmap_kring *kring = netmap_kring_on(NA(txq->vtntx_sc->vtnet_ifp), in vtnet_txq_free_mbufs()
2349 txq->vtntx_id, NR_TX); in vtnet_txq_free_mbufs()
2354 vq = txq->vtntx_vq; in vtnet_txq_free_mbufs()
2359 m_freem(txhdr->vth_mbuf); in vtnet_txq_free_mbufs()
2381 sc = txq->vtntx_sc; in vtnet_txq_offload_ctx()
2384 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { in vtnet_txq_offload_ctx()
2386 *etype = ntohs(evh->evl_proto); in vtnet_txq_offload_ctx()
2391 *etype = ntohs(evh->evl_encap_proto); in vtnet_txq_offload_ctx()
2401 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { in vtnet_txq_offload_ctx()
2406 ip = (struct ip *)(m->m_data + offset); in vtnet_txq_offload_ctx()
2407 *proto = ip->ip_p; in vtnet_txq_offload_ctx()
2408 *start = offset + (ip->ip_hl << 2); in vtnet_txq_offload_ctx()
2414 *proto = -1; in vtnet_txq_offload_ctx()
2423 sc->vtnet_stats.tx_csum_unknown_ethtype++; in vtnet_txq_offload_ctx()
2439 sc = txq->vtntx_sc; in vtnet_txq_offload_tso()
2441 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) { in vtnet_txq_offload_tso()
2445 tcp = (struct tcphdr *)(m->m_data + offset); in vtnet_txq_offload_tso()
2447 hdr->hdr_len = vtnet_gtoh16(sc, offset + (tcp->th_off << 2)); in vtnet_txq_offload_tso()
2448 hdr->gso_size = vtnet_gtoh16(sc, m->m_pkthdr.tso_segsz); in vtnet_txq_offload_tso()
2449 hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : in vtnet_txq_offload_tso()
2455 * FreeBSD, ECN support is not on a per-interface basis, in vtnet_txq_offload_tso()
2459 if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { in vtnet_txq_offload_tso()
2461 if_printf(sc->vtnet_ifp, in vtnet_txq_offload_tso()
2465 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; in vtnet_txq_offload_tso()
2468 txq->vtntx_stats.vtxs_tso++; in vtnet_txq_offload_tso()
2480 sc = txq->vtntx_sc; in vtnet_txq_offload()
2481 flags = m->m_pkthdr.csum_flags; in vtnet_txq_offload()
2492 sc->vtnet_stats.tx_csum_proto_mismatch++; in vtnet_txq_offload()
2496 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; in vtnet_txq_offload()
2497 hdr->csum_start = vtnet_gtoh16(sc, csum_start); in vtnet_txq_offload()
2498 hdr->csum_offset = vtnet_gtoh16(sc, m->m_pkthdr.csum_data); in vtnet_txq_offload()
2499 txq->vtntx_stats.vtxs_csum++; in vtnet_txq_offload()
2508 sc->vtnet_stats.tx_tso_not_tcp++; in vtnet_txq_offload()
2510 } else if (__predict_false((hdr->flags & in vtnet_txq_offload()
2512 sc->vtnet_stats.tx_tso_without_csum++; in vtnet_txq_offload()
2538 sc = txq->vtntx_sc; in vtnet_txq_enqueue_buf()
2539 vq = txq->vtntx_vq; in vtnet_txq_enqueue_buf()
2540 sg = txq->vtntx_sg; in vtnet_txq_enqueue_buf()
2544 error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size); in vtnet_txq_enqueue_buf()
2545 if (error != 0 || sg->sg_nseg != 1) { in vtnet_txq_enqueue_buf()
2547 __func__, error, sg->sg_nseg)); in vtnet_txq_enqueue_buf()
2558 sc->vtnet_stats.tx_defragged++; in vtnet_txq_enqueue_buf()
2565 txhdr->vth_mbuf = m; in vtnet_txq_enqueue_buf()
2566 error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0); in vtnet_txq_enqueue_buf()
2571 sc->vtnet_stats.tx_defrag_failed++; in vtnet_txq_enqueue_buf()
2597 * Always use the non-mergeable header, regardless if mergable headers in vtnet_txq_encap()
2601 hdr = &txhdr->vth_uhdr.hdr; in vtnet_txq_encap()
2603 if (m->m_flags & M_VLANTAG) { in vtnet_txq_encap()
2604 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); in vtnet_txq_encap()
2609 m->m_flags &= ~M_VLANTAG; in vtnet_txq_encap()
2612 if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) { in vtnet_txq_encap()
2637 sc = txq->vtntx_sc; in vtnet_start_locked()
2638 vq = txq->vtntx_vq; in vtnet_start_locked()
2644 sc->vtnet_link_active == 0) in vtnet_start_locked()
2674 txq->vtntx_stats.vtxs_rescheduled++; in vtnet_start_locked()
2675 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); in vtnet_start_locked()
2686 txq = &sc->vtnet_txqs[0]; in vtnet_start()
2703 sc = txq->vtntx_sc; in vtnet_txq_mq_start_locked()
2704 vq = txq->vtntx_vq; in vtnet_txq_mq_start_locked()
2705 br = txq->vtntx_br; in vtnet_txq_mq_start_locked()
2706 ifp = sc->vtnet_ifp; in vtnet_txq_mq_start_locked()
2713 sc->vtnet_link_active == 0) { in vtnet_txq_mq_start_locked()
2753 txq->vtntx_stats.vtxs_rescheduled++; in vtnet_txq_mq_start_locked()
2754 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); in vtnet_txq_mq_start_locked()
2768 npairs = sc->vtnet_act_vq_pairs; in vtnet_txq_mq_start()
2771 i = m->m_pkthdr.flowid % npairs; in vtnet_txq_mq_start()
2775 txq = &sc->vtnet_txqs[i]; in vtnet_txq_mq_start()
2781 error = drbr_enqueue(ifp, txq->vtntx_br, m); in vtnet_txq_mq_start()
2782 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask); in vtnet_txq_mq_start()
2795 sc = txq->vtntx_sc; in vtnet_txq_tq_deferred()
2798 if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br)) in vtnet_txq_tq_deferred()
2810 sc = txq->vtntx_sc; in vtnet_txq_start()
2811 ifp = sc->vtnet_ifp; in vtnet_txq_start()
2814 if (!drbr_empty(ifp, txq->vtntx_br)) in vtnet_txq_start()
2831 sc = txq->vtntx_sc; in vtnet_txq_tq_intr()
2832 ifp = sc->vtnet_ifp; in vtnet_txq_tq_intr()
2855 vq = txq->vtntx_vq; in vtnet_txq_eof()
2860 m = txhdr->vth_mbuf; in vtnet_txq_eof()
2863 txq->vtntx_stats.vtxs_opackets++; in vtnet_txq_eof()
2864 txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len; in vtnet_txq_eof()
2865 if (m->m_flags & M_MCAST) in vtnet_txq_eof()
2866 txq->vtntx_stats.vtxs_omcasts++; in vtnet_txq_eof()
2873 txq->vtntx_watchdog = 0; in vtnet_txq_eof()
2886 sc = txq->vtntx_sc; in vtnet_tx_vq_intr()
2887 ifp = sc->vtnet_ifp; in vtnet_tx_vq_intr()
2889 if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) { in vtnet_tx_vq_intr()
2892 * or multiqueue without per-VQ MSIX so every queue needs to in vtnet_tx_vq_intr()
2901 if (netmap_tx_irq(ifp, txq->vtntx_id) != NM_IRQ_PASS) in vtnet_tx_vq_intr()
2926 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { in vtnet_tx_start_all()
2927 txq = &sc->vtnet_txqs[i]; in vtnet_tx_start_all()
2945 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { in vtnet_qflush()
2946 txq = &sc->vtnet_txqs[i]; in vtnet_qflush()
2949 while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL) in vtnet_qflush()
2962 ifp = txq->vtntx_sc->vtnet_ifp; in vtnet_watchdog()
2965 if (txq->vtntx_watchdog == 1) { in vtnet_watchdog()
2978 if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) { in vtnet_watchdog()
2984 if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id); in vtnet_watchdog()
2996 for (int i = 0; i < sc->vtnet_max_vq_pairs; i++) { in vtnet_accum_stats()
3000 rxst = &sc->vtnet_rxqs[i].vtnrx_stats; in vtnet_accum_stats()
3001 rxacc->vrxs_ipackets += rxst->vrxs_ipackets; in vtnet_accum_stats()
3002 rxacc->vrxs_ibytes += rxst->vrxs_ibytes; in vtnet_accum_stats()
3003 rxacc->vrxs_iqdrops += rxst->vrxs_iqdrops; in vtnet_accum_stats()
3004 rxacc->vrxs_csum += rxst->vrxs_csum; in vtnet_accum_stats()
3005 rxacc->vrxs_csum_failed += rxst->vrxs_csum_failed; in vtnet_accum_stats()
3006 rxacc->vrxs_rescheduled += rxst->vrxs_rescheduled; in vtnet_accum_stats()
3008 txst = &sc->vtnet_txqs[i].vtntx_stats; in vtnet_accum_stats()
3009 txacc->vtxs_opackets += txst->vtxs_opackets; in vtnet_accum_stats()
3010 txacc->vtxs_obytes += txst->vtxs_obytes; in vtnet_accum_stats()
3011 txacc->vtxs_csum += txst->vtxs_csum; in vtnet_accum_stats()
3012 txacc->vtxs_tso += txst->vtxs_tso; in vtnet_accum_stats()
3013 txacc->vtxs_rescheduled += txst->vtxs_rescheduled; in vtnet_accum_stats()
3057 ifp = sc->vtnet_ifp; in vtnet_tick()
3062 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) in vtnet_tick()
3063 timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]); in vtnet_tick()
3069 callout_schedule(&sc->vtnet_tick_ch, hz); in vtnet_tick()
3080 dev = sc->vtnet_dev; in vtnet_start_taskqueues()
3083 * Errors here are very difficult to recover from - we cannot in vtnet_start_taskqueues()
3088 * Most drivers just ignore the return value - it only fails in vtnet_start_taskqueues()
3091 for (i = 0; i < sc->vtnet_req_vq_pairs; i++) { in vtnet_start_taskqueues()
3092 rxq = &sc->vtnet_rxqs[i]; in vtnet_start_taskqueues()
3093 error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET, in vtnet_start_taskqueues()
3094 "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id); in vtnet_start_taskqueues()
3097 rxq->vtnrx_id); in vtnet_start_taskqueues()
3100 txq = &sc->vtnet_txqs[i]; in vtnet_start_taskqueues()
3101 error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET, in vtnet_start_taskqueues()
3102 "%s txq %d", device_get_nameunit(dev), txq->vtntx_id); in vtnet_start_taskqueues()
3105 txq->vtntx_id); in vtnet_start_taskqueues()
3117 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { in vtnet_free_taskqueues()
3118 rxq = &sc->vtnet_rxqs[i]; in vtnet_free_taskqueues()
3119 if (rxq->vtnrx_tq != NULL) { in vtnet_free_taskqueues()
3120 taskqueue_free(rxq->vtnrx_tq); in vtnet_free_taskqueues()
3121 rxq->vtnrx_tq = NULL; in vtnet_free_taskqueues()
3124 txq = &sc->vtnet_txqs[i]; in vtnet_free_taskqueues()
3125 if (txq->vtntx_tq != NULL) { in vtnet_free_taskqueues()
3126 taskqueue_free(txq->vtntx_tq); in vtnet_free_taskqueues()
3127 txq->vtntx_tq = NULL; in vtnet_free_taskqueues()
3139 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { in vtnet_drain_taskqueues()
3140 rxq = &sc->vtnet_rxqs[i]; in vtnet_drain_taskqueues()
3141 if (rxq->vtnrx_tq != NULL) in vtnet_drain_taskqueues()
3142 taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); in vtnet_drain_taskqueues()
3144 txq = &sc->vtnet_txqs[i]; in vtnet_drain_taskqueues()
3145 if (txq->vtntx_tq != NULL) { in vtnet_drain_taskqueues()
3146 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask); in vtnet_drain_taskqueues()
3148 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask); in vtnet_drain_taskqueues()
3160 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { in vtnet_drain_rxtx_queues()
3161 rxq = &sc->vtnet_rxqs[i]; in vtnet_drain_rxtx_queues()
3164 txq = &sc->vtnet_txqs[i]; in vtnet_drain_rxtx_queues()
3179 * Lock and unlock the per-queue mutex so we known the stop in vtnet_stop_rendezvous()
3184 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { in vtnet_stop_rendezvous()
3185 rxq = &sc->vtnet_rxqs[i]; in vtnet_stop_rendezvous()
3189 txq = &sc->vtnet_txqs[i]; in vtnet_stop_rendezvous()
3201 dev = sc->vtnet_dev; in vtnet_stop()
3202 ifp = sc->vtnet_ifp; in vtnet_stop()
3207 sc->vtnet_link_active = 0; in vtnet_stop()
3208 callout_stop(&sc->vtnet_tick_ch); in vtnet_stop()
3219 * Stop the host adapter. This resets it to the pre-initialized in vtnet_stop()
3227 sc->vtnet_act_vq_pairs = 1; in vtnet_stop()
3238 dev = sc->vtnet_dev; in vtnet_virtio_reinit()
3239 ifp = sc->vtnet_ifp; in vtnet_virtio_reinit()
3240 features = sc->vtnet_negotiated_features; in vtnet_virtio_reinit()
3243 * Re-negotiate with the host, removing any disabled receive in vtnet_virtio_reinit()
3263 sc->vtnet_features = features; in vtnet_virtio_reinit()
3274 ifp = sc->vtnet_ifp; in vtnet_init_rx_filters()
3276 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { in vtnet_init_rx_filters()
3293 dev = sc->vtnet_dev; in vtnet_init_rx_queues()
3294 ifp = sc->vtnet_ifp; in vtnet_init_rx_queues()
3297 sc->vtnet_rx_clustersz = clustersz; in vtnet_init_rx_queues()
3299 if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) { in vtnet_init_rx_queues()
3300 sc->vtnet_rx_nmbufs = howmany(sizeof(struct vtnet_rx_header) + in vtnet_init_rx_queues()
3302 KASSERT(sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs, in vtnet_init_rx_queues()
3304 sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs)); in vtnet_init_rx_queues()
3306 sc->vtnet_rx_nmbufs = 1; in vtnet_init_rx_queues()
3308 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { in vtnet_init_rx_queues()
3309 rxq = &sc->vtnet_rxqs[i]; in vtnet_init_rx_queues()
3331 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { in vtnet_init_tx_queues()
3332 txq = &sc->vtnet_txqs[i]; in vtnet_init_tx_queues()
3333 txq->vtntx_watchdog = 0; in vtnet_init_tx_queues()
3334 txq->vtntx_intr_threshold = vtnet_txq_intr_threshold(txq); in vtnet_init_tx_queues()
3336 netmap_reset(NA(sc->vtnet_ifp), NR_TX, i, 0); in vtnet_init_tx_queues()
3365 dev = sc->vtnet_dev; in vtnet_set_active_vq_pairs()
3367 if ((sc->vtnet_flags & VTNET_FLAG_MQ) == 0) { in vtnet_set_active_vq_pairs()
3368 sc->vtnet_act_vq_pairs = 1; in vtnet_set_active_vq_pairs()
3372 npairs = sc->vtnet_req_vq_pairs; in vtnet_set_active_vq_pairs()
3380 sc->vtnet_act_vq_pairs = npairs; in vtnet_set_active_vq_pairs()
3390 ifp = sc->vtnet_ifp; in vtnet_update_rx_offloads()
3391 features = sc->vtnet_features; in vtnet_update_rx_offloads()
3414 device_printf(sc->vtnet_dev, in vtnet_update_rx_offloads()
3421 sc->vtnet_features = features; in vtnet_update_rx_offloads()
3430 ifp = sc->vtnet_ifp; in vtnet_reinit()
3432 bcopy(if_getlladdr(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN); in vtnet_reinit()
3441 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) in vtnet_reinit()
3466 ifp = sc->vtnet_ifp; in vtnet_init_locked()
3495 callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); in vtnet_init_locked()
3498 /* Re-enable txsync/rxsync. */ in vtnet_init_locked()
3523 KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq), in vtnet_free_ctrl_vq()
3524 ("%s: ctrl vq %p not empty", __func__, sc->vtnet_ctrl_vq)); in vtnet_free_ctrl_vq()
3533 vq = sc->vtnet_ctrl_vq; in vtnet_exec_ctrl_cmd()
3535 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ); in vtnet_exec_ctrl_cmd()
3566 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_MAC); in vtnet_ctrl_mac_cmd()
3580 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); in vtnet_ctrl_mac_cmd()
3600 MPASS(sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); in vtnet_ctrl_guest_offloads()
3614 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); in vtnet_ctrl_guest_offloads()
3634 MPASS(sc->vtnet_flags & VTNET_FLAG_MQ); in vtnet_ctrl_mq_cmd()
3648 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); in vtnet_ctrl_mq_cmd()
3668 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX); in vtnet_ctrl_rx_cmd()
3682 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); in vtnet_ctrl_rx_cmd()
3705 dev = sc->vtnet_dev; in vtnet_rx_filter()
3706 ifp = sc->vtnet_ifp; in vtnet_rx_filter()
3716 device_printf(dev, "cannot %s all-multicast mode\n", in vtnet_rx_filter()
3726 if (memcmp(LLADDR(sdl), sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0) in vtnet_copy_ifaddr()
3731 &sc->vtnet_mac_filter->vmf_unicast.macs[ucnt], in vtnet_copy_ifaddr()
3743 bcopy(LLADDR(sdl), &filter->vmf_multicast.macs[mcnt], in vtnet_copy_maddr()
3762 ifp = sc->vtnet_ifp; in vtnet_rx_filter_mac()
3763 filter = sc->vtnet_mac_filter; in vtnet_rx_filter_mac()
3766 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX); in vtnet_rx_filter_mac()
3787 "assigned, falling back to all-multicast mode\n", in vtnet_rx_filter_mac()
3794 filter->vmf_unicast.nentries = vtnet_gtoh32(sc, ucnt); in vtnet_rx_filter_mac()
3795 filter->vmf_multicast.nentries = vtnet_gtoh32(sc, mcnt); in vtnet_rx_filter_mac()
3803 error |= sglist_append(&sg, &filter->vmf_unicast, in vtnet_rx_filter_mac()
3805 error |= sglist_append(&sg, &filter->vmf_multicast, in vtnet_rx_filter_mac()
3811 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); in vtnet_rx_filter_mac()
3819 if_printf(ifp, "cannot enable all-multicast mode\n"); in vtnet_rx_filter_mac()
3837 MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER); in vtnet_exec_vlan_filter()
3851 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); in vtnet_exec_vlan_filter()
3863 MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER); in vtnet_rx_filter_vlan()
3868 w = sc->vtnet_vlan_filter[i]; in vtnet_rx_filter_vlan()
3870 while ((bit = ffs(w) - 1) != -1) { in vtnet_rx_filter_vlan()
3875 device_printf(sc->vtnet_dev, in vtnet_rx_filter_vlan()
3888 ifp = sc->vtnet_ifp; in vtnet_update_vlan_filter()
3898 sc->vtnet_vlan_filter[idx] |= (1 << bit); in vtnet_update_vlan_filter()
3900 sc->vtnet_vlan_filter[idx] &= ~(1 << bit); in vtnet_update_vlan_filter()
3905 device_printf(sc->vtnet_dev, in vtnet_update_vlan_filter()
3939 ifp = sc->vtnet_ifp; in vtnet_update_speed_duplex()
3941 if ((sc->vtnet_features & VIRTIO_NET_F_SPEED_DUPLEX) == 0) in vtnet_update_speed_duplex()
3945 speed = virtio_read_dev_config_4(sc->vtnet_dev, in vtnet_update_speed_duplex()
3956 if ((sc->vtnet_features & VIRTIO_NET_F_STATUS) == 0) in vtnet_is_link_up()
3959 status = virtio_read_dev_config_2(sc->vtnet_dev, in vtnet_is_link_up()
3971 ifp = sc->vtnet_ifp; in vtnet_update_link_status()
3976 if (link != 0 && sc->vtnet_link_active == 0) { in vtnet_update_link_status()
3978 sc->vtnet_link_active = 1; in vtnet_update_link_status()
3980 } else if (link == 0 && sc->vtnet_link_active != 0) { in vtnet_update_link_status()
3981 sc->vtnet_link_active = 0; in vtnet_update_link_status()
3999 ifmr->ifm_status = IFM_AVALID; in vtnet_ifmedia_sts()
4000 ifmr->ifm_active = IFM_ETHER; in vtnet_ifmedia_sts()
4004 ifmr->ifm_status |= IFM_ACTIVE; in vtnet_ifmedia_sts()
4005 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; in vtnet_ifmedia_sts()
4007 ifmr->ifm_active |= IFM_NONE; in vtnet_ifmedia_sts()
4015 if (sc->vtnet_flags & VTNET_FLAG_MAC) { in vtnet_get_macaddr()
4016 virtio_read_device_config_array(sc->vtnet_dev, in vtnet_get_macaddr()
4018 &sc->vtnet_hwaddr[0], sizeof(uint8_t), ETHER_ADDR_LEN); in vtnet_get_macaddr()
4021 sc->vtnet_hwaddr[0] = 0xB2; in vtnet_get_macaddr()
4022 arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0); in vtnet_get_macaddr()
4032 dev = sc->vtnet_dev; in vtnet_set_macaddr()
4034 if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) { in vtnet_set_macaddr()
4035 error = vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr); in vtnet_set_macaddr()
4041 /* MAC in config is read-only in modern VirtIO. */ in vtnet_set_macaddr()
4042 if (!vtnet_modern(sc) && sc->vtnet_flags & VTNET_FLAG_MAC) { in vtnet_set_macaddr()
4046 sc->vtnet_hwaddr[i]); in vtnet_set_macaddr()
4056 if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) in vtnet_attached_set_macaddr()
4066 m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); in vtnet_vlan_tag_remove()
4067 m->m_flags |= M_VLANTAG; in vtnet_vlan_tag_remove()
4071 ETHER_HDR_LEN - ETHER_TYPE_LEN); in vtnet_vlan_tag_remove()
4084 sc->vtnet_rx_process_limit = limit; in vtnet_set_rx_process_limit()
4093 struct vtnet_rxq_stats *stats; in vtnet_setup_rxq_sysctl() local
4096 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id); in vtnet_setup_rxq_sysctl()
4101 stats = &rxq->vtnrx_stats; in vtnet_setup_rxq_sysctl()
4104 &stats->vrxs_ipackets, "Receive packets"); in vtnet_setup_rxq_sysctl()
4106 &stats->vrxs_ibytes, "Receive bytes"); in vtnet_setup_rxq_sysctl()
4108 &stats->vrxs_iqdrops, "Receive drops"); in vtnet_setup_rxq_sysctl()
4110 &stats->vrxs_ierrors, "Receive errors"); in vtnet_setup_rxq_sysctl()
4112 &stats->vrxs_csum, "Receive checksum offloaded"); in vtnet_setup_rxq_sysctl()
4114 &stats->vrxs_csum_failed, "Receive checksum offload failed"); in vtnet_setup_rxq_sysctl()
4116 &stats->vrxs_host_lro, "Receive host segmentation offloaded"); in vtnet_setup_rxq_sysctl()
4118 &stats->vrxs_rescheduled, in vtnet_setup_rxq_sysctl()
4128 struct vtnet_txq_stats *stats; in vtnet_setup_txq_sysctl() local
4131 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id); in vtnet_setup_txq_sysctl()
4136 stats = &txq->vtntx_stats; in vtnet_setup_txq_sysctl()
4139 &stats->vtxs_opackets, "Transmit packets"); in vtnet_setup_txq_sysctl()
4141 &stats->vtxs_obytes, "Transmit bytes"); in vtnet_setup_txq_sysctl()
4143 &stats->vtxs_omcasts, "Transmit multicasts"); in vtnet_setup_txq_sysctl()
4145 &stats->vtxs_csum, "Transmit checksum offloaded"); in vtnet_setup_txq_sysctl()
4147 &stats->vtxs_tso, "Transmit TCP segmentation offloaded"); in vtnet_setup_txq_sysctl()
4149 &stats->vtxs_rescheduled, in vtnet_setup_txq_sysctl()
4162 dev = sc->vtnet_dev; in vtnet_setup_queue_sysctl()
4167 for (i = 0; i < sc->vtnet_req_vq_pairs; i++) { in vtnet_setup_queue_sysctl()
4168 vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]); in vtnet_setup_queue_sysctl()
4169 vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]); in vtnet_setup_queue_sysctl()
4177 struct vtnet_statistics *stats; in vtnet_setup_stat_sysctl() local
4183 stats = &sc->vtnet_stats; in vtnet_setup_stat_sysctl()
4184 stats->rx_csum_offloaded = rxaccum.vrxs_csum; in vtnet_setup_stat_sysctl()
4185 stats->rx_csum_failed = rxaccum.vrxs_csum_failed; in vtnet_setup_stat_sysctl()
4186 stats->rx_task_rescheduled = rxaccum.vrxs_rescheduled; in vtnet_setup_stat_sysctl()
4187 stats->tx_csum_offloaded = txaccum.vtxs_csum; in vtnet_setup_stat_sysctl()
4188 stats->tx_tso_offloaded = txaccum.vtxs_tso; in vtnet_setup_stat_sysctl()
4189 stats->tx_task_rescheduled = txaccum.vtxs_rescheduled; in vtnet_setup_stat_sysctl()
4192 CTLFLAG_RD, &stats->mbuf_alloc_failed, in vtnet_setup_stat_sysctl()
4196 CTLFLAG_RD, &stats->rx_frame_too_large, in vtnet_setup_stat_sysctl()
4199 CTLFLAG_RD, &stats->rx_enq_replacement_failed, in vtnet_setup_stat_sysctl()
4202 CTLFLAG_RD, &stats->rx_mergeable_failed, in vtnet_setup_stat_sysctl()
4205 CTLFLAG_RD, &stats->rx_csum_bad_ethtype, in vtnet_setup_stat_sysctl()
4209 CTLFLAG_RD, &stats->rx_csum_bad_ipproto, in vtnet_setup_stat_sysctl()
4212 CTLFLAG_RD, &stats->rx_csum_bad_offset, in vtnet_setup_stat_sysctl()
4215 CTLFLAG_RD, &stats->rx_csum_bad_proto, in vtnet_setup_stat_sysctl()
4218 CTLFLAG_RD, &stats->rx_csum_failed, in vtnet_setup_stat_sysctl()
4221 CTLFLAG_RD, &stats->rx_csum_offloaded, in vtnet_setup_stat_sysctl()
4224 CTLFLAG_RD, &stats->rx_task_rescheduled, in vtnet_setup_stat_sysctl()
4228 CTLFLAG_RD, &stats->tx_csum_unknown_ethtype, in vtnet_setup_stat_sysctl()
4232 CTLFLAG_RD, &stats->tx_csum_proto_mismatch, in vtnet_setup_stat_sysctl()
4236 CTLFLAG_RD, &stats->tx_tso_not_tcp, in vtnet_setup_stat_sysctl()
4239 CTLFLAG_RD, &stats->tx_tso_without_csum, in vtnet_setup_stat_sysctl()
4242 CTLFLAG_RD, &stats->tx_defragged, in vtnet_setup_stat_sysctl()
4245 CTLFLAG_RD, &stats->tx_defrag_failed, in vtnet_setup_stat_sysctl()
4248 CTLFLAG_RD, &stats->tx_csum_offloaded, in vtnet_setup_stat_sysctl()
4251 CTLFLAG_RD, &stats->tx_tso_offloaded, in vtnet_setup_stat_sysctl()
4254 CTLFLAG_RD, &stats->tx_task_rescheduled, in vtnet_setup_stat_sysctl()
4266 dev = sc->vtnet_dev; in vtnet_setup_sysctl()
4272 CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0, in vtnet_setup_sysctl()
4275 CTLFLAG_RD, &sc->vtnet_req_vq_pairs, 0, in vtnet_setup_sysctl()
4278 CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, in vtnet_setup_sysctl()
4288 sc->vtnet_lro_entry_count = vtnet_tunable_int(sc, in vtnet_load_tunables()
4290 if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES) in vtnet_load_tunables()
4291 sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES; in vtnet_load_tunables()
4293 sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc, in vtnet_load_tunables()
4301 return (virtqueue_enable_intr(rxq->vtnrx_vq)); in vtnet_rxq_enable_intr()
4308 virtqueue_disable_intr(rxq->vtnrx_vq); in vtnet_rxq_disable_intr()
4316 vq = txq->vtntx_vq; in vtnet_txq_enable_intr()
4332 virtqueue_disable_intr(txq->vtntx_vq); in vtnet_txq_disable_intr()
4341 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { in vtnet_enable_rx_interrupts()
4342 rxq = &sc->vtnet_rxqs[i]; in vtnet_enable_rx_interrupts()
4344 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); in vtnet_enable_rx_interrupts()
4353 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) in vtnet_enable_tx_interrupts()
4354 vtnet_txq_enable_intr(&sc->vtnet_txqs[i]); in vtnet_enable_tx_interrupts()
4370 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) in vtnet_disable_rx_interrupts()
4371 vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]); in vtnet_disable_rx_interrupts()
4379 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) in vtnet_disable_tx_interrupts()
4380 vtnet_txq_disable_intr(&sc->vtnet_txqs[i]); in vtnet_disable_tx_interrupts()
4397 "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob); in vtnet_tunable_int()
4412 *nrxr = sc->vtnet_req_vq_pairs; in vtnet_debugnet_init()
4414 *clsize = sc->vtnet_rx_clustersz; in vtnet_debugnet_init()
4431 sw_lro_enabled = (sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0; in vtnet_debugnet_event()
4433 sc->vtnet_flags &= ~VTNET_FLAG_SW_LRO; in vtnet_debugnet_event()
4437 sc->vtnet_flags |= VTNET_FLAG_SW_LRO; in vtnet_debugnet_event()
4454 txq = &sc->vtnet_txqs[0]; in vtnet_debugnet_transmit()
4472 (void)vtnet_txq_eof(&sc->vtnet_txqs[0]); in vtnet_debugnet_poll()
4473 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) in vtnet_debugnet_poll()
4474 (void)vtnet_rxq_eof(&sc->vtnet_rxqs[i]); in vtnet_debugnet_poll()