148 static void	vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries);
831 	struct vtnet_rxq *rxq;  in vtnet_init_rxq()  local
833 	rxq = &sc->vtnet_rxqs[id];  in vtnet_init_rxq()
835 	snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d",  in vtnet_init_rxq()
837 	mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF);  in vtnet_init_rxq()
839 	rxq->vtnrx_sc = sc;  in vtnet_init_rxq()
840 	rxq->vtnrx_id = id;  in vtnet_init_rxq()
842 	rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT);  in vtnet_init_rxq()
843 	if (rxq->vtnrx_sg == NULL)  in vtnet_init_rxq()
848 		if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp,  in vtnet_init_rxq()
854 	NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq);  in vtnet_init_rxq()
855 	rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT,  in vtnet_init_rxq()
856 	    taskqueue_thread_enqueue, &rxq->vtnrx_tq);  in vtnet_init_rxq()
858 	return (rxq->vtnrx_tq == NULL ? ENOMEM : 0);  in vtnet_init_rxq()
926 vtnet_destroy_rxq(struct vtnet_rxq *rxq)  in vtnet_destroy_rxq()  argument
929 	rxq->vtnrx_sc = NULL;  in vtnet_destroy_rxq()
930 	rxq->vtnrx_id = -1;  in vtnet_destroy_rxq()
933 	tcp_lro_free(&rxq->vtnrx_lro);  in vtnet_destroy_rxq()
936 	if (rxq->vtnrx_sg != NULL) {  in vtnet_destroy_rxq()
937 		sglist_free(rxq->vtnrx_sg);  in vtnet_destroy_rxq()
938 		rxq->vtnrx_sg = NULL;  in vtnet_destroy_rxq()
941 	if (mtx_initialized(&rxq->vtnrx_mtx) != 0)  in vtnet_destroy_rxq()
942 		mtx_destroy(&rxq->vtnrx_mtx);  in vtnet_destroy_rxq()
1029 	struct vtnet_rxq *rxq;  in vtnet_alloc_virtqueues()  local
1044 		rxq = &sc->vtnet_rxqs[i];  in vtnet_alloc_virtqueues()
1046 		    vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq,  in vtnet_alloc_virtqueues()
1047 		    "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id);  in vtnet_alloc_virtqueues()
1057 		rxq = &sc->vtnet_rxqs[i];  in vtnet_alloc_virtqueues()
1058 		VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, rxq, &rxq->vtnrx_vq,  in vtnet_alloc_virtqueues()
1059 		    "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id);  in vtnet_alloc_virtqueues()
1471 vtnet_rxq_populate(struct vtnet_rxq *rxq)  in vtnet_rxq_populate()  argument
1477 	error = vtnet_netmap_rxq_populate(rxq);  in vtnet_rxq_populate()
1482 	vq = rxq->vtnrx_vq;  in vtnet_rxq_populate()
1486 		error = vtnet_rxq_new_buf(rxq);  in vtnet_rxq_populate()
1506 vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq)  in vtnet_rxq_free_mbufs()  argument
1512 	struct netmap_kring *kring = netmap_kring_on(NA(rxq->vtnrx_sc->vtnet_ifp),  in vtnet_rxq_free_mbufs()
1513 							rxq->vtnrx_id, NR_RX);  in vtnet_rxq_free_mbufs()
1518 	vq = rxq->vtnrx_vq;  in vtnet_rxq_free_mbufs()
1527 	    ("%s: mbufs remaining in rx queue %p", __func__, rxq));  in vtnet_rxq_free_mbufs()
1575 vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *rxq, struct mbuf *m0,  in vtnet_rxq_replace_lro_nomrg_buf()  argument
1582 	sc = rxq->vtnrx_sc;  in vtnet_rxq_replace_lro_nomrg_buf()
1644 	error = vtnet_rxq_enqueue_buf(rxq, m_new);  in vtnet_rxq_replace_lro_nomrg_buf()
1666 vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len)  in vtnet_rxq_replace_buf()  argument
1672 	sc = rxq->vtnrx_sc;  in vtnet_rxq_replace_buf()
1675 		return (vtnet_rxq_replace_lro_nomrg_buf(rxq, m, len));  in vtnet_rxq_replace_buf()
1685 	error = vtnet_rxq_enqueue_buf(rxq, m_new);  in vtnet_rxq_replace_buf()
1696 vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m)  in vtnet_rxq_enqueue_buf()  argument
1702 	sc = rxq->vtnrx_sc;  in vtnet_rxq_enqueue_buf()
1703 	sg = rxq->vtnrx_sg;  in vtnet_rxq_enqueue_buf()
1707 	VTNET_RXQ_LOCK_ASSERT(rxq);  in vtnet_rxq_enqueue_buf()
1740 	return (virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg));  in vtnet_rxq_enqueue_buf()
1744 vtnet_rxq_new_buf(struct vtnet_rxq *rxq)  in vtnet_rxq_new_buf()  argument
1750 	sc = rxq->vtnrx_sc;  in vtnet_rxq_new_buf()
1756 	error = vtnet_rxq_enqueue_buf(rxq, m);  in vtnet_rxq_new_buf()
1764 vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, uint16_t etype,  in vtnet_rxq_csum_needs_csum()  argument
1770 	sc = rxq->vtnrx_sc;  in vtnet_rxq_csum_needs_csum()
1790 		error = vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr);  in vtnet_rxq_csum_needs_csum()
1837 vtnet_rxq_csum_data_valid(struct vtnet_rxq *rxq, struct mbuf *m,  in vtnet_rxq_csum_data_valid()  argument
1846 	sc = rxq->vtnrx_sc;  in vtnet_rxq_csum_data_valid()
1898 vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m,  in vtnet_rxq_csum()  argument
1917 		return (vtnet_rxq_csum_needs_csum(rxq, m, etype, hoff, hdr));  in vtnet_rxq_csum()
1919 		return (vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr));  in vtnet_rxq_csum()
1923 vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs)  in vtnet_rxq_discard_merged_bufs()  argument
1928 		m = virtqueue_dequeue(rxq->vtnrx_vq, NULL);  in vtnet_rxq_discard_merged_bufs()
1931 		vtnet_rxq_discard_buf(rxq, m);  in vtnet_rxq_discard_merged_bufs()
1936 vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m)  in vtnet_rxq_discard_buf()  argument
1944 	error = vtnet_rxq_enqueue_buf(rxq, m);  in vtnet_rxq_discard_buf()
1950 vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)  in vtnet_rxq_merged_eof()  argument
1956 	sc = rxq->vtnrx_sc;  in vtnet_rxq_merged_eof()
1957 	vq = rxq->vtnrx_vq;  in vtnet_rxq_merged_eof()
1966 			rxq->vtnrx_stats.vrxs_ierrors++;  in vtnet_rxq_merged_eof()
1970 		if (vtnet_rxq_new_buf(rxq) != 0) {  in vtnet_rxq_merged_eof()
1971 			rxq->vtnrx_stats.vrxs_iqdrops++;  in vtnet_rxq_merged_eof()
1972 			vtnet_rxq_discard_buf(rxq, m);  in vtnet_rxq_merged_eof()
1974 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);  in vtnet_rxq_merged_eof()
2000 vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m)  in vtnet_lro_rx()  argument
2004 	lro = &rxq->vtnrx_lro;  in vtnet_lro_rx()
2016 vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,  in vtnet_rxq_input()  argument
2022 	sc = rxq->vtnrx_sc;  in vtnet_rxq_input()
2038 	m->m_pkthdr.flowid = rxq->vtnrx_id;  in vtnet_rxq_input()
2043 		if (vtnet_rxq_csum(rxq, m, hdr) == 0)  in vtnet_rxq_input()
2044 			rxq->vtnrx_stats.vrxs_csum++;  in vtnet_rxq_input()
2046 			rxq->vtnrx_stats.vrxs_csum_failed++;  in vtnet_rxq_input()
2055 			rxq->vtnrx_stats.vrxs_host_lro++;  in vtnet_rxq_input()
2060 	rxq->vtnrx_stats.vrxs_ipackets++;  in vtnet_rxq_input()
2061 	rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len;  in vtnet_rxq_input()
2065 		if (vtnet_lro_rx(rxq, m) == 0)  in vtnet_rxq_input()
2074 vtnet_rxq_eof(struct vtnet_rxq *rxq)  in vtnet_rxq_eof()  argument
2082 	sc = rxq->vtnrx_sc;  in vtnet_rxq_eof()
2083 	vq = rxq->vtnrx_vq;  in vtnet_rxq_eof()
2088 	VTNET_RXQ_LOCK_ASSERT(rxq);  in vtnet_rxq_eof()
2101 			rxq->vtnrx_stats.vrxs_ierrors++;  in vtnet_rxq_eof()
2102 			vtnet_rxq_discard_buf(rxq, m);  in vtnet_rxq_eof()
2125 		if (vtnet_rxq_replace_buf(rxq, m, len) != 0) {  in vtnet_rxq_eof()
2126 			rxq->vtnrx_stats.vrxs_iqdrops++;  in vtnet_rxq_eof()
2127 			vtnet_rxq_discard_buf(rxq, m);  in vtnet_rxq_eof()
2129 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);  in vtnet_rxq_eof()
2139 			if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0)  in vtnet_rxq_eof()
2174 		vtnet_rxq_input(rxq, m, &lhdr);  in vtnet_rxq_eof()
2180 			tcp_lro_flush_all(&rxq->vtnrx_lro);  in vtnet_rxq_eof()
2190 vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries)  in vtnet_rx_vq_process()  argument
2199 	sc = rxq->vtnrx_sc;  in vtnet_rx_vq_process()
2202 	if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) {  in vtnet_rx_vq_process()
2209 		vtnet_rxq_disable_intr(rxq);  in vtnet_rx_vq_process()
2213 	VTNET_RXQ_LOCK(rxq);  in vtnet_rx_vq_process()
2224 	nmirq = netmap_rx_irq(ifp, rxq->vtnrx_id, &more);  in vtnet_rx_vq_process()
2226 		VTNET_RXQ_UNLOCK(rxq);  in vtnet_rx_vq_process()
2228 			taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);  in vtnet_rx_vq_process()
2236 		VTNET_RXQ_UNLOCK(rxq);  in vtnet_rx_vq_process()
2240 	more = vtnet_rxq_eof(rxq);  in vtnet_rx_vq_process()
2241 	if (more || vtnet_rxq_enable_intr(rxq) != 0) {  in vtnet_rx_vq_process()
2243 			vtnet_rxq_disable_intr(rxq);  in vtnet_rx_vq_process()
2251 		rxq->vtnrx_stats.vrxs_rescheduled++;  in vtnet_rx_vq_process()
2252 		VTNET_RXQ_UNLOCK(rxq);  in vtnet_rx_vq_process()
2253 		taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);  in vtnet_rx_vq_process()
2255 		VTNET_RXQ_UNLOCK(rxq);  in vtnet_rx_vq_process()
2261 	struct vtnet_rxq *rxq;  in vtnet_rx_vq_intr()  local
2263 	rxq = xrxq;  in vtnet_rx_vq_intr()
2264 	vtnet_rx_vq_process(rxq, VTNET_INTR_DISABLE_RETRIES);  in vtnet_rx_vq_intr()
2270 	struct vtnet_rxq *rxq;  in vtnet_rxq_tq_intr()  local
2272 	rxq = xrxq;  in vtnet_rxq_tq_intr()
2273 	vtnet_rx_vq_process(rxq, 0);  in vtnet_rxq_tq_intr()
3076 	struct vtnet_rxq *rxq;  in vtnet_start_taskqueues()  local
3092 		rxq = &sc->vtnet_rxqs[i];  in vtnet_start_taskqueues()
3093 		error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET,  in vtnet_start_taskqueues()
3094 		    "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id);  in vtnet_start_taskqueues()
3097 			    rxq->vtnrx_id);  in vtnet_start_taskqueues()
3113 	struct vtnet_rxq *rxq;  in vtnet_free_taskqueues()  local
3118 		rxq = &sc->vtnet_rxqs[i];  in vtnet_free_taskqueues()
3119 		if (rxq->vtnrx_tq != NULL) {  in vtnet_free_taskqueues()
3120 			taskqueue_free(rxq->vtnrx_tq);  in vtnet_free_taskqueues()
3121 			rxq->vtnrx_tq = NULL;  in vtnet_free_taskqueues()
3135 	struct vtnet_rxq *rxq;  in vtnet_drain_taskqueues()  local
3140 		rxq = &sc->vtnet_rxqs[i];  in vtnet_drain_taskqueues()
3141 		if (rxq->vtnrx_tq != NULL)  in vtnet_drain_taskqueues()
3142 			taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);  in vtnet_drain_taskqueues()
3156 	struct vtnet_rxq *rxq;  in vtnet_drain_rxtx_queues()  local
3161 		rxq = &sc->vtnet_rxqs[i];  in vtnet_drain_rxtx_queues()
3162 		vtnet_rxq_free_mbufs(rxq);  in vtnet_drain_rxtx_queues()
3172 	struct vtnet_rxq *rxq;  in vtnet_stop_rendezvous()  local
3185 		rxq = &sc->vtnet_rxqs[i];  in vtnet_stop_rendezvous()
3186 		VTNET_RXQ_LOCK(rxq);  in vtnet_stop_rendezvous()
3187 		VTNET_RXQ_UNLOCK(rxq);  in vtnet_stop_rendezvous()
3290 	struct vtnet_rxq *rxq;  in vtnet_init_rx_queues()  local
3309 		rxq = &sc->vtnet_rxqs[i];  in vtnet_init_rx_queues()
3312 		VTNET_RXQ_LOCK(rxq);  in vtnet_init_rx_queues()
3313 		error = vtnet_rxq_populate(rxq);  in vtnet_init_rx_queues()
3314 		VTNET_RXQ_UNLOCK(rxq);  in vtnet_init_rx_queues()
4089     struct sysctl_oid_list *child, struct vtnet_rxq *rxq)  in vtnet_setup_rxq_sysctl()  argument
4096 	snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id);  in vtnet_setup_rxq_sysctl()
4101 	stats = &rxq->vtnrx_stats;  in vtnet_setup_rxq_sysctl()
4298 vtnet_rxq_enable_intr(struct vtnet_rxq *rxq)  in vtnet_rxq_enable_intr()  argument
4301 	return (virtqueue_enable_intr(rxq->vtnrx_vq));  in vtnet_rxq_enable_intr()
4305 vtnet_rxq_disable_intr(struct vtnet_rxq *rxq)  in vtnet_rxq_disable_intr()  argument
4308 	virtqueue_disable_intr(rxq->vtnrx_vq);  in vtnet_rxq_disable_intr()
4338 	struct vtnet_rxq *rxq;  in vtnet_enable_rx_interrupts()  local
4342 		rxq = &sc->vtnet_rxqs[i];  in vtnet_enable_rx_interrupts()
4343 		if (vtnet_rxq_enable_intr(rxq) != 0)  in vtnet_enable_rx_interrupts()
4344 			taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);  in vtnet_enable_rx_interrupts()