699 	struct snd_queue *sq;  in nicvf_snd_pkt_handler()  local
703 	sq = &nic->qs->sq[cqe_tx->sq_idx];  in nicvf_snd_pkt_handler()
705 	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);  in nicvf_snd_pkt_handler()
714 	dmap = (bus_dmamap_t)sq->snd_buff[cqe_tx->sqe_ptr].dmap;  in nicvf_snd_pkt_handler()
715 	bus_dmamap_unload(sq->snd_buff_dmat, dmap);  in nicvf_snd_pkt_handler()
717 	mbuf = (struct mbuf *)sq->snd_buff[cqe_tx->sqe_ptr].mbuf;  in nicvf_snd_pkt_handler()
720 		sq->snd_buff[cqe_tx->sqe_ptr].mbuf = NULL;  in nicvf_snd_pkt_handler()
721 		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);  in nicvf_snd_pkt_handler()
739 	struct snd_queue *sq = &qs->sq[cq_idx];  in nicvf_cq_intr_handler()  local
812 		taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);  in nicvf_cq_intr_handler()
867 		nicvf_sq_free_used_descs(nic, &qs->sq[qidx], qidx);  in nicvf_qs_err_task()
868 		nicvf_sq_enable(nic, &qs->sq[qidx], qidx);  in nicvf_qs_err_task()
983 nicvf_xmit_locked(struct snd_queue *sq)  in nicvf_xmit_locked()  argument
990 	NICVF_TX_LOCK_ASSERT(sq);  in nicvf_xmit_locked()
992 	nic = sq->nic;  in nicvf_xmit_locked()
996 	while ((next = drbr_peek(ifp, sq->br)) != NULL) {  in nicvf_xmit_locked()
1000 		err = nicvf_tx_mbuf_locked(sq, &next);  in nicvf_xmit_locked()
1003 				drbr_advance(ifp, sq->br);  in nicvf_xmit_locked()
1005 				drbr_putback(ifp, sq->br, next);  in nicvf_xmit_locked()
1009 		drbr_advance(ifp, sq->br);  in nicvf_xmit_locked()
1017 	struct snd_queue *sq = (struct snd_queue *)arg;  in nicvf_snd_task()  local
1022 	nic = sq->nic;  in nicvf_snd_task()
1033 	NICVF_TX_LOCK(sq);  in nicvf_snd_task()
1034 	err = nicvf_xmit_locked(sq);  in nicvf_snd_task()
1035 	NICVF_TX_UNLOCK(sq);  in nicvf_snd_task()
1038 		taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);  in nicvf_snd_task()
1043 nicvf_init_snd_queue(struct nicvf *nic, struct snd_queue *sq, int q_len,  in nicvf_init_snd_queue()  argument
1050 	snprintf(sq->mtx_name, sizeof(sq->mtx_name), "%s: SQ(%d) lock",  in nicvf_init_snd_queue()
1052 	mtx_init(&sq->mtx, sq->mtx_name, NULL, MTX_DEF);  in nicvf_init_snd_queue()
1054 	NICVF_TX_LOCK(sq);  in nicvf_init_snd_queue()
1056 	sq->br = buf_ring_alloc(q_len / MIN_SQ_DESC_PER_PKT_XMIT, M_DEVBUF,  in nicvf_init_snd_queue()
1057 	    M_NOWAIT, &sq->mtx);  in nicvf_init_snd_queue()
1058 	if (sq->br == NULL) {  in nicvf_init_snd_queue()
1066 	err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE,  in nicvf_init_snd_queue()
1074 	sq->desc = sq->dmem.base;  in nicvf_init_snd_queue()
1075 	sq->head = sq->tail = 0;  in nicvf_init_snd_queue()
1076 	atomic_store_rel_int(&sq->free_cnt, q_len - 1);  in nicvf_init_snd_queue()
1077 	sq->thresh = SND_QUEUE_THRESH;  in nicvf_init_snd_queue()
1078 	sq->idx = qidx;  in nicvf_init_snd_queue()
1079 	sq->nic = nic;  in nicvf_init_snd_queue()
1098 	    &sq->snd_buff_dmat);		/* dmat */  in nicvf_init_snd_queue()
1107 	sq->snd_buff = malloc(sizeof(*sq->snd_buff) * q_len, M_NICVF,  in nicvf_init_snd_queue()
1109 	if (sq->snd_buff == NULL) {  in nicvf_init_snd_queue()
1118 		err = bus_dmamap_create(sq->snd_buff_dmat, 0,  in nicvf_init_snd_queue()
1119 		    &sq->snd_buff[i].dmap);  in nicvf_init_snd_queue()
1126 	NICVF_TX_UNLOCK(sq);  in nicvf_init_snd_queue()
1129 	TASK_INIT(&sq->snd_task, 0, nicvf_snd_task, sq);  in nicvf_init_snd_queue()
1130 	sq->snd_taskq = taskqueue_create_fast("nicvf_snd_taskq", M_WAITOK,  in nicvf_init_snd_queue()
1131 	    taskqueue_thread_enqueue, &sq->snd_taskq);  in nicvf_init_snd_queue()
1132 	taskqueue_start_threads(&sq->snd_taskq, 1, PI_NET, "%s: snd_taskq(%d)",  in nicvf_init_snd_queue()
1137 	NICVF_TX_UNLOCK(sq);  in nicvf_init_snd_queue()
1142 nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)  in nicvf_free_snd_queue()  argument
1148 	if (sq == NULL)  in nicvf_free_snd_queue()
1151 	if (sq->snd_taskq != NULL) {  in nicvf_free_snd_queue()
1153 		while (taskqueue_cancel(sq->snd_taskq, &sq->snd_task, NULL) != 0)  in nicvf_free_snd_queue()
1154 			taskqueue_drain(sq->snd_taskq, &sq->snd_task);  in nicvf_free_snd_queue()
1156 		taskqueue_free(sq->snd_taskq);  in nicvf_free_snd_queue()
1157 		sq->snd_taskq = NULL;  in nicvf_free_snd_queue()
1160 	NICVF_TX_LOCK(sq);  in nicvf_free_snd_queue()
1161 	if (sq->snd_buff_dmat != NULL) {  in nicvf_free_snd_queue()
1162 		if (sq->snd_buff != NULL) {  in nicvf_free_snd_queue()
1164 				m_freem(sq->snd_buff[i].mbuf);  in nicvf_free_snd_queue()
1165 				sq->snd_buff[i].mbuf = NULL;  in nicvf_free_snd_queue()
1167 				bus_dmamap_unload(sq->snd_buff_dmat,  in nicvf_free_snd_queue()
1168 				    sq->snd_buff[i].dmap);  in nicvf_free_snd_queue()
1169 				err = bus_dmamap_destroy(sq->snd_buff_dmat,  in nicvf_free_snd_queue()
1170 				    sq->snd_buff[i].dmap);  in nicvf_free_snd_queue()
1182 		free(sq->snd_buff, M_NICVF);  in nicvf_free_snd_queue()
1184 		err = bus_dma_tag_destroy(sq->snd_buff_dmat);  in nicvf_free_snd_queue()
1190 	if (sq->br != NULL)  in nicvf_free_snd_queue()
1191 		drbr_free(sq->br, M_DEVBUF);  in nicvf_free_snd_queue()
1193 	if (sq->dmem.base != NULL)  in nicvf_free_snd_queue()
1194 		nicvf_free_q_desc_mem(nic, &sq->dmem);  in nicvf_free_snd_queue()
1196 	NICVF_TX_UNLOCK(sq);  in nicvf_free_snd_queue()
1198 	mtx_destroy(&sq->mtx);  in nicvf_free_snd_queue()
1199 	memset(sq->mtx_name, 0, sizeof(sq->mtx_name));  in nicvf_free_snd_queue()
1411 	struct snd_queue *sq;  in nicvf_snd_queue_config()  local
1414 	sq = &qs->sq[qidx];  in nicvf_snd_queue_config()
1415 	sq->enable = enable;  in nicvf_snd_queue_config()
1417 	if (!sq->enable) {  in nicvf_snd_queue_config()
1425 	sq->cq_qs = qs->vnic_id;  in nicvf_snd_queue_config()
1426 	sq->cq_idx = qidx;  in nicvf_snd_queue_config()
1429 	mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG;  in nicvf_snd_queue_config()
1430 	mbx.sq.qs_num = qs->vnic_id;  in nicvf_snd_queue_config()
1431 	mbx.sq.sq_num = qidx;  in nicvf_snd_queue_config()
1432 	mbx.sq.sqs_mode = nic->sqs_mode;  in nicvf_snd_queue_config()
1433 	mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx;  in nicvf_snd_queue_config()
1438 	    (uint64_t)(sq->dmem.phys_base));  in nicvf_snd_queue_config()
1449 	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh);  in nicvf_snd_queue_config()
1550 		nicvf_free_snd_queue(nic, &qs->sq[qidx]);  in nicvf_free_resources()
1568 		if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx))  in nicvf_alloc_resources()
1661 nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)  in nicvf_get_sq_desc()  argument
1665 	qentry = sq->tail;  in nicvf_get_sq_desc()
1666 	atomic_subtract_int(&sq->free_cnt, desc_cnt);  in nicvf_get_sq_desc()
1667 	sq->tail += desc_cnt;  in nicvf_get_sq_desc()
1668 	sq->tail &= (sq->dmem.q_len - 1);  in nicvf_get_sq_desc()
1675 nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)  in nicvf_put_sq_desc()  argument
1678 	atomic_add_int(&sq->free_cnt, desc_cnt);  in nicvf_put_sq_desc()
1679 	sq->head += desc_cnt;  in nicvf_put_sq_desc()
1680 	sq->head &= (sq->dmem.q_len - 1);  in nicvf_put_sq_desc()
1684 nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry)  in nicvf_get_nxt_sqentry()  argument
1687 	qentry &= (sq->dmem.q_len - 1);  in nicvf_get_nxt_sqentry()
1692 nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)  in nicvf_sq_enable()  argument
1714 nicvf_sq_free_used_descs(struct nicvf *nic, struct snd_queue *sq, int qidx)  in nicvf_sq_free_used_descs()  argument
1720 	NICVF_TX_LOCK(sq);  in nicvf_sq_free_used_descs()
1722 	while (sq->head != head) {  in nicvf_sq_free_used_descs()
1723 		hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);  in nicvf_sq_free_used_descs()
1725 			nicvf_put_sq_desc(sq, 1);  in nicvf_sq_free_used_descs()
1728 		snd_buff = &sq->snd_buff[sq->head];  in nicvf_sq_free_used_descs()
1730 			bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);  in nicvf_sq_free_used_descs()
1732 			sq->snd_buff[sq->head].mbuf = NULL;  in nicvf_sq_free_used_descs()
1734 		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);  in nicvf_sq_free_used_descs()
1736 	NICVF_TX_UNLOCK(sq);  in nicvf_sq_free_used_descs()
1744 nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,  in nicvf_sq_add_hdr_subdesc()  argument
1762 	nic = sq->nic;  in nicvf_sq_add_hdr_subdesc()
1764 	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);  in nicvf_sq_add_hdr_subdesc()
1765 	sq->snd_buff[qentry].mbuf = mbuf;  in nicvf_sq_add_hdr_subdesc()
1790 			sq->snd_buff[qentry].mbuf = NULL;  in nicvf_sq_add_hdr_subdesc()
1804 			sq->snd_buff[qentry].mbuf = mbuf;  in nicvf_sq_add_hdr_subdesc()
1828 				sq->snd_buff[qentry].mbuf = mbuf;  in nicvf_sq_add_hdr_subdesc()
1840 				sq->snd_buff[qentry].mbuf = mbuf;  in nicvf_sq_add_hdr_subdesc()
1852 				sq->snd_buff[qentry].mbuf = mbuf;  in nicvf_sq_add_hdr_subdesc()
1883 static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,  in nicvf_sq_add_gather_subdesc()  argument
1888 	qentry &= (sq->dmem.q_len - 1);  in nicvf_sq_add_gather_subdesc()
1889 	gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry);  in nicvf_sq_add_gather_subdesc()
1900 nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf **mbufp)  in nicvf_tx_mbuf_locked()  argument
1909 	NICVF_TX_LOCK_ASSERT(sq);  in nicvf_tx_mbuf_locked()
1911 	if (sq->free_cnt == 0)  in nicvf_tx_mbuf_locked()
1914 	snd_buff = &sq->snd_buff[sq->tail];  in nicvf_tx_mbuf_locked()
1916 	err = bus_dmamap_load_mbuf_sg(sq->snd_buff_dmat, snd_buff->dmap,  in nicvf_tx_mbuf_locked()
1927 	if (subdesc_cnt > sq->free_cnt) {  in nicvf_tx_mbuf_locked()
1929 		bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);  in nicvf_tx_mbuf_locked()
1933 	qentry = nicvf_get_sq_desc(sq, subdesc_cnt);  in nicvf_tx_mbuf_locked()
1936 	err = nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, *mbufp,  in nicvf_tx_mbuf_locked()
1939 		nicvf_put_sq_desc(sq, subdesc_cnt);  in nicvf_tx_mbuf_locked()
1940 		bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);  in nicvf_tx_mbuf_locked()
1950 		qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_tx_mbuf_locked()
1951 		nicvf_sq_add_gather_subdesc(sq, qentry, segs[seg].ds_len,  in nicvf_tx_mbuf_locked()
1956 	bus_dmamap_sync(sq->dmem.dmat, sq->dmem.dmap, BUS_DMASYNC_PREWRITE);  in nicvf_tx_mbuf_locked()
1958 	dprintf(sq->nic->dev, "%s: sq->idx: %d, subdesc_cnt: %d\n",  in nicvf_tx_mbuf_locked()
1959 	    __func__, sq->idx, subdesc_cnt);  in nicvf_tx_mbuf_locked()
1961 	nicvf_queue_reg_write(sq->nic, NIC_QSET_SQ_0_7_DOOR,  in nicvf_tx_mbuf_locked()
1962 	    sq->idx, subdesc_cnt);  in nicvf_tx_mbuf_locked()
2215 	struct snd_queue *sq;  in nicvf_update_sq_stats()  local
2221 	sq = &nic->qs->sq[sq_idx];  in nicvf_update_sq_stats()
2222 	sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS);  in nicvf_update_sq_stats()
2223 	sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS);  in nicvf_update_sq_stats()