92 static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q,
95 static void t1_sge_check_pause(pesge *sge, struct freelQ *Q);
97 static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q);
99 static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend);
105 static void free_freelQ_buffers(pesge *sge, struct freelQ *Q);
306 	struct cmdQ *Q = &sge->cmdQ[qid];  in sge_data_out()  local
308 	spinlock_t *qlock = &Q->cq_qlock;  in sge_data_out()
310 	cmdQ_e *q = Q->cq_entries;  in sge_data_out()
314 	uint32_t entries_n = Q->cq_entries_n;  in sge_data_out()
316 	cmdQ_ce_t *cq = Q->cq_centries;  in sge_data_out()
380 	reclaim_cnt = Q->cq_complete;  in sge_data_out()
383 		free_cmdQ_buffers(sge, Q, reclaim_cnt);  in sge_data_out()
384 		Q->cq_complete = 0;  in sge_data_out()
387 	genbit = Q->cq_genbit;  in sge_data_out()
388 	pidx = Q->cq_pidx;  in sge_data_out()
389 	credits = Q->cq_credits;  in sge_data_out()
397 	atomic_sub(count, &Q->cq_credits);  in sge_data_out()
398 	Q->cq_pidx += count;  in sge_data_out()
399 	if (Q->cq_pidx >= entries_n) {  in sge_data_out()
400 		Q->cq_pidx -= entries_n;  in sge_data_out()
401 		Q->cq_genbit ^= 1;  in sge_data_out()
488 		if (atomic_read(Q->cq_asleep)) {  in sge_data_out()
489 			atomic_set(&Q->cq_asleep, 0);  in sge_data_out()
491 			atomic_set(&Q->cq_pio_pidx, Q->cq_pidx);  in sge_data_out()
589 	struct respQ *Q = &sge->respQ;  in sge_data_in()  local
591 	respQ_e *q = Q->rq_entries;		/* base response queue */  in sge_data_in()
592 	uint32_t cidx = Q->rq_cidx;  in sge_data_in()
593 	uint32_t genbit = Q->rq_genbit;  in sge_data_in()
594 	uint32_t entries_n = Q->rq_entries_n;  in sge_data_in()
595 	uint32_t credits = Q->rq_credits;  in sge_data_in()
596 	uint32_t credits_thresh = Q->rq_credits_thresh;  in sge_data_in()
603 	ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh;  in sge_data_in()
734 	Q->rq_genbit = genbit;  in sge_data_in()
735 	Q->rq_cidx = cidx;  in sge_data_in()
736 	Q->rq_credits = credits;  in sge_data_in()
803 t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload)  in t1_sge_rx()  argument
807 	struct freelQ_ce *cq = Q->fq_centries;  in t1_sge_rx()
808 	struct freelQ_ce *ce = &cq[Q->fq_cidx];  in t1_sge_rx()
810 	uint32_t cidx = Q->fq_cidx;  in t1_sge_rx()
811 	uint32_t entries_n = Q->fq_entries_n;  in t1_sge_rx()
812 	uint32_t sz = Q->fq_rx_buffer_size;  in t1_sge_rx()
819 	if (Q->fq_id)  in t1_sge_rx()
846 		e = &Q->fq_entries[cidx];  in t1_sge_rx()
885 		e = &Q->fq_entries[cidx];  in t1_sge_rx()
1007 	Q->fq_cidx = cidx;  in t1_sge_rx()
1009 	if (unlikely(--Q->fq_credits < (entries_n>>2)))  in t1_sge_rx()
1011 		alloc_freelQ_buffers(sge, Q);  in t1_sge_rx()
1017 t1_sge_check_pause(pesge *sge, struct freelQ *Q)  in t1_sge_check_pause()  argument
1029 	if (Q->fq_credits < Q->fq_pause_on_thresh) {  in t1_sge_check_pause()
1040 		alloc_freelQ_buffers(sge, Q);  in t1_sge_check_pause()
1042 	    (Q->fq_credits > Q->fq_pause_off_thresh)) {  in t1_sge_check_pause()
1055 	sge->intr_cnt.rx_fl_credits = Q->fq_credits;  in t1_sge_check_pause()
1060 alloc_freelQ_buffers(pesge *sge, struct freelQ *Q)  in alloc_freelQ_buffers()  argument
1062 	uint32_t pidx = Q->fq_pidx;  in alloc_freelQ_buffers()
1063 	struct freelQ_ce *ce = &Q->fq_centries[pidx];  in alloc_freelQ_buffers()
1064 	freelQ_e *fq = Q->fq_entries;		/* base of freelist Q */  in alloc_freelQ_buffers()
1065 	freelQ_e *e = &Q->fq_entries[pidx];  in alloc_freelQ_buffers()
1066 	uint32_t sz = Q->fq_rx_buffer_size;  in alloc_freelQ_buffers()
1068 	uint32_t credits = Q->fq_credits;  in alloc_freelQ_buffers()
1069 	uint32_t entries_n = Q->fq_entries_n;  in alloc_freelQ_buffers()
1070 	uint32_t genbit = Q->fq_genbit;  in alloc_freelQ_buffers()
1071 	ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh;  in alloc_freelQ_buffers()
1121 			ce = Q->fq_centries;  in alloc_freelQ_buffers()
1122 			e = Q->fq_entries;  in alloc_freelQ_buffers()
1130 	Q->fq_genbit = genbit;  in alloc_freelQ_buffers()
1131 	Q->fq_pidx = pidx;  in alloc_freelQ_buffers()
1132 	Q->fq_credits = credits;  in alloc_freelQ_buffers()
1165 free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend)  in free_cmdQ_buffers()  argument
1169 	struct cmdQ_ce *cq = Q->cq_centries;  in free_cmdQ_buffers()
1170 	uint32_t entries_n = Q->cq_entries_n;  in free_cmdQ_buffers()
1171 	uint32_t cidx = Q->cq_cidx;  in free_cmdQ_buffers()
1240 	Q->cq_cidx = cidx;  in free_cmdQ_buffers()
1241 	atomic_add(credits_pend, &Q->cq_credits);  in free_cmdQ_buffers()
1293 		struct freelQ *Q = &sge->freelQ[i];  in alloc_rx_resources()  local
1295 		Q->fq_id = i;  in alloc_rx_resources()
1296 		Q->fq_genbit = 1;  in alloc_rx_resources()
1297 		Q->fq_entries_n = p->freelQ_size[i];  in alloc_rx_resources()
1299 		Q->fq_pause_on_thresh = flq_pause_window;  in alloc_rx_resources()
1300 		Q->fq_pause_off_thresh = Q->fq_entries_n >> 1;  in alloc_rx_resources()
1302 		size = sizeof (freelQ_e) * Q->fq_entries_n;  in alloc_rx_resources()
1304 		Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj,  in alloc_rx_resources()
1305 		    size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT);  in alloc_rx_resources()
1308 		if (!Q->fq_entries)  in alloc_rx_resources()
1310 		memset(Q->fq_entries, 0, size);  in alloc_rx_resources()
1311 		size = sizeof (struct freelQ_ce) * Q->fq_entries_n;  in alloc_rx_resources()
1312 		Q->fq_centries = t1_os_malloc_wait_zero(size);  in alloc_rx_resources()
1313 		if (!Q->fq_centries)  in alloc_rx_resources()
1315 		memset(Q->fq_centries, 0, size);  in alloc_rx_resources()
1376 		struct cmdQ *Q = &sge->cmdQ[i];  in alloc_tx_resources()  local
1378 		Q->cq_genbit = 1;  in alloc_tx_resources()
1379 		Q->cq_entries_n = p->cmdQ_size[i];  in alloc_tx_resources()
1380 		atomic_set(&Q->cq_credits, Q->cq_entries_n);  in alloc_tx_resources()
1381 		atomic_set(&Q->cq_asleep, 1);  in alloc_tx_resources()
1383 		mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER,  in alloc_tx_resources()
1386 		size = sizeof (cmdQ_e) * Q->cq_entries_n;  in alloc_tx_resources()
1387 		Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj,  in alloc_tx_resources()
1388 		    size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT);  in alloc_tx_resources()
1390 		if (!Q->cq_entries)  in alloc_tx_resources()
1392 		memset(Q->cq_entries, 0, size);  in alloc_tx_resources()
1393 		size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;  in alloc_tx_resources()
1394 		Q->cq_centries = t1_os_malloc_wait_zero(size);  in alloc_tx_resources()
1395 		if (!Q->cq_centries)  in alloc_tx_resources()
1397 		memset(Q->cq_centries, 0, size);  in alloc_tx_resources()
1400 		pe_dma_handle_init(sge->obj, Q->cq_entries_n);  in alloc_tx_resources()
1529 		struct freelQ *Q = &sge->freelQ[i];  in free_rx_resources()  local
1531 		if (Q->fq_centries) {  in free_rx_resources()
1532 			free_freelQ_buffers(sge, Q);  in free_rx_resources()
1534 			t1_os_free(Q->fq_centries,  in free_rx_resources()
1535 			    Q->fq_entries_n * sizeof (freelQ_ce_t));  in free_rx_resources()
1537 		if (Q->fq_entries) {  in free_rx_resources()
1538 			size = sizeof (freelQ_e) * Q->fq_entries_n;  in free_rx_resources()
1541 			pe_os_free_contig(sge->obj, size, Q->fq_entries,  in free_rx_resources()
1542 			    Q->fq_pa, Q->fq_dh, Q->fq_ah);  in free_rx_resources()
1553 free_freelQ_buffers(pesge *sge, struct freelQ *Q)  in free_freelQ_buffers()  argument
1556 	struct freelQ_ce *cq = Q->fq_centries;  in free_freelQ_buffers()
1557 	uint32_t credits = Q->fq_credits;  in free_freelQ_buffers()
1558 	uint32_t entries_n = Q->fq_entries_n;  in free_freelQ_buffers()
1559 	uint32_t cidx = Q->fq_cidx;  in free_freelQ_buffers()
1560 	uint32_t i = Q->fq_id;  in free_freelQ_buffers()
1592 	Q->fq_cidx = cidx;  in free_freelQ_buffers()
1593 	Q->fq_credits = credits;  in free_freelQ_buffers()
1608 		struct cmdQ *Q = &sge->cmdQ[i];  in free_tx_resources()  local
1610 		if (Q->cq_centries) {  in free_tx_resources()
1611 			unsigned int pending = Q->cq_entries_n -  in free_tx_resources()
1612 			    atomic_read(Q->cq_credits);  in free_tx_resources()
1614 			mutex_destroy(&Q->cq_qlock);  in free_tx_resources()
1617 				free_cmdQ_buffers(sge, Q, pending);  in free_tx_resources()
1619 			size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;  in free_tx_resources()
1620 			t1_os_free(Q->cq_centries, size);  in free_tx_resources()
1623 		if (Q->cq_entries) {  in free_tx_resources()
1624 			size = sizeof (cmdQ_e) * Q->cq_entries_n;  in free_tx_resources()
1625 			pe_os_free_contig(sge->obj, size, Q->cq_entries,  in free_tx_resources()
1626 			    Q->cq_pa, Q->cq_dh, Q->cq_ah);  in free_tx_resources()