19 ena_refill_rx(ena_rxq_t *rxq, uint16_t num)  in ena_refill_rx()  argument
21 	VERIFY3P(rxq, !=, NULL);  in ena_refill_rx()
22 	ASSERT(MUTEX_HELD(&rxq->er_lock));  in ena_refill_rx()
23 	ASSERT3U(num, <=, rxq->er_sq_num_descs);  in ena_refill_rx()
25 	const uint16_t modulo_mask = rxq->er_sq_num_descs - 1;  in ena_refill_rx()
26 	uint16_t tail_mod = rxq->er_sq_tail_idx & modulo_mask;  in ena_refill_rx()
29 		enahw_rx_desc_t *desc = &rxq->er_sq_descs[tail_mod];  in ena_refill_rx()
30 		ena_rx_ctrl_block_t *rcb = &rxq->er_rcbs[tail_mod];  in ena_refill_rx()
31 		uint16_t phase = rxq->er_sq_phase;  in ena_refill_rx()
33 		VERIFY3U(tail_mod, <, rxq->er_sq_num_descs);  in ena_refill_rx()
36 		VERIFY3P(desc, >=, rxq->er_sq_descs);  in ena_refill_rx()
38 		    (rxq->er_sq_descs + rxq->er_sq_num_descs - 1));  in ena_refill_rx()
43 		ena_set_dma_addr_values(rxq->er_ena,  in ena_refill_rx()
53 		rxq->er_sq_tail_idx++;  in ena_refill_rx()
54 		tail_mod = rxq->er_sq_tail_idx & modulo_mask;  in ena_refill_rx()
57 			rxq->er_sq_phase ^= 1;  in ena_refill_rx()
62 	ENA_DMA_SYNC(rxq->er_sq_dma, DDI_DMA_SYNC_FORDEV);  in ena_refill_rx()
63 	ena_hw_abs_write32(rxq->er_ena, rxq->er_sq_db_addr,  in ena_refill_rx()
64 	    rxq->er_sq_tail_idx);  in ena_refill_rx()
68 ena_free_rx_dma(ena_rxq_t *rxq)  in ena_free_rx_dma()  argument
70 	if (rxq->er_rcbs != NULL) {  in ena_free_rx_dma()
71 		for (uint_t i = 0; i < rxq->er_sq_num_descs; i++) {  in ena_free_rx_dma()
72 			ena_rx_ctrl_block_t *rcb = &rxq->er_rcbs[i];  in ena_free_rx_dma()
76 		kmem_free(rxq->er_rcbs,  in ena_free_rx_dma()
77 		    sizeof (*rxq->er_rcbs) * rxq->er_sq_num_descs);  in ena_free_rx_dma()
79 		rxq->er_rcbs = NULL;  in ena_free_rx_dma()
82 	ena_dma_free(&rxq->er_cq_dma);  in ena_free_rx_dma()
83 	rxq->er_cq_descs = NULL;  in ena_free_rx_dma()
84 	rxq->er_cq_num_descs = 0;  in ena_free_rx_dma()
86 	ena_dma_free(&rxq->er_sq_dma);  in ena_free_rx_dma()
87 	rxq->er_sq_descs = NULL;  in ena_free_rx_dma()
88 	rxq->er_sq_num_descs = 0;  in ena_free_rx_dma()
90 	rxq->er_state &= ~ENA_RXQ_STATE_HOST_ALLOC;  in ena_free_rx_dma()
94 ena_alloc_rx_dma(ena_rxq_t *rxq)  in ena_alloc_rx_dma()  argument
96 	ena_t *ena = rxq->er_ena;  in ena_alloc_rx_dma()
101 	cq_descs_sz = rxq->er_cq_num_descs * sizeof (*rxq->er_cq_descs);  in ena_alloc_rx_dma()
102 	sq_descs_sz = rxq->er_sq_num_descs * sizeof (*rxq->er_sq_descs);  in ena_alloc_rx_dma()
112 	if (!ena_dma_alloc(ena, &rxq->er_sq_dma, &sq_conf, sq_descs_sz)) {  in ena_alloc_rx_dma()
116 	rxq->er_sq_descs = (void *)rxq->er_sq_dma.edb_va;  in ena_alloc_rx_dma()
117 	rxq->er_rcbs = kmem_zalloc(sizeof (*rxq->er_rcbs) *  in ena_alloc_rx_dma()
118 	    rxq->er_sq_num_descs, KM_SLEEP);  in ena_alloc_rx_dma()
120 	for (uint_t i = 0; i < rxq->er_sq_num_descs; i++) {  in ena_alloc_rx_dma()
121 		ena_rx_ctrl_block_t *rcb = &rxq->er_rcbs[i];  in ena_alloc_rx_dma()
145 	if (!ena_dma_alloc(ena, &rxq->er_cq_dma, &cq_conf, cq_descs_sz)) {  in ena_alloc_rx_dma()
150 	rxq->er_cq_descs = (void *)rxq->er_cq_dma.edb_va;  in ena_alloc_rx_dma()
151 	rxq->er_state |= ENA_RXQ_STATE_HOST_ALLOC;  in ena_alloc_rx_dma()
155 	ena_free_rx_dma(rxq);  in ena_alloc_rx_dma()
160 ena_alloc_rxq(ena_rxq_t *rxq)  in ena_alloc_rxq()  argument
163 	ena_t *ena = rxq->er_ena;  in ena_alloc_rxq()
171 	if ((ret = ena_alloc_rx_dma(rxq)) != 0) {  in ena_alloc_rxq()
173 		    rxq->er_rxqs_idx, ret);  in ena_alloc_rxq()
177 	ASSERT(rxq->er_state & ENA_RXQ_STATE_HOST_ALLOC);  in ena_alloc_rxq()
182 	ret = ena_create_cq(ena,  rxq->er_cq_num_descs,  in ena_alloc_rxq()
183 	    rxq->er_cq_dma.edb_cookie->dmac_laddress, false,  in ena_alloc_rxq()
184 	    rxq->er_intr_vector, &cq_hw_idx, &cq_unmask_addr, &cq_numanode);  in ena_alloc_rxq()
187 		ena_err(ena, "failed to create Rx CQ %u: %d", rxq->er_rxqs_idx,  in ena_alloc_rxq()
193 	rxq->er_cq_phase = 1;  in ena_alloc_rxq()
194 	rxq->er_cq_head_idx = 0;  in ena_alloc_rxq()
195 	rxq->er_cq_hw_idx = cq_hw_idx;  in ena_alloc_rxq()
196 	rxq->er_cq_unmask_addr = cq_unmask_addr;  in ena_alloc_rxq()
197 	rxq->er_cq_numa_addr = cq_numanode;  in ena_alloc_rxq()
198 	rxq->er_state |= ENA_RXQ_STATE_CQ_CREATED;  in ena_alloc_rxq()
208 	ASSERT3U(rxq->er_sq_num_descs, ==, rxq->er_cq_num_descs);  in ena_alloc_rxq()
209 	ret = ena_create_sq(ena, rxq->er_sq_num_descs,  in ena_alloc_rxq()
210 	    rxq->er_sq_dma.edb_cookie->dmac_laddress, false, cq_hw_idx,  in ena_alloc_rxq()
214 		ena_err(ena, "failed to create Rx SQ %u: %d", rxq->er_rxqs_idx,  in ena_alloc_rxq()
220 	rxq->er_sq_hw_idx = sq_hw_idx;  in ena_alloc_rxq()
221 	rxq->er_sq_db_addr = sq_db_addr;  in ena_alloc_rxq()
223 	rxq->er_sq_phase = 1;  in ena_alloc_rxq()
224 	rxq->er_sq_tail_idx = 0;  in ena_alloc_rxq()
225 	rxq->er_sq_avail_descs = rxq->er_sq_num_descs;  in ena_alloc_rxq()
226 	rxq->er_mode = ENA_RXQ_MODE_INTR;  in ena_alloc_rxq()
227 	rxq->er_state |= ENA_RXQ_STATE_SQ_CREATED;  in ena_alloc_rxq()
233 ena_cleanup_rxq(ena_rxq_t *rxq, bool resetting)  in ena_cleanup_rxq()  argument
236 	ena_t *ena = rxq->er_ena;  in ena_cleanup_rxq()
238 	if ((rxq->er_state & ENA_RXQ_STATE_SQ_CREATED) != 0) {  in ena_cleanup_rxq()
240 			ret = ena_destroy_sq(ena, rxq->er_sq_hw_idx, false);  in ena_cleanup_rxq()
244 				    rxq->er_rxqs_idx, ret);  in ena_cleanup_rxq()
248 		rxq->er_sq_hw_idx = 0;  in ena_cleanup_rxq()
249 		rxq->er_sq_db_addr = NULL;  in ena_cleanup_rxq()
250 		rxq->er_sq_tail_idx = 0;  in ena_cleanup_rxq()
251 		rxq->er_sq_phase = 0;  in ena_cleanup_rxq()
252 		rxq->er_state &= ~ENA_RXQ_STATE_SQ_CREATED;  in ena_cleanup_rxq()
253 		rxq->er_state &= ~ENA_RXQ_STATE_SQ_FILLED;  in ena_cleanup_rxq()
256 	if ((rxq->er_state & ENA_RXQ_STATE_CQ_CREATED) != 0) {  in ena_cleanup_rxq()
258 			ret = ena_destroy_cq(ena, rxq->er_cq_hw_idx);  in ena_cleanup_rxq()
262 				    rxq->er_rxqs_idx, ret);  in ena_cleanup_rxq()
266 		rxq->er_cq_hw_idx = 0;  in ena_cleanup_rxq()
267 		rxq->er_cq_head_idx = 0;  in ena_cleanup_rxq()
268 		rxq->er_cq_phase = 0;  in ena_cleanup_rxq()
269 		rxq->er_cq_unmask_addr = NULL;  in ena_cleanup_rxq()
270 		rxq->er_cq_numa_addr = NULL;  in ena_cleanup_rxq()
271 		rxq->er_state &= ~ENA_RXQ_STATE_CQ_CREATED;  in ena_cleanup_rxq()
274 	ena_free_rx_dma(rxq);  in ena_cleanup_rxq()
275 	ASSERT3S(rxq->er_state, ==, ENA_RXQ_STATE_NONE);  in ena_cleanup_rxq()
281 	ena_rxq_t *rxq = (ena_rxq_t *)rh;  in ena_ring_rx_stop()  local
284 	intr_ctrl = ena_hw_abs_read32(rxq->er_ena, rxq->er_cq_unmask_addr);  in ena_ring_rx_stop()
286 	ena_hw_abs_write32(rxq->er_ena, rxq->er_cq_unmask_addr, intr_ctrl);  in ena_ring_rx_stop()
288 	rxq->er_state &= ~ENA_RXQ_STATE_RUNNING;  in ena_ring_rx_stop()
289 	rxq->er_state &= ~ENA_RXQ_STATE_READY;  in ena_ring_rx_stop()
295 	ena_rxq_t *rxq = (ena_rxq_t *)rh;  in ena_ring_rx_start()  local
296 	ena_t *ena = rxq->er_ena;  in ena_ring_rx_start()
299 	ena_dbg(ena, "ring_rx_start %p: state 0x%x", rxq, rxq->er_state);  in ena_ring_rx_start()
301 	mutex_enter(&rxq->er_lock);  in ena_ring_rx_start()
302 	if ((rxq->er_state & ENA_RXQ_STATE_SQ_FILLED) == 0) {  in ena_ring_rx_start()
318 		ena_refill_rx(rxq, rxq->er_sq_num_descs - 1);  in ena_ring_rx_start()
319 		rxq->er_state |= ENA_RXQ_STATE_SQ_FILLED;  in ena_ring_rx_start()
321 	rxq->er_m_gen_num = gen_num;  in ena_ring_rx_start()
322 	rxq->er_intr_limit = ena->ena_rxq_intr_limit;  in ena_ring_rx_start()
323 	mutex_exit(&rxq->er_lock);  in ena_ring_rx_start()
325 	rxq->er_state |= ENA_RXQ_STATE_READY;  in ena_ring_rx_start()
327 	intr_ctrl = ena_hw_abs_read32(ena, rxq->er_cq_unmask_addr);  in ena_ring_rx_start()
329 	ena_hw_abs_write32(ena, rxq->er_cq_unmask_addr, intr_ctrl);  in ena_ring_rx_start()
330 	rxq->er_state |= ENA_RXQ_STATE_RUNNING;  in ena_ring_rx_start()
335 ena_ring_rx(ena_rxq_t *rxq, int poll_bytes)  in ena_ring_rx()  argument
337 	ena_t *ena = rxq->er_ena;  in ena_ring_rx()
338 	const uint16_t modulo_mask = rxq->er_cq_num_descs - 1;  in ena_ring_rx()
339 	uint16_t head_mod = rxq->er_cq_head_idx & modulo_mask;  in ena_ring_rx()
347 	ASSERT(MUTEX_HELD(&rxq->er_lock));  in ena_ring_rx()
348 	ENA_DMA_SYNC(rxq->er_cq_dma, DDI_DMA_SYNC_FORKERNEL);  in ena_ring_rx()
354 	cdesc = &rxq->er_cq_descs[head_mod];  in ena_ring_rx()
355 	VERIFY3P(cdesc, >=, rxq->er_cq_descs);  in ena_ring_rx()
356 	VERIFY3P(cdesc, <=, (rxq->er_cq_descs + rxq->er_cq_num_descs - 1));  in ena_ring_rx()
358 	while (ENAHW_RX_CDESC_PHASE(cdesc) == rxq->er_cq_phase) {  in ena_ring_rx()
368 		VERIFY3U(head_mod, <, rxq->er_cq_num_descs);  in ena_ring_rx()
385 			mutex_enter(&rxq->er_stat_lock);  in ena_ring_rx()
386 			rxq->er_stat.ers_multi_desc.value.ui64++;  in ena_ring_rx()
387 			mutex_exit(&rxq->er_stat_lock);  in ena_ring_rx()
392 		VERIFY3U(req_id, <, rxq->er_cq_num_descs);  in ena_ring_rx()
393 		rcb = &rxq->er_rcbs[req_id];  in ena_ring_rx()
404 			mutex_enter(&rxq->er_stat_lock);  in ena_ring_rx()
405 			rxq->er_stat.ers_allocb_fail.value.ui64++;  in ena_ring_rx()
406 			mutex_exit(&rxq->er_stat_lock);  in ena_ring_rx()
458 				mutex_enter(&rxq->er_stat_lock);  in ena_ring_rx()
459 				rxq->er_stat.ers_hck_ipv4_err.value.ui64++;  in ena_ring_rx()
460 				mutex_exit(&rxq->er_stat_lock);  in ena_ring_rx()
474 				mutex_enter(&rxq->er_stat_lock);  in ena_ring_rx()
475 				rxq->er_stat.ers_hck_l4_err.value.ui64++;  in ena_ring_rx()
476 				mutex_exit(&rxq->er_stat_lock);  in ena_ring_rx()
496 		rxq->er_cq_head_idx++;  in ena_ring_rx()
497 		head_mod = rxq->er_cq_head_idx & modulo_mask;  in ena_ring_rx()
499 			rxq->er_cq_phase ^= 1;  in ena_ring_rx()
503 		} else if (!polling && num_frames >= rxq->er_intr_limit) {  in ena_ring_rx()
504 			mutex_enter(&rxq->er_stat_lock);  in ena_ring_rx()
505 			rxq->er_stat.ers_intr_limit.value.ui64++;  in ena_ring_rx()
506 			mutex_exit(&rxq->er_stat_lock);  in ena_ring_rx()
510 		cdesc = &rxq->er_cq_descs[head_mod];  in ena_ring_rx()
511 		VERIFY3P(cdesc, >=, rxq->er_cq_descs);  in ena_ring_rx()
513 		    (rxq->er_cq_descs + rxq->er_cq_num_descs - 1));  in ena_ring_rx()
517 		mutex_enter(&rxq->er_stat_lock);  in ena_ring_rx()
518 		rxq->er_stat.ers_packets.value.ui64 += num_frames;  in ena_ring_rx()
519 		rxq->er_stat.ers_bytes.value.ui64 += total_bytes;  in ena_ring_rx()
520 		mutex_exit(&rxq->er_stat_lock);  in ena_ring_rx()
522 		DTRACE_PROBE5(rx__frames, ena_rxq_t *, rxq, mblk_t *, head,  in ena_ring_rx()
524 		ena_refill_rx(rxq, num_frames);  in ena_ring_rx()
531 ena_rx_intr_work(ena_rxq_t *rxq)  in ena_rx_intr_work()  argument
535 	mutex_enter(&rxq->er_lock);  in ena_rx_intr_work()
536 	mp = ena_ring_rx(rxq, ENA_INTERRUPT_MODE);  in ena_rx_intr_work()
537 	mutex_exit(&rxq->er_lock);  in ena_rx_intr_work()
543 	mac_rx_ring(rxq->er_ena->ena_mh, rxq->er_mrh, mp, rxq->er_m_gen_num);  in ena_rx_intr_work()
549 	ena_rxq_t *rxq = rh;  in ena_ring_rx_poll()  local
554 	mutex_enter(&rxq->er_lock);  in ena_ring_rx_poll()
555 	mp = ena_ring_rx(rxq, poll_bytes);  in ena_ring_rx_poll()
556 	mutex_exit(&rxq->er_lock);  in ena_ring_rx_poll()