49 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)  in sfxge_ev_qcomplete()  argument
56 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_qcomplete()
58 	sc = evq->sc;  in sfxge_ev_qcomplete()
59 	index = evq->index;  in sfxge_ev_qcomplete()
62 	if ((txq = evq->txq) != NULL) {  in sfxge_ev_qcomplete()
63 		evq->txq = NULL;  in sfxge_ev_qcomplete()
64 		evq->txqs = &(evq->txq);  in sfxge_ev_qcomplete()
76 				sfxge_tx_qcomplete(txq, evq);  in sfxge_ev_qcomplete()
87 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)  in sfxge_get_rxq_by_label()  argument
93 	rxq = evq->sc->rxq[evq->index];  in sfxge_get_rxq_by_label()
96 	KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));  in sfxge_get_rxq_by_label()
105 	struct sfxge_evq *evq;  in sfxge_ev_rx()  local
112 	evq = arg;  in sfxge_ev_rx()
113 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_rx()
115 	sc = evq->sc;  in sfxge_ev_rx()
117 	if (evq->exception)  in sfxge_ev_rx()
120 	rxq = sfxge_get_rxq_by_label(evq, label);  in sfxge_ev_rx()
132 			evq->exception = B_TRUE;  in sfxge_ev_rx()
157 	evq->rx_done++;  in sfxge_ev_rx()
160 		sfxge_ev_qcomplete(evq, B_FALSE);  in sfxge_ev_rx()
163 	return (evq->rx_done >= SFXGE_EV_BATCH);  in sfxge_ev_rx()
169 	struct sfxge_evq *evq;  in sfxge_ev_exception()  local
172 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_exception()
173 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_exception()
175 	sc = evq->sc;  in sfxge_ev_exception()
177 	DBGPRINT(sc->dev, "[%d] %s", evq->index,  in sfxge_ev_exception()
189 	evq->exception = B_TRUE;  in sfxge_ev_exception()
204 	struct sfxge_evq *evq;  in sfxge_ev_rxq_flush_done()  local
210 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_rxq_flush_done()
211 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_rxq_flush_done()
213 	sc = evq->sc;  in sfxge_ev_rxq_flush_done()
220 	if (index == evq->index) {  in sfxge_ev_rxq_flush_done()
225 	evq = sc->evq[index];  in sfxge_ev_rxq_flush_done()
228 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,  in sfxge_ev_rxq_flush_done()
230 	efx_ev_qpost(evq->common, magic);  in sfxge_ev_rxq_flush_done()
238 	struct sfxge_evq *evq;  in sfxge_ev_rxq_flush_failed()  local
244 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_rxq_flush_failed()
245 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_rxq_flush_failed()
247 	sc = evq->sc;  in sfxge_ev_rxq_flush_failed()
254 	evq = sc->evq[index];  in sfxge_ev_rxq_flush_failed()
257 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,  in sfxge_ev_rxq_flush_failed()
259 	efx_ev_qpost(evq->common, magic);  in sfxge_ev_rxq_flush_failed()
265 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)  in sfxge_get_txq_by_label()  argument
269 	KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) :  in sfxge_get_txq_by_label()
270 		((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||  in sfxge_get_txq_by_label()
274 	index = (evq->index == 0) ?  in sfxge_get_txq_by_label()
275 		label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc));  in sfxge_get_txq_by_label()
276 	return (evq->sc->txq[index]);  in sfxge_get_txq_by_label()
282 	struct sfxge_evq *evq;  in sfxge_ev_tx()  local
287 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_tx()
288 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_tx()
290 	txq = sfxge_get_txq_by_label(evq, label);  in sfxge_ev_tx()
293 	KASSERT(evq->index == txq->evq_index,  in sfxge_ev_tx()
305 	evq->tx_done++;  in sfxge_ev_tx()
308 	    evq->txqs != &(txq->next)) {  in sfxge_ev_tx()
309 		*(evq->txqs) = txq;  in sfxge_ev_tx()
310 		evq->txqs = &(txq->next);  in sfxge_ev_tx()
314 		sfxge_tx_qcomplete(txq, evq);  in sfxge_ev_tx()
317 	return (evq->tx_done >= SFXGE_EV_BATCH);  in sfxge_ev_tx()
323 	struct sfxge_evq *evq;  in sfxge_ev_txq_flush_done()  local
328 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_txq_flush_done()
329 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_txq_flush_done()
331 	sc = evq->sc;  in sfxge_ev_txq_flush_done()
338 	if (txq->evq_index == evq->index) {  in sfxge_ev_txq_flush_done()
344 	evq = sc->evq[txq->evq_index];  in sfxge_ev_txq_flush_done()
347 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,  in sfxge_ev_txq_flush_done()
349 	efx_ev_qpost(evq->common, magic);  in sfxge_ev_txq_flush_done()
357 	struct sfxge_evq *evq;  in sfxge_ev_software()  local
360 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_software()
361 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_software()
368 		sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));  in sfxge_ev_software()
372 		sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));  in sfxge_ev_software()
376 		sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));  in sfxge_ev_software()
380 		struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);  in sfxge_ev_software()
383 		KASSERT(evq->index == txq->evq_index,  in sfxge_ev_software()
444 sfxge_evq_stat_update(struct sfxge_evq *evq)  in sfxge_evq_stat_update()  argument
448 	SFXGE_EVQ_LOCK(evq);  in sfxge_evq_stat_update()
450 	if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED))  in sfxge_evq_stat_update()
454 	if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz)  in sfxge_evq_stat_update()
457 	evq->stats_update_time = now;  in sfxge_evq_stat_update()
458 	efx_ev_qstats_update(evq->common, evq->stats);  in sfxge_evq_stat_update()
461 	SFXGE_EVQ_UNLOCK(evq);  in sfxge_evq_stat_update()
467 	struct sfxge_evq *evq = arg1;  in sfxge_evq_stat_handler()  local
468 	struct sfxge_softc *sc = evq->sc;  in sfxge_evq_stat_handler()
473 	sfxge_evq_stat_update(evq);  in sfxge_evq_stat_handler()
477 	return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id])));  in sfxge_evq_stat_handler()
481 sfxge_evq_stat_init(struct sfxge_evq *evq)  in sfxge_evq_stat_init()  argument
483 	struct sfxge_softc *sc = evq->sc;  in sfxge_evq_stat_init()
489 	snprintf(name, sizeof(name), "%u", evq->index);  in sfxge_evq_stat_init()
499 		    CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id,  in sfxge_evq_stat_init()
509 	struct sfxge_evq *evq;  in sfxge_ev_stat_update()  local
526 		evq = sc->evq[index];  in sfxge_ev_stat_update()
527 		sfxge_evq_stat_update(evq);  in sfxge_ev_stat_update()
529 			sc->ev_stats[id] += evq->stats[id];  in sfxge_ev_stat_update()
570 	struct sfxge_evq *evq;  in sfxge_ev_qmoderate()  local
573 	evq = sc->evq[idx];  in sfxge_ev_qmoderate()
574 	eep = evq->common;  in sfxge_ev_qmoderate()
576 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,  in sfxge_ev_qmoderate()
626 	struct sfxge_evq *evq;  in sfxge_ev_initialized()  local
628 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_initialized()
629 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_initialized()
632 	KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||  in sfxge_ev_initialized()
633 		evq->init_state == SFXGE_EVQ_STARTED,  in sfxge_ev_initialized()
636 	evq->init_state = SFXGE_EVQ_STARTED;  in sfxge_ev_initialized()
644 	struct sfxge_evq *evq;  in sfxge_ev_link_change()  local
647 	evq = (struct sfxge_evq *)arg;  in sfxge_ev_link_change()
648 	SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);  in sfxge_ev_link_change()
650 	sc = evq->sc;  in sfxge_ev_link_change()
673 sfxge_ev_qpoll(struct sfxge_evq *evq)  in sfxge_ev_qpoll()  argument
677 	SFXGE_EVQ_LOCK(evq);  in sfxge_ev_qpoll()
679 	if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&  in sfxge_ev_qpoll()
680 			    evq->init_state != SFXGE_EVQ_STARTED)) {  in sfxge_ev_qpoll()
686 	bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,  in sfxge_ev_qpoll()
689 	KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));  in sfxge_ev_qpoll()
690 	KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));  in sfxge_ev_qpoll()
691 	KASSERT(evq->txq == NULL, ("evq->txq != NULL"));  in sfxge_ev_qpoll()
692 	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));  in sfxge_ev_qpoll()
695 	efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);  in sfxge_ev_qpoll()
697 	evq->rx_done = 0;  in sfxge_ev_qpoll()
698 	evq->tx_done = 0;  in sfxge_ev_qpoll()
701 	sfxge_ev_qcomplete(evq, B_TRUE);  in sfxge_ev_qpoll()
704 	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)  in sfxge_ev_qpoll()
707 	SFXGE_EVQ_UNLOCK(evq);  in sfxge_ev_qpoll()
712 	SFXGE_EVQ_UNLOCK(evq);  in sfxge_ev_qpoll()
719 	struct sfxge_evq *evq;  in sfxge_ev_qstop()  local
721 	evq = sc->evq[index];  in sfxge_ev_qstop()
723 	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,  in sfxge_ev_qstop()
726 	SFXGE_EVQ_LOCK(evq);  in sfxge_ev_qstop()
727 	evq->init_state = SFXGE_EVQ_INITIALIZED;  in sfxge_ev_qstop()
728 	evq->read_ptr = 0;  in sfxge_ev_qstop()
729 	evq->exception = B_FALSE;  in sfxge_ev_qstop()
733 	efx_ev_qstats_update(evq->common, evq->stats);  in sfxge_ev_qstop()
736 	efx_ev_qdestroy(evq->common);  in sfxge_ev_qstop()
737 	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,  in sfxge_ev_qstop()
738 	    EFX_EVQ_NBUFS(evq->entries));  in sfxge_ev_qstop()
739 	SFXGE_EVQ_UNLOCK(evq);  in sfxge_ev_qstop()
745 	struct sfxge_evq *evq;  in sfxge_ev_qstart()  local
750 	evq = sc->evq[index];  in sfxge_ev_qstart()
751 	esmp = &evq->mem;  in sfxge_ev_qstart()
753 	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,  in sfxge_ev_qstart()
757 	(void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));  in sfxge_ev_qstart()
760 	if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,  in sfxge_ev_qstart()
761 	    EFX_EVQ_NBUFS(evq->entries))) != 0)  in sfxge_ev_qstart()
765 	if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,  in sfxge_ev_qstart()
766 	    evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,  in sfxge_ev_qstart()
767 	    &evq->common)) != 0)  in sfxge_ev_qstart()
770 	SFXGE_EVQ_LOCK(evq);  in sfxge_ev_qstart()
773 	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)  in sfxge_ev_qstart()
776 	evq->init_state = SFXGE_EVQ_STARTING;  in sfxge_ev_qstart()
778 	SFXGE_EVQ_UNLOCK(evq);  in sfxge_ev_qstart()
787 		if (evq->init_state == SFXGE_EVQ_STARTED)  in sfxge_ev_qstart()
799 	SFXGE_EVQ_LOCK(evq);  in sfxge_ev_qstart()
800 	evq->init_state = SFXGE_EVQ_INITIALIZED;  in sfxge_ev_qstart()
802 	SFXGE_EVQ_UNLOCK(evq);  in sfxge_ev_qstart()
803 	efx_ev_qdestroy(evq->common);  in sfxge_ev_qstart()
805 	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,  in sfxge_ev_qstart()
806 	    EFX_EVQ_NBUFS(evq->entries));  in sfxge_ev_qstart()
871 	struct sfxge_evq *evq;  in sfxge_ev_qfini()  local
873 	evq = sc->evq[index];  in sfxge_ev_qfini()
875 	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,  in sfxge_ev_qfini()
877 	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));  in sfxge_ev_qfini()
879 	sfxge_dma_free(&evq->mem);  in sfxge_ev_qfini()
881 	sc->evq[index] = NULL;  in sfxge_ev_qfini()
883 	SFXGE_EVQ_LOCK_DESTROY(evq);  in sfxge_ev_qfini()
885 	free(evq, M_SFXGE);  in sfxge_ev_qfini()
891 	struct sfxge_evq *evq;  in sfxge_ev_qinit()  local
897 	evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);  in sfxge_ev_qinit()
898 	evq->sc = sc;  in sfxge_ev_qinit()
899 	evq->index = index;  in sfxge_ev_qinit()
900 	sc->evq[index] = evq;  in sfxge_ev_qinit()
901 	esmp = &evq->mem;  in sfxge_ev_qinit()
909 		evq->entries =  in sfxge_ev_qinit()
914 		evq->entries =  in sfxge_ev_qinit()
920 	evq->txqs = &evq->txq;  in sfxge_ev_qinit()
923 	if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)  in sfxge_ev_qinit()
927 	sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),  in sfxge_ev_qinit()
928 				 &evq->buf_base_id);  in sfxge_ev_qinit()
930 	SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);  in sfxge_ev_qinit()
932 	evq->init_state = SFXGE_EVQ_INITIALIZED;  in sfxge_ev_qinit()
935 	rc = sfxge_evq_stat_init(evq);  in sfxge_ev_qinit()
944 	evq->init_state = SFXGE_EVQ_UNINITIALIZED;  in sfxge_ev_qinit()
945 	SFXGE_EVQ_LOCK_DESTROY(evq);  in sfxge_ev_qinit()
947 	sc->evq[index] = NULL;  in sfxge_ev_qinit()
948 	free(evq, M_SFXGE);  in sfxge_ev_qinit()