xref: /freebsd/sys/dev/sfxge/common/efx_ev.c (revision 5095efd4f5cb6fd1cb1b897c9e150260ae012196)
1e948693eSPhilip Paeps /*-
2718cf2ccSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3718cf2ccSPedro F. Giffuni  *
4929c7febSAndrew Rybchenko  * Copyright (c) 2007-2016 Solarflare Communications Inc.
53c838a9fSAndrew Rybchenko  * All rights reserved.
6e948693eSPhilip Paeps  *
7e948693eSPhilip Paeps  * Redistribution and use in source and binary forms, with or without
83c838a9fSAndrew Rybchenko  * modification, are permitted provided that the following conditions are met:
9e948693eSPhilip Paeps  *
103c838a9fSAndrew Rybchenko  * 1. Redistributions of source code must retain the above copyright notice,
113c838a9fSAndrew Rybchenko  *    this list of conditions and the following disclaimer.
123c838a9fSAndrew Rybchenko  * 2. Redistributions in binary form must reproduce the above copyright notice,
133c838a9fSAndrew Rybchenko  *    this list of conditions and the following disclaimer in the documentation
143c838a9fSAndrew Rybchenko  *    and/or other materials provided with the distribution.
153c838a9fSAndrew Rybchenko  *
163c838a9fSAndrew Rybchenko  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
173c838a9fSAndrew Rybchenko  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
183c838a9fSAndrew Rybchenko  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
193c838a9fSAndrew Rybchenko  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
203c838a9fSAndrew Rybchenko  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
213c838a9fSAndrew Rybchenko  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
223c838a9fSAndrew Rybchenko  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
233c838a9fSAndrew Rybchenko  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
243c838a9fSAndrew Rybchenko  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
253c838a9fSAndrew Rybchenko  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
263c838a9fSAndrew Rybchenko  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
273c838a9fSAndrew Rybchenko  *
283c838a9fSAndrew Rybchenko  * The views and conclusions contained in the software and documentation are
293c838a9fSAndrew Rybchenko  * those of the authors and should not be interpreted as representing official
303c838a9fSAndrew Rybchenko  * policies, either expressed or implied, of the FreeBSD Project.
31e948693eSPhilip Paeps  */
32e948693eSPhilip Paeps 
335dee87d7SPhilip Paeps #include <sys/cdefs.h>
345dee87d7SPhilip Paeps __FBSDID("$FreeBSD$");
355dee87d7SPhilip Paeps 
36e948693eSPhilip Paeps #include "efx.h"
37e948693eSPhilip Paeps #include "efx_impl.h"
3847d4cf23SAndrew Rybchenko #if EFSYS_OPT_MON_MCDI
393c838a9fSAndrew Rybchenko #include "mcdi_mon.h"
4047d4cf23SAndrew Rybchenko #endif
41e948693eSPhilip Paeps 
42e948693eSPhilip Paeps #if EFSYS_OPT_QSTATS
43e948693eSPhilip Paeps #define	EFX_EV_QSTAT_INCR(_eep, _stat)					\
44e948693eSPhilip Paeps 	do {								\
45e948693eSPhilip Paeps 		(_eep)->ee_stat[_stat]++;				\
46e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
47e948693eSPhilip Paeps 	} while (B_FALSE)
48e948693eSPhilip Paeps #else
49e948693eSPhilip Paeps #define	EFX_EV_QSTAT_INCR(_eep, _stat)
50e948693eSPhilip Paeps #endif
51e948693eSPhilip Paeps 
523c838a9fSAndrew Rybchenko #define	EFX_EV_PRESENT(_qword)						\
533c838a9fSAndrew Rybchenko 	(EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&	\
543c838a9fSAndrew Rybchenko 	EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
553c838a9fSAndrew Rybchenko 
563c838a9fSAndrew Rybchenko 
573c838a9fSAndrew Rybchenko 
58e75412c9SAndrew Rybchenko #if EFSYS_OPT_SIENA
593c838a9fSAndrew Rybchenko 
60460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
610f981da7SAndrew Rybchenko siena_ev_init(
623c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp);
633c838a9fSAndrew Rybchenko 
643c838a9fSAndrew Rybchenko static			void
650f981da7SAndrew Rybchenko siena_ev_fini(
663c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp);
673c838a9fSAndrew Rybchenko 
68460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
690f981da7SAndrew Rybchenko siena_ev_qcreate(
703c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp,
713c838a9fSAndrew Rybchenko 	__in		unsigned int index,
723c838a9fSAndrew Rybchenko 	__in		efsys_mem_t *esmp,
7369953083SAndrew Rybchenko 	__in		size_t ndescs,
743c838a9fSAndrew Rybchenko 	__in		uint32_t id,
755c4c3d92SAndrew Rybchenko 	__in		uint32_t us,
76a3fe009aSAndrew Rybchenko 	__in		uint32_t flags,
773c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep);
783c838a9fSAndrew Rybchenko 
793c838a9fSAndrew Rybchenko static			void
800f981da7SAndrew Rybchenko siena_ev_qdestroy(
813c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep);
823c838a9fSAndrew Rybchenko 
83460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
840f981da7SAndrew Rybchenko siena_ev_qprime(
853c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
863c838a9fSAndrew Rybchenko 	__in		unsigned int count);
873c838a9fSAndrew Rybchenko 
883c838a9fSAndrew Rybchenko static			void
890f981da7SAndrew Rybchenko siena_ev_qpost(
903c838a9fSAndrew Rybchenko 	__in	efx_evq_t *eep,
913c838a9fSAndrew Rybchenko 	__in	uint16_t data);
923c838a9fSAndrew Rybchenko 
93460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
940f981da7SAndrew Rybchenko siena_ev_qmoderate(
953c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
963c838a9fSAndrew Rybchenko 	__in		unsigned int us);
973c838a9fSAndrew Rybchenko 
983c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
993c838a9fSAndrew Rybchenko static			void
1000f981da7SAndrew Rybchenko siena_ev_qstats_update(
1013c838a9fSAndrew Rybchenko 	__in				efx_evq_t *eep,
1023c838a9fSAndrew Rybchenko 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat);
1033c838a9fSAndrew Rybchenko 
1043c838a9fSAndrew Rybchenko #endif
1053c838a9fSAndrew Rybchenko 
106e75412c9SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1073c838a9fSAndrew Rybchenko 
1083c838a9fSAndrew Rybchenko #if EFSYS_OPT_SIENA
109ec831f7fSAndrew Rybchenko static const efx_ev_ops_t	__efx_ev_siena_ops = {
1100f981da7SAndrew Rybchenko 	siena_ev_init,				/* eevo_init */
1110f981da7SAndrew Rybchenko 	siena_ev_fini,				/* eevo_fini */
1120f981da7SAndrew Rybchenko 	siena_ev_qcreate,			/* eevo_qcreate */
1130f981da7SAndrew Rybchenko 	siena_ev_qdestroy,			/* eevo_qdestroy */
1140f981da7SAndrew Rybchenko 	siena_ev_qprime,			/* eevo_qprime */
1150f981da7SAndrew Rybchenko 	siena_ev_qpost,				/* eevo_qpost */
1160f981da7SAndrew Rybchenko 	siena_ev_qmoderate,			/* eevo_qmoderate */
1173c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
1180f981da7SAndrew Rybchenko 	siena_ev_qstats_update,			/* eevo_qstats_update */
1193c838a9fSAndrew Rybchenko #endif
1203c838a9fSAndrew Rybchenko };
1213c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1223c838a9fSAndrew Rybchenko 
123*5095efd4SAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
124ec831f7fSAndrew Rybchenko static const efx_ev_ops_t	__efx_ev_ef10_ops = {
125ad8a32cbSAndrew Rybchenko 	ef10_ev_init,				/* eevo_init */
126ad8a32cbSAndrew Rybchenko 	ef10_ev_fini,				/* eevo_fini */
127ad8a32cbSAndrew Rybchenko 	ef10_ev_qcreate,			/* eevo_qcreate */
128ad8a32cbSAndrew Rybchenko 	ef10_ev_qdestroy,			/* eevo_qdestroy */
129ad8a32cbSAndrew Rybchenko 	ef10_ev_qprime,				/* eevo_qprime */
130ad8a32cbSAndrew Rybchenko 	ef10_ev_qpost,				/* eevo_qpost */
131ad8a32cbSAndrew Rybchenko 	ef10_ev_qmoderate,			/* eevo_qmoderate */
1323c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
133ad8a32cbSAndrew Rybchenko 	ef10_ev_qstats_update,			/* eevo_qstats_update */
1343c838a9fSAndrew Rybchenko #endif
1353c838a9fSAndrew Rybchenko };
136*5095efd4SAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
1373c838a9fSAndrew Rybchenko 
1383c838a9fSAndrew Rybchenko 
139460cb568SAndrew Rybchenko 	__checkReturn	efx_rc_t
140e948693eSPhilip Paeps efx_ev_init(
141e948693eSPhilip Paeps 	__in		efx_nic_t *enp)
142e948693eSPhilip Paeps {
143ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop;
144460cb568SAndrew Rybchenko 	efx_rc_t rc;
145e948693eSPhilip Paeps 
146e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
147e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
148e948693eSPhilip Paeps 
149e948693eSPhilip Paeps 	if (enp->en_mod_flags & EFX_MOD_EV) {
150e948693eSPhilip Paeps 		rc = EINVAL;
151e948693eSPhilip Paeps 		goto fail1;
152e948693eSPhilip Paeps 	}
153e948693eSPhilip Paeps 
1543c838a9fSAndrew Rybchenko 	switch (enp->en_family) {
1553c838a9fSAndrew Rybchenko #if EFSYS_OPT_SIENA
1563c838a9fSAndrew Rybchenko 	case EFX_FAMILY_SIENA:
157ec831f7fSAndrew Rybchenko 		eevop = &__efx_ev_siena_ops;
1583c838a9fSAndrew Rybchenko 		break;
1593c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1603c838a9fSAndrew Rybchenko 
1613c838a9fSAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON
1623c838a9fSAndrew Rybchenko 	case EFX_FAMILY_HUNTINGTON:
163ec831f7fSAndrew Rybchenko 		eevop = &__efx_ev_ef10_ops;
1643c838a9fSAndrew Rybchenko 		break;
1653c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON */
1663c838a9fSAndrew Rybchenko 
167ad8a32cbSAndrew Rybchenko #if EFSYS_OPT_MEDFORD
168ad8a32cbSAndrew Rybchenko 	case EFX_FAMILY_MEDFORD:
169ec831f7fSAndrew Rybchenko 		eevop = &__efx_ev_ef10_ops;
170ad8a32cbSAndrew Rybchenko 		break;
171ad8a32cbSAndrew Rybchenko #endif /* EFSYS_OPT_MEDFORD */
172ad8a32cbSAndrew Rybchenko 
173*5095efd4SAndrew Rybchenko #if EFSYS_OPT_MEDFORD2
174*5095efd4SAndrew Rybchenko 	case EFX_FAMILY_MEDFORD2:
175*5095efd4SAndrew Rybchenko 		eevop = &__efx_ev_ef10_ops;
176*5095efd4SAndrew Rybchenko 		break;
177*5095efd4SAndrew Rybchenko #endif /* EFSYS_OPT_MEDFORD2 */
178*5095efd4SAndrew Rybchenko 
1793c838a9fSAndrew Rybchenko 	default:
1803c838a9fSAndrew Rybchenko 		EFSYS_ASSERT(0);
1813c838a9fSAndrew Rybchenko 		rc = ENOTSUP;
1823c838a9fSAndrew Rybchenko 		goto fail1;
1833c838a9fSAndrew Rybchenko 	}
1843c838a9fSAndrew Rybchenko 
185e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
186e948693eSPhilip Paeps 
1873c838a9fSAndrew Rybchenko 	if ((rc = eevop->eevo_init(enp)) != 0)
1883c838a9fSAndrew Rybchenko 		goto fail2;
1893c838a9fSAndrew Rybchenko 
1903c838a9fSAndrew Rybchenko 	enp->en_eevop = eevop;
1913c838a9fSAndrew Rybchenko 	enp->en_mod_flags |= EFX_MOD_EV;
1923c838a9fSAndrew Rybchenko 	return (0);
1933c838a9fSAndrew Rybchenko 
1943c838a9fSAndrew Rybchenko fail2:
1953c838a9fSAndrew Rybchenko 	EFSYS_PROBE(fail2);
1963c838a9fSAndrew Rybchenko 
1973c838a9fSAndrew Rybchenko fail1:
198460cb568SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1993c838a9fSAndrew Rybchenko 
2003c838a9fSAndrew Rybchenko 	enp->en_eevop = NULL;
2013c838a9fSAndrew Rybchenko 	enp->en_mod_flags &= ~EFX_MOD_EV;
2023c838a9fSAndrew Rybchenko 	return (rc);
2033c838a9fSAndrew Rybchenko }
2043c838a9fSAndrew Rybchenko 
2053c838a9fSAndrew Rybchenko 		void
2063c838a9fSAndrew Rybchenko efx_ev_fini(
2073c838a9fSAndrew Rybchenko 	__in	efx_nic_t *enp)
2083c838a9fSAndrew Rybchenko {
209ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
2103c838a9fSAndrew Rybchenko 
2113c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
2123c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
2133c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
2143c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
2153c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
2163c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
2173c838a9fSAndrew Rybchenko 
2183c838a9fSAndrew Rybchenko 	eevop->eevo_fini(enp);
2193c838a9fSAndrew Rybchenko 
2203c838a9fSAndrew Rybchenko 	enp->en_eevop = NULL;
2213c838a9fSAndrew Rybchenko 	enp->en_mod_flags &= ~EFX_MOD_EV;
2223c838a9fSAndrew Rybchenko }
2233c838a9fSAndrew Rybchenko 
2243c838a9fSAndrew Rybchenko 
225460cb568SAndrew Rybchenko 	__checkReturn	efx_rc_t
2263c838a9fSAndrew Rybchenko efx_ev_qcreate(
2273c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp,
2283c838a9fSAndrew Rybchenko 	__in		unsigned int index,
2293c838a9fSAndrew Rybchenko 	__in		efsys_mem_t *esmp,
23069953083SAndrew Rybchenko 	__in		size_t ndescs,
2313c838a9fSAndrew Rybchenko 	__in		uint32_t id,
2325c4c3d92SAndrew Rybchenko 	__in		uint32_t us,
233a3fe009aSAndrew Rybchenko 	__in		uint32_t flags,
2343c838a9fSAndrew Rybchenko 	__deref_out	efx_evq_t **eepp)
2353c838a9fSAndrew Rybchenko {
236ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
2373c838a9fSAndrew Rybchenko 	efx_evq_t *eep;
238460cb568SAndrew Rybchenko 	efx_rc_t rc;
2393c838a9fSAndrew Rybchenko 
2403c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
2413c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
2423c838a9fSAndrew Rybchenko 
243cc8d2b23SAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <,
244cc8d2b23SAndrew Rybchenko 	    enp->en_nic_cfg.enc_evq_limit);
2453c838a9fSAndrew Rybchenko 
24682d2a148SAndrew Rybchenko 	switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) {
24782d2a148SAndrew Rybchenko 	case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT:
24882d2a148SAndrew Rybchenko 		break;
24982d2a148SAndrew Rybchenko 	case EFX_EVQ_FLAGS_NOTIFY_DISABLED:
25082d2a148SAndrew Rybchenko 		if (us != 0) {
25182d2a148SAndrew Rybchenko 			rc = EINVAL;
25282d2a148SAndrew Rybchenko 			goto fail1;
25382d2a148SAndrew Rybchenko 		}
25482d2a148SAndrew Rybchenko 		break;
25582d2a148SAndrew Rybchenko 	default:
25682d2a148SAndrew Rybchenko 		rc = EINVAL;
25782d2a148SAndrew Rybchenko 		goto fail2;
25882d2a148SAndrew Rybchenko 	}
25982d2a148SAndrew Rybchenko 
2603c838a9fSAndrew Rybchenko 	/* Allocate an EVQ object */
2613c838a9fSAndrew Rybchenko 	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
2623c838a9fSAndrew Rybchenko 	if (eep == NULL) {
2633c838a9fSAndrew Rybchenko 		rc = ENOMEM;
26482d2a148SAndrew Rybchenko 		goto fail3;
2653c838a9fSAndrew Rybchenko 	}
2663c838a9fSAndrew Rybchenko 
2673c838a9fSAndrew Rybchenko 	eep->ee_magic = EFX_EVQ_MAGIC;
2683c838a9fSAndrew Rybchenko 	eep->ee_enp = enp;
2693c838a9fSAndrew Rybchenko 	eep->ee_index = index;
27069953083SAndrew Rybchenko 	eep->ee_mask = ndescs - 1;
27182d2a148SAndrew Rybchenko 	eep->ee_flags = flags;
2723c838a9fSAndrew Rybchenko 	eep->ee_esmp = esmp;
2733c838a9fSAndrew Rybchenko 
274bdb482bfSAndrew Rybchenko 	/*
275bdb482bfSAndrew Rybchenko 	 * Set outputs before the queue is created because interrupts may be
276bdb482bfSAndrew Rybchenko 	 * raised for events immediately after the queue is created, before the
277bdb482bfSAndrew Rybchenko 	 * function call below returns. See bug58606.
278bdb482bfSAndrew Rybchenko 	 *
279bdb482bfSAndrew Rybchenko 	 * The eepp pointer passed in by the client must therefore point to data
280bdb482bfSAndrew Rybchenko 	 * shared with the client's event processing context.
281bdb482bfSAndrew Rybchenko 	 */
2823c838a9fSAndrew Rybchenko 	enp->en_ev_qcount++;
2833c838a9fSAndrew Rybchenko 	*eepp = eep;
2843c838a9fSAndrew Rybchenko 
28569953083SAndrew Rybchenko 	if ((rc = eevop->eevo_qcreate(enp, index, esmp, ndescs, id, us, flags,
286a3fe009aSAndrew Rybchenko 	    eep)) != 0)
28782d2a148SAndrew Rybchenko 		goto fail4;
288bdb482bfSAndrew Rybchenko 
2893c838a9fSAndrew Rybchenko 	return (0);
2903c838a9fSAndrew Rybchenko 
29182d2a148SAndrew Rybchenko fail4:
29282d2a148SAndrew Rybchenko 	EFSYS_PROBE(fail4);
293bdb482bfSAndrew Rybchenko 
294bdb482bfSAndrew Rybchenko 	*eepp = NULL;
295bdb482bfSAndrew Rybchenko 	enp->en_ev_qcount--;
2963c838a9fSAndrew Rybchenko 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
29782d2a148SAndrew Rybchenko fail3:
29882d2a148SAndrew Rybchenko 	EFSYS_PROBE(fail3);
29982d2a148SAndrew Rybchenko fail2:
30082d2a148SAndrew Rybchenko 	EFSYS_PROBE(fail2);
3013c838a9fSAndrew Rybchenko fail1:
302460cb568SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
3033c838a9fSAndrew Rybchenko 	return (rc);
3043c838a9fSAndrew Rybchenko }
3053c838a9fSAndrew Rybchenko 
3063c838a9fSAndrew Rybchenko 		void
3073c838a9fSAndrew Rybchenko efx_ev_qdestroy(
3083c838a9fSAndrew Rybchenko 	__in	efx_evq_t *eep)
3093c838a9fSAndrew Rybchenko {
3103c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
311ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
3123c838a9fSAndrew Rybchenko 
3133c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
3143c838a9fSAndrew Rybchenko 
3153c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(enp->en_ev_qcount != 0);
3163c838a9fSAndrew Rybchenko 	--enp->en_ev_qcount;
3173c838a9fSAndrew Rybchenko 
3183c838a9fSAndrew Rybchenko 	eevop->eevo_qdestroy(eep);
3193c838a9fSAndrew Rybchenko 
3203c838a9fSAndrew Rybchenko 	/* Free the EVQ object */
3213c838a9fSAndrew Rybchenko 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
3223c838a9fSAndrew Rybchenko }
3233c838a9fSAndrew Rybchenko 
324460cb568SAndrew Rybchenko 	__checkReturn	efx_rc_t
3253c838a9fSAndrew Rybchenko efx_ev_qprime(
3263c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
3273c838a9fSAndrew Rybchenko 	__in		unsigned int count)
3283c838a9fSAndrew Rybchenko {
3293c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
330ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
331460cb568SAndrew Rybchenko 	efx_rc_t rc;
3323c838a9fSAndrew Rybchenko 
3333c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
3343c838a9fSAndrew Rybchenko 
3353c838a9fSAndrew Rybchenko 	if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
3363c838a9fSAndrew Rybchenko 		rc = EINVAL;
3373c838a9fSAndrew Rybchenko 		goto fail1;
3383c838a9fSAndrew Rybchenko 	}
3393c838a9fSAndrew Rybchenko 
3403c838a9fSAndrew Rybchenko 	if ((rc = eevop->eevo_qprime(eep, count)) != 0)
3413c838a9fSAndrew Rybchenko 		goto fail2;
3423c838a9fSAndrew Rybchenko 
3433c838a9fSAndrew Rybchenko 	return (0);
3443c838a9fSAndrew Rybchenko 
3453c838a9fSAndrew Rybchenko fail2:
3463c838a9fSAndrew Rybchenko 	EFSYS_PROBE(fail2);
3473c838a9fSAndrew Rybchenko fail1:
348460cb568SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
3493c838a9fSAndrew Rybchenko 	return (rc);
3503c838a9fSAndrew Rybchenko }
3513c838a9fSAndrew Rybchenko 
3523c838a9fSAndrew Rybchenko 	__checkReturn	boolean_t
3533c838a9fSAndrew Rybchenko efx_ev_qpending(
3543c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
3553c838a9fSAndrew Rybchenko 	__in		unsigned int count)
3563c838a9fSAndrew Rybchenko {
3573c838a9fSAndrew Rybchenko 	size_t offset;
3583c838a9fSAndrew Rybchenko 	efx_qword_t qword;
3593c838a9fSAndrew Rybchenko 
3603c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
3613c838a9fSAndrew Rybchenko 
3623c838a9fSAndrew Rybchenko 	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
3633c838a9fSAndrew Rybchenko 	EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
3643c838a9fSAndrew Rybchenko 
3653c838a9fSAndrew Rybchenko 	return (EFX_EV_PRESENT(qword));
3663c838a9fSAndrew Rybchenko }
3673c838a9fSAndrew Rybchenko 
3683c838a9fSAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH
3693c838a9fSAndrew Rybchenko 
3703c838a9fSAndrew Rybchenko 			void
3713c838a9fSAndrew Rybchenko efx_ev_qprefetch(
3723c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
3733c838a9fSAndrew Rybchenko 	__in		unsigned int count)
3743c838a9fSAndrew Rybchenko {
3753c838a9fSAndrew Rybchenko 	unsigned int offset;
3763c838a9fSAndrew Rybchenko 
3773c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
3783c838a9fSAndrew Rybchenko 
3793c838a9fSAndrew Rybchenko 	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
3803c838a9fSAndrew Rybchenko 	EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
3813c838a9fSAndrew Rybchenko }
3823c838a9fSAndrew Rybchenko 
3833c838a9fSAndrew Rybchenko #endif	/* EFSYS_OPT_EV_PREFETCH */
3843c838a9fSAndrew Rybchenko 
3850ca76fa9SAndrew Rybchenko #define	EFX_EV_BATCH	8
3860ca76fa9SAndrew Rybchenko 
3873c838a9fSAndrew Rybchenko 			void
3883c838a9fSAndrew Rybchenko efx_ev_qpoll(
3893c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
3903c838a9fSAndrew Rybchenko 	__inout		unsigned int *countp,
3913c838a9fSAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
3923c838a9fSAndrew Rybchenko 	__in_opt	void *arg)
3933c838a9fSAndrew Rybchenko {
3940ca76fa9SAndrew Rybchenko 	efx_qword_t ev[EFX_EV_BATCH];
3950ca76fa9SAndrew Rybchenko 	unsigned int batch;
3960ca76fa9SAndrew Rybchenko 	unsigned int total;
3970ca76fa9SAndrew Rybchenko 	unsigned int count;
3980ca76fa9SAndrew Rybchenko 	unsigned int index;
3990ca76fa9SAndrew Rybchenko 	size_t offset;
4003c838a9fSAndrew Rybchenko 
4010ca76fa9SAndrew Rybchenko 	/* Ensure events codes match for EF10 (Huntington/Medford) and Siena */
4023c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
4033c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
4043c838a9fSAndrew Rybchenko 
4053c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
4063c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
4073c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
4083c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
4093c838a9fSAndrew Rybchenko 	    FSE_AZ_EV_CODE_DRV_GEN_EV);
4103c838a9fSAndrew Rybchenko #if EFSYS_OPT_MCDI
4113c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
4123c838a9fSAndrew Rybchenko 	    FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
4133c838a9fSAndrew Rybchenko #endif
4140ca76fa9SAndrew Rybchenko 
4150ca76fa9SAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
4160ca76fa9SAndrew Rybchenko 	EFSYS_ASSERT(countp != NULL);
4170ca76fa9SAndrew Rybchenko 	EFSYS_ASSERT(eecp != NULL);
4180ca76fa9SAndrew Rybchenko 
4190ca76fa9SAndrew Rybchenko 	count = *countp;
4200ca76fa9SAndrew Rybchenko 	do {
4210ca76fa9SAndrew Rybchenko 		/* Read up until the end of the batch period */
4220ca76fa9SAndrew Rybchenko 		batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
4230ca76fa9SAndrew Rybchenko 		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
4240ca76fa9SAndrew Rybchenko 		for (total = 0; total < batch; ++total) {
4250ca76fa9SAndrew Rybchenko 			EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
4260ca76fa9SAndrew Rybchenko 
4270ca76fa9SAndrew Rybchenko 			if (!EFX_EV_PRESENT(ev[total]))
4280ca76fa9SAndrew Rybchenko 				break;
4290ca76fa9SAndrew Rybchenko 
4300ca76fa9SAndrew Rybchenko 			EFSYS_PROBE3(event, unsigned int, eep->ee_index,
4310ca76fa9SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
4320ca76fa9SAndrew Rybchenko 			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
4330ca76fa9SAndrew Rybchenko 
4340ca76fa9SAndrew Rybchenko 			offset += sizeof (efx_qword_t);
4350ca76fa9SAndrew Rybchenko 		}
4360ca76fa9SAndrew Rybchenko 
4370ca76fa9SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
4380ca76fa9SAndrew Rybchenko 		/*
4390ca76fa9SAndrew Rybchenko 		 * Prefetch the next batch when we get within PREFETCH_PERIOD
4400ca76fa9SAndrew Rybchenko 		 * of a completed batch. If the batch is smaller, then prefetch
4410ca76fa9SAndrew Rybchenko 		 * immediately.
4420ca76fa9SAndrew Rybchenko 		 */
4430ca76fa9SAndrew Rybchenko 		if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
4440ca76fa9SAndrew Rybchenko 			EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
4450ca76fa9SAndrew Rybchenko #endif	/* EFSYS_OPT_EV_PREFETCH */
4460ca76fa9SAndrew Rybchenko 
4470ca76fa9SAndrew Rybchenko 		/* Process the batch of events */
4480ca76fa9SAndrew Rybchenko 		for (index = 0; index < total; ++index) {
4490ca76fa9SAndrew Rybchenko 			boolean_t should_abort;
4500ca76fa9SAndrew Rybchenko 			uint32_t code;
4510ca76fa9SAndrew Rybchenko 
4520ca76fa9SAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH
4530ca76fa9SAndrew Rybchenko 			/* Prefetch if we've now reached the batch period */
4540ca76fa9SAndrew Rybchenko 			if (total == batch &&
4550ca76fa9SAndrew Rybchenko 			    index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
4560ca76fa9SAndrew Rybchenko 				offset = (count + batch) & eep->ee_mask;
4570ca76fa9SAndrew Rybchenko 				offset *= sizeof (efx_qword_t);
4580ca76fa9SAndrew Rybchenko 
4590ca76fa9SAndrew Rybchenko 				EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
4600ca76fa9SAndrew Rybchenko 			}
4610ca76fa9SAndrew Rybchenko #endif	/* EFSYS_OPT_EV_PREFETCH */
4620ca76fa9SAndrew Rybchenko 
4630ca76fa9SAndrew Rybchenko 			EFX_EV_QSTAT_INCR(eep, EV_ALL);
4640ca76fa9SAndrew Rybchenko 
4650ca76fa9SAndrew Rybchenko 			code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
4660ca76fa9SAndrew Rybchenko 			switch (code) {
4670ca76fa9SAndrew Rybchenko 			case FSE_AZ_EV_CODE_RX_EV:
4680ca76fa9SAndrew Rybchenko 				should_abort = eep->ee_rx(eep,
4690ca76fa9SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
4700ca76fa9SAndrew Rybchenko 				break;
4710ca76fa9SAndrew Rybchenko 			case FSE_AZ_EV_CODE_TX_EV:
4720ca76fa9SAndrew Rybchenko 				should_abort = eep->ee_tx(eep,
4730ca76fa9SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
4740ca76fa9SAndrew Rybchenko 				break;
4750ca76fa9SAndrew Rybchenko 			case FSE_AZ_EV_CODE_DRIVER_EV:
4760ca76fa9SAndrew Rybchenko 				should_abort = eep->ee_driver(eep,
4770ca76fa9SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
4780ca76fa9SAndrew Rybchenko 				break;
4790ca76fa9SAndrew Rybchenko 			case FSE_AZ_EV_CODE_DRV_GEN_EV:
4800ca76fa9SAndrew Rybchenko 				should_abort = eep->ee_drv_gen(eep,
4810ca76fa9SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
4820ca76fa9SAndrew Rybchenko 				break;
4830ca76fa9SAndrew Rybchenko #if EFSYS_OPT_MCDI
4840ca76fa9SAndrew Rybchenko 			case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
4850ca76fa9SAndrew Rybchenko 				should_abort = eep->ee_mcdi(eep,
4860ca76fa9SAndrew Rybchenko 				    &(ev[index]), eecp, arg);
4870ca76fa9SAndrew Rybchenko 				break;
4880ca76fa9SAndrew Rybchenko #endif
4890ca76fa9SAndrew Rybchenko 			case FSE_AZ_EV_CODE_GLOBAL_EV:
4900ca76fa9SAndrew Rybchenko 				if (eep->ee_global) {
4910ca76fa9SAndrew Rybchenko 					should_abort = eep->ee_global(eep,
4920ca76fa9SAndrew Rybchenko 					    &(ev[index]), eecp, arg);
4930ca76fa9SAndrew Rybchenko 					break;
4940ca76fa9SAndrew Rybchenko 				}
4950ca76fa9SAndrew Rybchenko 				/* else fallthrough */
4960ca76fa9SAndrew Rybchenko 			default:
4970ca76fa9SAndrew Rybchenko 				EFSYS_PROBE3(bad_event,
4980ca76fa9SAndrew Rybchenko 				    unsigned int, eep->ee_index,
4990ca76fa9SAndrew Rybchenko 				    uint32_t,
5000ca76fa9SAndrew Rybchenko 				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
5010ca76fa9SAndrew Rybchenko 				    uint32_t,
5020ca76fa9SAndrew Rybchenko 				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
5030ca76fa9SAndrew Rybchenko 
5040ca76fa9SAndrew Rybchenko 				EFSYS_ASSERT(eecp->eec_exception != NULL);
5050ca76fa9SAndrew Rybchenko 				(void) eecp->eec_exception(arg,
5060ca76fa9SAndrew Rybchenko 					EFX_EXCEPTION_EV_ERROR, code);
5070ca76fa9SAndrew Rybchenko 				should_abort = B_TRUE;
5080ca76fa9SAndrew Rybchenko 			}
5090ca76fa9SAndrew Rybchenko 			if (should_abort) {
5100ca76fa9SAndrew Rybchenko 				/* Ignore subsequent events */
5110ca76fa9SAndrew Rybchenko 				total = index + 1;
5120ca76fa9SAndrew Rybchenko 				break;
5130ca76fa9SAndrew Rybchenko 			}
5140ca76fa9SAndrew Rybchenko 		}
5150ca76fa9SAndrew Rybchenko 
5160ca76fa9SAndrew Rybchenko 		/*
5170ca76fa9SAndrew Rybchenko 		 * Now that the hardware has most likely moved onto dma'ing
5180ca76fa9SAndrew Rybchenko 		 * into the next cache line, clear the processed events. Take
5190ca76fa9SAndrew Rybchenko 		 * care to only clear out events that we've processed
5200ca76fa9SAndrew Rybchenko 		 */
5210ca76fa9SAndrew Rybchenko 		EFX_SET_QWORD(ev[0]);
5220ca76fa9SAndrew Rybchenko 		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
5230ca76fa9SAndrew Rybchenko 		for (index = 0; index < total; ++index) {
5240ca76fa9SAndrew Rybchenko 			EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
5250ca76fa9SAndrew Rybchenko 			offset += sizeof (efx_qword_t);
5260ca76fa9SAndrew Rybchenko 		}
5270ca76fa9SAndrew Rybchenko 
5280ca76fa9SAndrew Rybchenko 		count += total;
5290ca76fa9SAndrew Rybchenko 
5300ca76fa9SAndrew Rybchenko 	} while (total == batch);
5310ca76fa9SAndrew Rybchenko 
5320ca76fa9SAndrew Rybchenko 	*countp = count;
5333c838a9fSAndrew Rybchenko }
5343c838a9fSAndrew Rybchenko 
5353c838a9fSAndrew Rybchenko 			void
5363c838a9fSAndrew Rybchenko efx_ev_qpost(
5373c838a9fSAndrew Rybchenko 	__in	efx_evq_t *eep,
5383c838a9fSAndrew Rybchenko 	__in	uint16_t data)
5393c838a9fSAndrew Rybchenko {
5403c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
541ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
5423c838a9fSAndrew Rybchenko 
5433c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5443c838a9fSAndrew Rybchenko 
5453c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(eevop != NULL &&
5463c838a9fSAndrew Rybchenko 	    eevop->eevo_qpost != NULL);
5473c838a9fSAndrew Rybchenko 
5483c838a9fSAndrew Rybchenko 	eevop->eevo_qpost(eep, data);
5493c838a9fSAndrew Rybchenko }
5503c838a9fSAndrew Rybchenko 
551460cb568SAndrew Rybchenko 	__checkReturn	efx_rc_t
552b839ed60SAndrew Rybchenko efx_ev_usecs_to_ticks(
553b839ed60SAndrew Rybchenko 	__in		efx_nic_t *enp,
554b839ed60SAndrew Rybchenko 	__in		unsigned int us,
555b839ed60SAndrew Rybchenko 	__out		unsigned int *ticksp)
556b839ed60SAndrew Rybchenko {
557b839ed60SAndrew Rybchenko 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
558b839ed60SAndrew Rybchenko 	unsigned int ticks;
559b839ed60SAndrew Rybchenko 
560b839ed60SAndrew Rybchenko 	/* Convert microseconds to a timer tick count */
561b839ed60SAndrew Rybchenko 	if (us == 0)
562b839ed60SAndrew Rybchenko 		ticks = 0;
563b839ed60SAndrew Rybchenko 	else if (us * 1000 < encp->enc_evq_timer_quantum_ns)
564b839ed60SAndrew Rybchenko 		ticks = 1;	/* Never round down to zero */
565b839ed60SAndrew Rybchenko 	else
566b839ed60SAndrew Rybchenko 		ticks = us * 1000 / encp->enc_evq_timer_quantum_ns;
567b839ed60SAndrew Rybchenko 
568b839ed60SAndrew Rybchenko 	*ticksp = ticks;
569b839ed60SAndrew Rybchenko 	return (0);
570b839ed60SAndrew Rybchenko }
571b839ed60SAndrew Rybchenko 
572b839ed60SAndrew Rybchenko 	__checkReturn	efx_rc_t
5733c838a9fSAndrew Rybchenko efx_ev_qmoderate(
5743c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
5753c838a9fSAndrew Rybchenko 	__in		unsigned int us)
5763c838a9fSAndrew Rybchenko {
5773c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
578ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
579460cb568SAndrew Rybchenko 	efx_rc_t rc;
5803c838a9fSAndrew Rybchenko 
5813c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
5823c838a9fSAndrew Rybchenko 
58382d2a148SAndrew Rybchenko 	if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
58482d2a148SAndrew Rybchenko 	    EFX_EVQ_FLAGS_NOTIFY_DISABLED) {
58582d2a148SAndrew Rybchenko 		rc = EINVAL;
5863c838a9fSAndrew Rybchenko 		goto fail1;
58782d2a148SAndrew Rybchenko 	}
58882d2a148SAndrew Rybchenko 
58982d2a148SAndrew Rybchenko 	if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
59082d2a148SAndrew Rybchenko 		goto fail2;
5913c838a9fSAndrew Rybchenko 
5923c838a9fSAndrew Rybchenko 	return (0);
5933c838a9fSAndrew Rybchenko 
59482d2a148SAndrew Rybchenko fail2:
59582d2a148SAndrew Rybchenko 	EFSYS_PROBE(fail2);
5963c838a9fSAndrew Rybchenko fail1:
597460cb568SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
5983c838a9fSAndrew Rybchenko 	return (rc);
5993c838a9fSAndrew Rybchenko }
6003c838a9fSAndrew Rybchenko 
6013c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
6023c838a9fSAndrew Rybchenko 					void
6033c838a9fSAndrew Rybchenko efx_ev_qstats_update(
6043c838a9fSAndrew Rybchenko 	__in				efx_evq_t *eep,
6053c838a9fSAndrew Rybchenko 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
6063c838a9fSAndrew Rybchenko 
6073c838a9fSAndrew Rybchenko {	efx_nic_t *enp = eep->ee_enp;
608ec831f7fSAndrew Rybchenko 	const efx_ev_ops_t *eevop = enp->en_eevop;
6093c838a9fSAndrew Rybchenko 
6103c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
6113c838a9fSAndrew Rybchenko 
6123c838a9fSAndrew Rybchenko 	eevop->eevo_qstats_update(eep, stat);
6133c838a9fSAndrew Rybchenko }
6143c838a9fSAndrew Rybchenko 
6153c838a9fSAndrew Rybchenko #endif	/* EFSYS_OPT_QSTATS */
6163c838a9fSAndrew Rybchenko 
617e75412c9SAndrew Rybchenko #if EFSYS_OPT_SIENA
6183c838a9fSAndrew Rybchenko 
619460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
6200f981da7SAndrew Rybchenko siena_ev_init(
6213c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp)
6223c838a9fSAndrew Rybchenko {
6233c838a9fSAndrew Rybchenko 	efx_oword_t oword;
6243c838a9fSAndrew Rybchenko 
625e948693eSPhilip Paeps 	/*
626e948693eSPhilip Paeps 	 * Program the event queue for receive and transmit queue
627e948693eSPhilip Paeps 	 * flush events.
628e948693eSPhilip Paeps 	 */
629e948693eSPhilip Paeps 	EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
630e948693eSPhilip Paeps 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
631e948693eSPhilip Paeps 	EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
632e948693eSPhilip Paeps 
633e948693eSPhilip Paeps 	return (0);
634e948693eSPhilip Paeps 
635e948693eSPhilip Paeps }
636e948693eSPhilip Paeps 
637e948693eSPhilip Paeps static  __checkReturn   boolean_t
6380f981da7SAndrew Rybchenko siena_ev_rx_not_ok(
639e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
640e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
641e948693eSPhilip Paeps 	__in		uint32_t label,
642e948693eSPhilip Paeps 	__in		uint32_t id,
643e948693eSPhilip Paeps 	__inout		uint16_t *flagsp)
644e948693eSPhilip Paeps {
645e948693eSPhilip Paeps 	boolean_t ignore = B_FALSE;
646e948693eSPhilip Paeps 
647e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
648e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
649e948693eSPhilip Paeps 		EFSYS_PROBE(tobe_disc);
6500ff23789SAndrew Rybchenko 		/*
6510ff23789SAndrew Rybchenko 		 * Assume this is a unicast address mismatch, unless below
652e948693eSPhilip Paeps 		 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
653e948693eSPhilip Paeps 		 * EV_RX_PAUSE_FRM_ERR is set.
654e948693eSPhilip Paeps 		 */
655e948693eSPhilip Paeps 		(*flagsp) |= EFX_ADDR_MISMATCH;
656e948693eSPhilip Paeps 	}
657e948693eSPhilip Paeps 
658e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
659e948693eSPhilip Paeps 		EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
660e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
661e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
662e948693eSPhilip Paeps 
6631b02e3c0SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
6640ff23789SAndrew Rybchenko 		/*
6650ff23789SAndrew Rybchenko 		 * Lookout for payload queue ran dry errors and ignore them.
666e948693eSPhilip Paeps 		 *
667e948693eSPhilip Paeps 		 * Sadly for the header/data split cases, the descriptor
668e948693eSPhilip Paeps 		 * pointer in this event refers to the header queue and
669e948693eSPhilip Paeps 		 * therefore cannot be easily detected as duplicate.
670e948693eSPhilip Paeps 		 * So we drop these and rely on the receive processing seeing
671e948693eSPhilip Paeps 		 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
672e948693eSPhilip Paeps 		 * the partially received packet.
673e948693eSPhilip Paeps 		 */
674e948693eSPhilip Paeps 		if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
675e948693eSPhilip Paeps 		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
676e948693eSPhilip Paeps 		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
677e948693eSPhilip Paeps 			ignore = B_TRUE;
6781b02e3c0SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
679e948693eSPhilip Paeps 	}
680e948693eSPhilip Paeps 
681e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
682e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
683e948693eSPhilip Paeps 		EFSYS_PROBE(crc_err);
684e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_ADDR_MISMATCH;
685e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
686e948693eSPhilip Paeps 	}
687e948693eSPhilip Paeps 
688e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
689e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
690e948693eSPhilip Paeps 		EFSYS_PROBE(pause_frm_err);
691e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_ADDR_MISMATCH;
692e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
693e948693eSPhilip Paeps 	}
694e948693eSPhilip Paeps 
695e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
696e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
697e948693eSPhilip Paeps 		EFSYS_PROBE(owner_id_err);
698e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
699e948693eSPhilip Paeps 	}
700e948693eSPhilip Paeps 
701e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
702e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
703e948693eSPhilip Paeps 		EFSYS_PROBE(ipv4_err);
704e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_CKSUM_IPV4;
705e948693eSPhilip Paeps 	}
706e948693eSPhilip Paeps 
707e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
708e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
709e948693eSPhilip Paeps 		EFSYS_PROBE(udp_chk_err);
710e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_CKSUM_TCPUDP;
711e948693eSPhilip Paeps 	}
712e948693eSPhilip Paeps 
713e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
714e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
715e948693eSPhilip Paeps 
716e948693eSPhilip Paeps 		/*
717e948693eSPhilip Paeps 		 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
718e948693eSPhilip Paeps 		 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
719e948693eSPhilip Paeps 		 * condition.
720e948693eSPhilip Paeps 		 */
721e948693eSPhilip Paeps 		(*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
722e948693eSPhilip Paeps 	}
723e948693eSPhilip Paeps 
724e948693eSPhilip Paeps 	return (ignore);
725e948693eSPhilip Paeps }
726e948693eSPhilip Paeps 
727e948693eSPhilip Paeps static	__checkReturn	boolean_t
7280f981da7SAndrew Rybchenko siena_ev_rx(
729e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
730e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
731e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
732e948693eSPhilip Paeps 	__in_opt	void *arg)
733e948693eSPhilip Paeps {
734e948693eSPhilip Paeps 	uint32_t id;
735e948693eSPhilip Paeps 	uint32_t size;
736e948693eSPhilip Paeps 	uint32_t label;
737e948693eSPhilip Paeps 	boolean_t ok;
7381b02e3c0SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
739e948693eSPhilip Paeps 	boolean_t sop;
740e948693eSPhilip Paeps 	boolean_t jumbo_cont;
7411b02e3c0SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
742e948693eSPhilip Paeps 	uint32_t hdr_type;
743e948693eSPhilip Paeps 	boolean_t is_v6;
744e948693eSPhilip Paeps 	uint16_t flags;
745e948693eSPhilip Paeps 	boolean_t ignore;
746e948693eSPhilip Paeps 	boolean_t should_abort;
747e948693eSPhilip Paeps 
748e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_RX);
749e948693eSPhilip Paeps 
750e948693eSPhilip Paeps 	/* Basic packet information */
751e948693eSPhilip Paeps 	id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
752e948693eSPhilip Paeps 	size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
753e948693eSPhilip Paeps 	label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
754e948693eSPhilip Paeps 	ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
755e948693eSPhilip Paeps 
7561b02e3c0SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
757e948693eSPhilip Paeps 	sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
758e948693eSPhilip Paeps 	jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
7591b02e3c0SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
760e948693eSPhilip Paeps 
761e948693eSPhilip Paeps 	hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
762e948693eSPhilip Paeps 
763a4983a11SAndrew Rybchenko 	is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
764e948693eSPhilip Paeps 
765e948693eSPhilip Paeps 	/*
766e948693eSPhilip Paeps 	 * If packet is marked as OK and packet type is TCP/IP or
767e948693eSPhilip Paeps 	 * UDP/IP or other IP, then we can rely on the hardware checksums.
768e948693eSPhilip Paeps 	 */
769e948693eSPhilip Paeps 	switch (hdr_type) {
770e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
771e948693eSPhilip Paeps 		flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
772e948693eSPhilip Paeps 		if (is_v6) {
773e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
774e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV6;
775e948693eSPhilip Paeps 		} else {
776e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
777e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
778e948693eSPhilip Paeps 		}
779e948693eSPhilip Paeps 		break;
780e948693eSPhilip Paeps 
781e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
782e948693eSPhilip Paeps 		flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
783e948693eSPhilip Paeps 		if (is_v6) {
784e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
785e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV6;
786e948693eSPhilip Paeps 		} else {
787e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
788e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
789e948693eSPhilip Paeps 		}
790e948693eSPhilip Paeps 		break;
791e948693eSPhilip Paeps 
792e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
793e948693eSPhilip Paeps 		if (is_v6) {
794e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
795e948693eSPhilip Paeps 			flags = EFX_PKT_IPV6;
796e948693eSPhilip Paeps 		} else {
797e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
798e948693eSPhilip Paeps 			flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
799e948693eSPhilip Paeps 		}
800e948693eSPhilip Paeps 		break;
801e948693eSPhilip Paeps 
802e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
803e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
804e948693eSPhilip Paeps 		flags = 0;
805e948693eSPhilip Paeps 		break;
806e948693eSPhilip Paeps 
807e948693eSPhilip Paeps 	default:
808e948693eSPhilip Paeps 		EFSYS_ASSERT(B_FALSE);
809e948693eSPhilip Paeps 		flags = 0;
810e948693eSPhilip Paeps 		break;
811e948693eSPhilip Paeps 	}
812e948693eSPhilip Paeps 
8131b02e3c0SAndrew Rybchenko #if EFSYS_OPT_RX_SCATTER
814e948693eSPhilip Paeps 	/* Report scatter and header/lookahead split buffer flags */
815e948693eSPhilip Paeps 	if (sop)
816e948693eSPhilip Paeps 		flags |= EFX_PKT_START;
817e948693eSPhilip Paeps 	if (jumbo_cont)
818e948693eSPhilip Paeps 		flags |= EFX_PKT_CONT;
8191b02e3c0SAndrew Rybchenko #endif	/* EFSYS_OPT_RX_SCATTER */
820e948693eSPhilip Paeps 
821e948693eSPhilip Paeps 	/* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
822e948693eSPhilip Paeps 	if (!ok) {
8230f981da7SAndrew Rybchenko 		ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
824e948693eSPhilip Paeps 		if (ignore) {
825e948693eSPhilip Paeps 			EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
826e948693eSPhilip Paeps 			    uint32_t, size, uint16_t, flags);
827e948693eSPhilip Paeps 
828e948693eSPhilip Paeps 			return (B_FALSE);
829e948693eSPhilip Paeps 		}
830e948693eSPhilip Paeps 	}
831e948693eSPhilip Paeps 
832e948693eSPhilip Paeps 	/* If we're not discarding the packet then it is ok */
833e948693eSPhilip Paeps 	if (~flags & EFX_DISCARD)
834e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
835e948693eSPhilip Paeps 
836e948693eSPhilip Paeps 	/* Detect multicast packets that didn't match the filter */
837e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
838e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
839e948693eSPhilip Paeps 
840e948693eSPhilip Paeps 		if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
841e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
842e948693eSPhilip Paeps 		} else {
843e948693eSPhilip Paeps 			EFSYS_PROBE(mcast_mismatch);
844e948693eSPhilip Paeps 			flags |= EFX_ADDR_MISMATCH;
845e948693eSPhilip Paeps 		}
846e948693eSPhilip Paeps 	} else {
847e948693eSPhilip Paeps 		flags |= EFX_PKT_UNICAST;
848e948693eSPhilip Paeps 	}
849e948693eSPhilip Paeps 
850e948693eSPhilip Paeps 	/*
851e948693eSPhilip Paeps 	 * The packet parser in Siena can abort parsing packets under
852e948693eSPhilip Paeps 	 * certain error conditions, setting the PKT_NOT_PARSED bit
853e948693eSPhilip Paeps 	 * (which clears PKT_OK). If this is set, then don't trust
854e948693eSPhilip Paeps 	 * the PKT_TYPE field.
855e948693eSPhilip Paeps 	 */
856a4983a11SAndrew Rybchenko 	if (!ok) {
857e948693eSPhilip Paeps 		uint32_t parse_err;
858e948693eSPhilip Paeps 
859e948693eSPhilip Paeps 		parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
860e948693eSPhilip Paeps 		if (parse_err != 0)
861e948693eSPhilip Paeps 			flags |= EFX_CHECK_VLAN;
862e948693eSPhilip Paeps 	}
863e948693eSPhilip Paeps 
864e948693eSPhilip Paeps 	if (~flags & EFX_CHECK_VLAN) {
865e948693eSPhilip Paeps 		uint32_t pkt_type;
866e948693eSPhilip Paeps 
867e948693eSPhilip Paeps 		pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
868e948693eSPhilip Paeps 		if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
869e948693eSPhilip Paeps 			flags |= EFX_PKT_VLAN_TAGGED;
870e948693eSPhilip Paeps 	}
871e948693eSPhilip Paeps 
872e948693eSPhilip Paeps 	EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
873e948693eSPhilip Paeps 	    uint32_t, size, uint16_t, flags);
874e948693eSPhilip Paeps 
875e948693eSPhilip Paeps 	EFSYS_ASSERT(eecp->eec_rx != NULL);
876e948693eSPhilip Paeps 	should_abort = eecp->eec_rx(arg, label, id, size, flags);
877e948693eSPhilip Paeps 
878e948693eSPhilip Paeps 	return (should_abort);
879e948693eSPhilip Paeps }
880e948693eSPhilip Paeps 
881e948693eSPhilip Paeps static	__checkReturn	boolean_t
8820f981da7SAndrew Rybchenko siena_ev_tx(
883e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
884e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
885e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
886e948693eSPhilip Paeps 	__in_opt	void *arg)
887e948693eSPhilip Paeps {
888e948693eSPhilip Paeps 	uint32_t id;
889e948693eSPhilip Paeps 	uint32_t label;
890e948693eSPhilip Paeps 	boolean_t should_abort;
891e948693eSPhilip Paeps 
892e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_TX);
893e948693eSPhilip Paeps 
894e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
895e948693eSPhilip Paeps 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
896e948693eSPhilip Paeps 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
897e948693eSPhilip Paeps 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
898e948693eSPhilip Paeps 
899e948693eSPhilip Paeps 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
900e948693eSPhilip Paeps 		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
901e948693eSPhilip Paeps 
902e948693eSPhilip Paeps 		EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
903e948693eSPhilip Paeps 
904e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_tx != NULL);
905e948693eSPhilip Paeps 		should_abort = eecp->eec_tx(arg, label, id);
906e948693eSPhilip Paeps 
907e948693eSPhilip Paeps 		return (should_abort);
908e948693eSPhilip Paeps 	}
909e948693eSPhilip Paeps 
910e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
911e948693eSPhilip Paeps 		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
912e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
913e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
914e948693eSPhilip Paeps 
915e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
916e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
917e948693eSPhilip Paeps 
918e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
919e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
920e948693eSPhilip Paeps 
921e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
922e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
923e948693eSPhilip Paeps 
924e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
925e948693eSPhilip Paeps 	return (B_FALSE);
926e948693eSPhilip Paeps }
927e948693eSPhilip Paeps 
928e948693eSPhilip Paeps static	__checkReturn	boolean_t
9290f981da7SAndrew Rybchenko siena_ev_global(
930e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
931e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
932e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
933e948693eSPhilip Paeps 	__in_opt	void *arg)
934e948693eSPhilip Paeps {
9350de035aaSAndrew Rybchenko 	_NOTE(ARGUNUSED(eqp, eecp, arg))
936e948693eSPhilip Paeps 
937e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
938e948693eSPhilip Paeps 
9390de035aaSAndrew Rybchenko 	return (B_FALSE);
940e948693eSPhilip Paeps }
941e948693eSPhilip Paeps 
942e948693eSPhilip Paeps static	__checkReturn	boolean_t
9430f981da7SAndrew Rybchenko siena_ev_driver(
944e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
945e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
946e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
947e948693eSPhilip Paeps 	__in_opt	void *arg)
948e948693eSPhilip Paeps {
949e948693eSPhilip Paeps 	boolean_t should_abort;
950e948693eSPhilip Paeps 
951e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
952e948693eSPhilip Paeps 	should_abort = B_FALSE;
953e948693eSPhilip Paeps 
954e948693eSPhilip Paeps 	switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
955e948693eSPhilip Paeps 	case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
956cf07c70dSGeorge V. Neville-Neil 		uint32_t txq_index;
957e948693eSPhilip Paeps 
958e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
959e948693eSPhilip Paeps 
960cf07c70dSGeorge V. Neville-Neil 		txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
961e948693eSPhilip Paeps 
962cf07c70dSGeorge V. Neville-Neil 		EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
963e948693eSPhilip Paeps 
964e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
965cf07c70dSGeorge V. Neville-Neil 		should_abort = eecp->eec_txq_flush_done(arg, txq_index);
966e948693eSPhilip Paeps 
967e948693eSPhilip Paeps 		break;
968e948693eSPhilip Paeps 	}
969e948693eSPhilip Paeps 	case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
970cf07c70dSGeorge V. Neville-Neil 		uint32_t rxq_index;
971e948693eSPhilip Paeps 		uint32_t failed;
972e948693eSPhilip Paeps 
973cf07c70dSGeorge V. Neville-Neil 		rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
974e948693eSPhilip Paeps 		failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
975e948693eSPhilip Paeps 
976e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
977e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
978e948693eSPhilip Paeps 
979e948693eSPhilip Paeps 		if (failed) {
980e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
981e948693eSPhilip Paeps 
982cf07c70dSGeorge V. Neville-Neil 			EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
983e948693eSPhilip Paeps 
9843c838a9fSAndrew Rybchenko 			should_abort = eecp->eec_rxq_flush_failed(arg,
9853c838a9fSAndrew Rybchenko 								    rxq_index);
986e948693eSPhilip Paeps 		} else {
987e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
988e948693eSPhilip Paeps 
989cf07c70dSGeorge V. Neville-Neil 			EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
990e948693eSPhilip Paeps 
991cf07c70dSGeorge V. Neville-Neil 			should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
992e948693eSPhilip Paeps 		}
993e948693eSPhilip Paeps 
994e948693eSPhilip Paeps 		break;
995e948693eSPhilip Paeps 	}
996e948693eSPhilip Paeps 	case FSE_AZ_EVQ_INIT_DONE_EV:
997e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_initialized != NULL);
998e948693eSPhilip Paeps 		should_abort = eecp->eec_initialized(arg);
999e948693eSPhilip Paeps 
1000e948693eSPhilip Paeps 		break;
1001e948693eSPhilip Paeps 
1002e948693eSPhilip Paeps 	case FSE_AZ_EVQ_NOT_EN_EV:
1003e948693eSPhilip Paeps 		EFSYS_PROBE(evq_not_en);
1004e948693eSPhilip Paeps 		break;
1005e948693eSPhilip Paeps 
1006e948693eSPhilip Paeps 	case FSE_AZ_SRM_UPD_DONE_EV: {
1007e948693eSPhilip Paeps 		uint32_t code;
1008e948693eSPhilip Paeps 
1009e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
1010e948693eSPhilip Paeps 
1011e948693eSPhilip Paeps 		code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1012e948693eSPhilip Paeps 
1013e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_sram != NULL);
1014e948693eSPhilip Paeps 		should_abort = eecp->eec_sram(arg, code);
1015e948693eSPhilip Paeps 
1016e948693eSPhilip Paeps 		break;
1017e948693eSPhilip Paeps 	}
1018e948693eSPhilip Paeps 	case FSE_AZ_WAKE_UP_EV: {
1019e948693eSPhilip Paeps 		uint32_t id;
1020e948693eSPhilip Paeps 
1021e948693eSPhilip Paeps 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1022e948693eSPhilip Paeps 
1023e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_wake_up != NULL);
1024e948693eSPhilip Paeps 		should_abort = eecp->eec_wake_up(arg, id);
1025e948693eSPhilip Paeps 
1026e948693eSPhilip Paeps 		break;
1027e948693eSPhilip Paeps 	}
1028e948693eSPhilip Paeps 	case FSE_AZ_TX_PKT_NON_TCP_UDP:
1029e948693eSPhilip Paeps 		EFSYS_PROBE(tx_pkt_non_tcp_udp);
1030e948693eSPhilip Paeps 		break;
1031e948693eSPhilip Paeps 
1032e948693eSPhilip Paeps 	case FSE_AZ_TIMER_EV: {
1033e948693eSPhilip Paeps 		uint32_t id;
1034e948693eSPhilip Paeps 
1035e948693eSPhilip Paeps 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1036e948693eSPhilip Paeps 
1037e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_timer != NULL);
1038e948693eSPhilip Paeps 		should_abort = eecp->eec_timer(arg, id);
1039e948693eSPhilip Paeps 
1040e948693eSPhilip Paeps 		break;
1041e948693eSPhilip Paeps 	}
1042e948693eSPhilip Paeps 	case FSE_AZ_RX_DSC_ERROR_EV:
1043e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
1044e948693eSPhilip Paeps 
1045e948693eSPhilip Paeps 		EFSYS_PROBE(rx_dsc_error);
1046e948693eSPhilip Paeps 
1047e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_exception != NULL);
1048e948693eSPhilip Paeps 		should_abort = eecp->eec_exception(arg,
1049e948693eSPhilip Paeps 			EFX_EXCEPTION_RX_DSC_ERROR, 0);
1050e948693eSPhilip Paeps 
1051e948693eSPhilip Paeps 		break;
1052e948693eSPhilip Paeps 
1053e948693eSPhilip Paeps 	case FSE_AZ_TX_DSC_ERROR_EV:
1054e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
1055e948693eSPhilip Paeps 
1056e948693eSPhilip Paeps 		EFSYS_PROBE(tx_dsc_error);
1057e948693eSPhilip Paeps 
1058e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_exception != NULL);
1059e948693eSPhilip Paeps 		should_abort = eecp->eec_exception(arg,
1060e948693eSPhilip Paeps 			EFX_EXCEPTION_TX_DSC_ERROR, 0);
1061e948693eSPhilip Paeps 
1062e948693eSPhilip Paeps 		break;
1063e948693eSPhilip Paeps 
1064e948693eSPhilip Paeps 	default:
1065e948693eSPhilip Paeps 		break;
1066e948693eSPhilip Paeps 	}
1067e948693eSPhilip Paeps 
1068e948693eSPhilip Paeps 	return (should_abort);
1069e948693eSPhilip Paeps }
1070e948693eSPhilip Paeps 
1071e948693eSPhilip Paeps static	__checkReturn	boolean_t
10720f981da7SAndrew Rybchenko siena_ev_drv_gen(
1073e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
1074e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
1075e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
1076e948693eSPhilip Paeps 	__in_opt	void *arg)
1077e948693eSPhilip Paeps {
1078e948693eSPhilip Paeps 	uint32_t data;
1079e948693eSPhilip Paeps 	boolean_t should_abort;
1080e948693eSPhilip Paeps 
1081e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
1082e948693eSPhilip Paeps 
1083e948693eSPhilip Paeps 	data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
1084e948693eSPhilip Paeps 	if (data >= ((uint32_t)1 << 16)) {
1085e948693eSPhilip Paeps 		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
1086e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
1087e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
1088e948693eSPhilip Paeps 		return (B_TRUE);
1089e948693eSPhilip Paeps 	}
1090e948693eSPhilip Paeps 
1091e948693eSPhilip Paeps 	EFSYS_ASSERT(eecp->eec_software != NULL);
1092e948693eSPhilip Paeps 	should_abort = eecp->eec_software(arg, (uint16_t)data);
1093e948693eSPhilip Paeps 
1094e948693eSPhilip Paeps 	return (should_abort);
1095e948693eSPhilip Paeps }
1096e948693eSPhilip Paeps 
1097e948693eSPhilip Paeps #if EFSYS_OPT_MCDI
1098e948693eSPhilip Paeps 
1099e948693eSPhilip Paeps static	__checkReturn	boolean_t
11000f981da7SAndrew Rybchenko siena_ev_mcdi(
1101e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
1102e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
1103e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
1104e948693eSPhilip Paeps 	__in_opt	void *arg)
1105e948693eSPhilip Paeps {
1106e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
110798a9ac91SAndrew Rybchenko 	unsigned int code;
1108e948693eSPhilip Paeps 	boolean_t should_abort = B_FALSE;
1109e948693eSPhilip Paeps 
1110e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
1111e948693eSPhilip Paeps 
1112e948693eSPhilip Paeps 	if (enp->en_family != EFX_FAMILY_SIENA)
1113e948693eSPhilip Paeps 		goto out;
1114e948693eSPhilip Paeps 
11156da0beb5SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_link_change != NULL);
11166da0beb5SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_exception != NULL);
11176da0beb5SAndrew Rybchenko #if EFSYS_OPT_MON_STATS
11186da0beb5SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_monitor != NULL);
11196da0beb5SAndrew Rybchenko #endif
11206da0beb5SAndrew Rybchenko 
1121e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
1122e948693eSPhilip Paeps 
1123e948693eSPhilip Paeps 	code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
1124e948693eSPhilip Paeps 	switch (code) {
1125e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_BADSSERT:
1126e948693eSPhilip Paeps 		efx_mcdi_ev_death(enp, EINTR);
1127e948693eSPhilip Paeps 		break;
1128e948693eSPhilip Paeps 
1129e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_CMDDONE:
1130e948693eSPhilip Paeps 		efx_mcdi_ev_cpl(enp,
1131d8574f4fSAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
1132d8574f4fSAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
1133d8574f4fSAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
1134e948693eSPhilip Paeps 		break;
1135e948693eSPhilip Paeps 
1136e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_LINKCHANGE: {
1137e948693eSPhilip Paeps 		efx_link_mode_t link_mode;
1138e948693eSPhilip Paeps 
1139e948693eSPhilip Paeps 		siena_phy_link_ev(enp, eqp, &link_mode);
1140e948693eSPhilip Paeps 		should_abort = eecp->eec_link_change(arg, link_mode);
1141e948693eSPhilip Paeps 		break;
1142e948693eSPhilip Paeps 	}
1143e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_SENSOREVT: {
1144e948693eSPhilip Paeps #if EFSYS_OPT_MON_STATS
1145e948693eSPhilip Paeps 		efx_mon_stat_t id;
1146e948693eSPhilip Paeps 		efx_mon_stat_value_t value;
1147460cb568SAndrew Rybchenko 		efx_rc_t rc;
1148e948693eSPhilip Paeps 
11493c838a9fSAndrew Rybchenko 		if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
1150e948693eSPhilip Paeps 			should_abort = eecp->eec_monitor(arg, id, value);
1151e948693eSPhilip Paeps 		else if (rc == ENOTSUP) {
1152e948693eSPhilip Paeps 			should_abort = eecp->eec_exception(arg,
1153e948693eSPhilip Paeps 				EFX_EXCEPTION_UNKNOWN_SENSOREVT,
1154e948693eSPhilip Paeps 				MCDI_EV_FIELD(eqp, DATA));
1155e948693eSPhilip Paeps 		} else
1156e948693eSPhilip Paeps 			EFSYS_ASSERT(rc == ENODEV);	/* Wrong port */
1157e948693eSPhilip Paeps #else
1158e948693eSPhilip Paeps 		should_abort = B_FALSE;
1159e948693eSPhilip Paeps #endif
1160e948693eSPhilip Paeps 		break;
1161e948693eSPhilip Paeps 	}
1162e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_SCHEDERR:
1163e948693eSPhilip Paeps 		/* Informational only */
1164e948693eSPhilip Paeps 		break;
1165e948693eSPhilip Paeps 
1166e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_REBOOT:
1167e948693eSPhilip Paeps 		efx_mcdi_ev_death(enp, EIO);
1168e948693eSPhilip Paeps 		break;
1169e948693eSPhilip Paeps 
1170e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_MAC_STATS_DMA:
1171e948693eSPhilip Paeps #if EFSYS_OPT_MAC_STATS
1172e948693eSPhilip Paeps 		if (eecp->eec_mac_stats != NULL) {
1173e948693eSPhilip Paeps 			eecp->eec_mac_stats(arg,
1174e948693eSPhilip Paeps 			    MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
1175e948693eSPhilip Paeps 		}
1176e948693eSPhilip Paeps #endif
1177e948693eSPhilip Paeps 		break;
1178e948693eSPhilip Paeps 
1179e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_FWALERT: {
1180e948693eSPhilip Paeps 		uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
1181e948693eSPhilip Paeps 
1182e948693eSPhilip Paeps 		if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
1183e948693eSPhilip Paeps 			should_abort = eecp->eec_exception(arg,
1184e948693eSPhilip Paeps 				EFX_EXCEPTION_FWALERT_SRAM,
1185e948693eSPhilip Paeps 				MCDI_EV_FIELD(eqp, FWALERT_DATA));
1186e948693eSPhilip Paeps 		else
1187e948693eSPhilip Paeps 			should_abort = eecp->eec_exception(arg,
1188e948693eSPhilip Paeps 				EFX_EXCEPTION_UNKNOWN_FWALERT,
1189e948693eSPhilip Paeps 				MCDI_EV_FIELD(eqp, DATA));
1190e948693eSPhilip Paeps 		break;
1191e948693eSPhilip Paeps 	}
1192e948693eSPhilip Paeps 
1193e948693eSPhilip Paeps 	default:
1194e948693eSPhilip Paeps 		EFSYS_PROBE1(mc_pcol_error, int, code);
1195e948693eSPhilip Paeps 		break;
1196e948693eSPhilip Paeps 	}
1197e948693eSPhilip Paeps 
1198e948693eSPhilip Paeps out:
1199e948693eSPhilip Paeps 	return (should_abort);
1200e948693eSPhilip Paeps }
1201e948693eSPhilip Paeps 
1202c071447aSAndrew Rybchenko #endif	/* EFSYS_OPT_MCDI */
1203e948693eSPhilip Paeps 
1204460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
12050f981da7SAndrew Rybchenko siena_ev_qprime(
1206e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
1207e948693eSPhilip Paeps 	__in		unsigned int count)
1208e948693eSPhilip Paeps {
1209e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
1210e948693eSPhilip Paeps 	uint32_t rptr;
1211e948693eSPhilip Paeps 	efx_dword_t dword;
1212e948693eSPhilip Paeps 
1213e948693eSPhilip Paeps 	rptr = count & eep->ee_mask;
1214e948693eSPhilip Paeps 
1215e948693eSPhilip Paeps 	EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
1216e948693eSPhilip Paeps 
1217e948693eSPhilip Paeps 	EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
1218e948693eSPhilip Paeps 			    &dword, B_FALSE);
1219e948693eSPhilip Paeps 
1220e948693eSPhilip Paeps 	return (0);
1221e948693eSPhilip Paeps }
1222e948693eSPhilip Paeps 
12233c838a9fSAndrew Rybchenko static		void
12240f981da7SAndrew Rybchenko siena_ev_qpost(
1225e948693eSPhilip Paeps 	__in	efx_evq_t *eep,
1226e948693eSPhilip Paeps 	__in	uint16_t data)
1227e948693eSPhilip Paeps {
1228e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
1229e948693eSPhilip Paeps 	efx_qword_t ev;
1230e948693eSPhilip Paeps 	efx_oword_t oword;
1231e948693eSPhilip Paeps 
1232e948693eSPhilip Paeps 	EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
1233e948693eSPhilip Paeps 	    FSF_AZ_EV_DATA_DW0, (uint32_t)data);
1234e948693eSPhilip Paeps 
1235e948693eSPhilip Paeps 	EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
1236e948693eSPhilip Paeps 	    EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
1237e948693eSPhilip Paeps 	    EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
1238e948693eSPhilip Paeps 
1239e948693eSPhilip Paeps 	EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
1240e948693eSPhilip Paeps }
1241e948693eSPhilip Paeps 
1242460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
12430f981da7SAndrew Rybchenko siena_ev_qmoderate(
1244e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
1245e948693eSPhilip Paeps 	__in		unsigned int us)
1246e948693eSPhilip Paeps {
1247e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
12488bf93a04SAndrew Rybchenko 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1249e948693eSPhilip Paeps 	unsigned int locked;
1250e948693eSPhilip Paeps 	efx_dword_t dword;
1251460cb568SAndrew Rybchenko 	efx_rc_t rc;
1252e948693eSPhilip Paeps 
1253af9078c3SAndrew Rybchenko 	if (us > encp->enc_evq_timer_max_us) {
1254e948693eSPhilip Paeps 		rc = EINVAL;
1255e948693eSPhilip Paeps 		goto fail1;
1256e948693eSPhilip Paeps 	}
1257e948693eSPhilip Paeps 
1258e948693eSPhilip Paeps 	/* If the value is zero then disable the timer */
1259e948693eSPhilip Paeps 	if (us == 0) {
1260e948693eSPhilip Paeps 		EFX_POPULATE_DWORD_2(dword,
1261e948693eSPhilip Paeps 		    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
1262e948693eSPhilip Paeps 		    FRF_CZ_TC_TIMER_VAL, 0);
1263e948693eSPhilip Paeps 	} else {
1264b839ed60SAndrew Rybchenko 		unsigned int ticks;
1265e948693eSPhilip Paeps 
1266b839ed60SAndrew Rybchenko 		if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0)
1267b839ed60SAndrew Rybchenko 			goto fail2;
1268e948693eSPhilip Paeps 
1269b839ed60SAndrew Rybchenko 		EFSYS_ASSERT(ticks > 0);
1270e948693eSPhilip Paeps 		EFX_POPULATE_DWORD_2(dword,
1271e948693eSPhilip Paeps 		    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
1272b839ed60SAndrew Rybchenko 		    FRF_CZ_TC_TIMER_VAL, ticks - 1);
1273e948693eSPhilip Paeps 	}
1274e948693eSPhilip Paeps 
1275e948693eSPhilip Paeps 	locked = (eep->ee_index == 0) ? 1 : 0;
1276e948693eSPhilip Paeps 
1277e948693eSPhilip Paeps 	EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
1278e948693eSPhilip Paeps 	    eep->ee_index, &dword, locked);
1279e948693eSPhilip Paeps 
1280e948693eSPhilip Paeps 	return (0);
1281e948693eSPhilip Paeps 
1282b839ed60SAndrew Rybchenko fail2:
1283b839ed60SAndrew Rybchenko 	EFSYS_PROBE(fail2);
1284e948693eSPhilip Paeps fail1:
1285460cb568SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1286e948693eSPhilip Paeps 
1287e948693eSPhilip Paeps 	return (rc);
1288e948693eSPhilip Paeps }
1289e948693eSPhilip Paeps 
1290460cb568SAndrew Rybchenko static	__checkReturn	efx_rc_t
12910f981da7SAndrew Rybchenko siena_ev_qcreate(
1292e948693eSPhilip Paeps 	__in		efx_nic_t *enp,
1293e948693eSPhilip Paeps 	__in		unsigned int index,
1294e948693eSPhilip Paeps 	__in		efsys_mem_t *esmp,
129569953083SAndrew Rybchenko 	__in		size_t ndescs,
1296e948693eSPhilip Paeps 	__in		uint32_t id,
12975c4c3d92SAndrew Rybchenko 	__in		uint32_t us,
1298a3fe009aSAndrew Rybchenko 	__in		uint32_t flags,
12993c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep)
1300e948693eSPhilip Paeps {
1301e948693eSPhilip Paeps 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1302e948693eSPhilip Paeps 	uint32_t size;
1303e948693eSPhilip Paeps 	efx_oword_t oword;
1304460cb568SAndrew Rybchenko 	efx_rc_t rc;
130582d2a148SAndrew Rybchenko 	boolean_t notify_mode;
1306e948693eSPhilip Paeps 
1307a92a2133SAndrew Rybchenko 	_NOTE(ARGUNUSED(esmp))
1308a92a2133SAndrew Rybchenko 
13093c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS));
13103c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS));
1311e948693eSPhilip Paeps 
131269953083SAndrew Rybchenko 	if (!ISP2(ndescs) ||
131369953083SAndrew Rybchenko 	    (ndescs < EFX_EVQ_MINNEVS) || (ndescs > EFX_EVQ_MAXNEVS)) {
1314e948693eSPhilip Paeps 		rc = EINVAL;
1315e948693eSPhilip Paeps 		goto fail1;
1316e948693eSPhilip Paeps 	}
1317e948693eSPhilip Paeps 	if (index >= encp->enc_evq_limit) {
1318e948693eSPhilip Paeps 		rc = EINVAL;
1319e948693eSPhilip Paeps 		goto fail2;
1320e948693eSPhilip Paeps 	}
1321e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCALE
1322e948693eSPhilip Paeps 	if (enp->en_intr.ei_type == EFX_INTR_LINE &&
1323e948693eSPhilip Paeps 	    index >= EFX_MAXRSS_LEGACY) {
1324e948693eSPhilip Paeps 		rc = EINVAL;
1325e948693eSPhilip Paeps 		goto fail3;
1326e948693eSPhilip Paeps 	}
1327e948693eSPhilip Paeps #endif
1328e948693eSPhilip Paeps 	for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
1329e948693eSPhilip Paeps 	    size++)
133069953083SAndrew Rybchenko 		if ((1 << size) == (int)(ndescs / EFX_EVQ_MINNEVS))
1331e948693eSPhilip Paeps 			break;
1332e948693eSPhilip Paeps 	if (id + (1 << size) >= encp->enc_buftbl_limit) {
1333e948693eSPhilip Paeps 		rc = EINVAL;
1334e948693eSPhilip Paeps 		goto fail4;
1335e948693eSPhilip Paeps 	}
1336e948693eSPhilip Paeps 
1337e948693eSPhilip Paeps 	/* Set up the handler table */
13380f981da7SAndrew Rybchenko 	eep->ee_rx	= siena_ev_rx;
13390f981da7SAndrew Rybchenko 	eep->ee_tx	= siena_ev_tx;
13400f981da7SAndrew Rybchenko 	eep->ee_driver	= siena_ev_driver;
13410f981da7SAndrew Rybchenko 	eep->ee_global	= siena_ev_global;
13420f981da7SAndrew Rybchenko 	eep->ee_drv_gen	= siena_ev_drv_gen;
1343e948693eSPhilip Paeps #if EFSYS_OPT_MCDI
13440f981da7SAndrew Rybchenko 	eep->ee_mcdi	= siena_ev_mcdi;
1345c071447aSAndrew Rybchenko #endif	/* EFSYS_OPT_MCDI */
1346e948693eSPhilip Paeps 
134782d2a148SAndrew Rybchenko 	notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) !=
134882d2a148SAndrew Rybchenko 	    EFX_EVQ_FLAGS_NOTIFY_INTERRUPT);
134982d2a148SAndrew Rybchenko 
1350e948693eSPhilip Paeps 	/* Set up the new event queue */
135182d2a148SAndrew Rybchenko 	EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1,
135282d2a148SAndrew Rybchenko 	    FRF_CZ_HOST_NOTIFY_MODE, notify_mode,
135382d2a148SAndrew Rybchenko 	    FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
13543c838a9fSAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
1355e948693eSPhilip Paeps 
1356e948693eSPhilip Paeps 	EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
1357e948693eSPhilip Paeps 	    FRF_AZ_EVQ_BUF_BASE_ID, id);
1358e948693eSPhilip Paeps 
13593c838a9fSAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
1360e948693eSPhilip Paeps 
13615c4c3d92SAndrew Rybchenko 	/* Set initial interrupt moderation */
13625c4c3d92SAndrew Rybchenko 	siena_ev_qmoderate(eep, us);
13635c4c3d92SAndrew Rybchenko 
1364e948693eSPhilip Paeps 	return (0);
1365e948693eSPhilip Paeps 
1366e948693eSPhilip Paeps fail4:
1367e948693eSPhilip Paeps 	EFSYS_PROBE(fail4);
1368e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCALE
1369e948693eSPhilip Paeps fail3:
1370e948693eSPhilip Paeps 	EFSYS_PROBE(fail3);
1371e948693eSPhilip Paeps #endif
1372e948693eSPhilip Paeps fail2:
1373e948693eSPhilip Paeps 	EFSYS_PROBE(fail2);
1374e948693eSPhilip Paeps fail1:
1375460cb568SAndrew Rybchenko 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1376e948693eSPhilip Paeps 
1377e948693eSPhilip Paeps 	return (rc);
1378e948693eSPhilip Paeps }
1379e948693eSPhilip Paeps 
1380e75412c9SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
13813c838a9fSAndrew Rybchenko 
1382fe900081SAndrew Rybchenko #if EFSYS_OPT_QSTATS
1383e948693eSPhilip Paeps #if EFSYS_OPT_NAMES
1384a260bd77SAndrew Rybchenko /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock c0f3bc5083b40532 */
1385a260bd77SAndrew Rybchenko static const char * const __efx_ev_qstat_name[] = {
1386e948693eSPhilip Paeps 	"all",
1387e948693eSPhilip Paeps 	"rx",
1388e948693eSPhilip Paeps 	"rx_ok",
1389e948693eSPhilip Paeps 	"rx_frm_trunc",
1390e948693eSPhilip Paeps 	"rx_tobe_disc",
1391e948693eSPhilip Paeps 	"rx_pause_frm_err",
1392e948693eSPhilip Paeps 	"rx_buf_owner_id_err",
1393e948693eSPhilip Paeps 	"rx_ipv4_hdr_chksum_err",
1394e948693eSPhilip Paeps 	"rx_tcp_udp_chksum_err",
1395e948693eSPhilip Paeps 	"rx_eth_crc_err",
1396e948693eSPhilip Paeps 	"rx_ip_frag_err",
1397e948693eSPhilip Paeps 	"rx_mcast_pkt",
1398e948693eSPhilip Paeps 	"rx_mcast_hash_match",
1399e948693eSPhilip Paeps 	"rx_tcp_ipv4",
1400e948693eSPhilip Paeps 	"rx_tcp_ipv6",
1401e948693eSPhilip Paeps 	"rx_udp_ipv4",
1402e948693eSPhilip Paeps 	"rx_udp_ipv6",
1403e948693eSPhilip Paeps 	"rx_other_ipv4",
1404e948693eSPhilip Paeps 	"rx_other_ipv6",
1405e948693eSPhilip Paeps 	"rx_non_ip",
14063c838a9fSAndrew Rybchenko 	"rx_batch",
1407e948693eSPhilip Paeps 	"tx",
1408e948693eSPhilip Paeps 	"tx_wq_ff_full",
1409e948693eSPhilip Paeps 	"tx_pkt_err",
1410e948693eSPhilip Paeps 	"tx_pkt_too_big",
1411e948693eSPhilip Paeps 	"tx_unexpected",
1412e948693eSPhilip Paeps 	"global",
1413e948693eSPhilip Paeps 	"global_mnt",
1414e948693eSPhilip Paeps 	"driver",
1415e948693eSPhilip Paeps 	"driver_srm_upd_done",
1416e948693eSPhilip Paeps 	"driver_tx_descq_fls_done",
1417e948693eSPhilip Paeps 	"driver_rx_descq_fls_done",
1418e948693eSPhilip Paeps 	"driver_rx_descq_fls_failed",
1419e948693eSPhilip Paeps 	"driver_rx_dsc_error",
1420e948693eSPhilip Paeps 	"driver_tx_dsc_error",
1421e948693eSPhilip Paeps 	"drv_gen",
1422e948693eSPhilip Paeps 	"mcdi_response",
1423e948693eSPhilip Paeps };
1424e948693eSPhilip Paeps /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1425e948693eSPhilip Paeps 
14263c838a9fSAndrew Rybchenko 		const char *
1427e948693eSPhilip Paeps efx_ev_qstat_name(
1428e948693eSPhilip Paeps 	__in	efx_nic_t *enp,
1429e948693eSPhilip Paeps 	__in	unsigned int id)
1430e948693eSPhilip Paeps {
1431bbea9604SAndrew Rybchenko 	_NOTE(ARGUNUSED(enp))
1432bbea9604SAndrew Rybchenko 
1433e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1434e948693eSPhilip Paeps 	EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1435e948693eSPhilip Paeps 
1436e948693eSPhilip Paeps 	return (__efx_ev_qstat_name[id]);
1437e948693eSPhilip Paeps }
1438e948693eSPhilip Paeps #endif	/* EFSYS_OPT_NAMES */
1439fe900081SAndrew Rybchenko #endif	/* EFSYS_OPT_QSTATS */
1440e948693eSPhilip Paeps 
1441e75412c9SAndrew Rybchenko #if EFSYS_OPT_SIENA
14423c838a9fSAndrew Rybchenko 
1443e948693eSPhilip Paeps #if EFSYS_OPT_QSTATS
14443c838a9fSAndrew Rybchenko static					void
14450f981da7SAndrew Rybchenko siena_ev_qstats_update(
1446e948693eSPhilip Paeps 	__in				efx_evq_t *eep,
1447e948693eSPhilip Paeps 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
1448e948693eSPhilip Paeps {
1449e948693eSPhilip Paeps 	unsigned int id;
1450e948693eSPhilip Paeps 
1451e948693eSPhilip Paeps 	for (id = 0; id < EV_NQSTATS; id++) {
1452e948693eSPhilip Paeps 		efsys_stat_t *essp = &stat[id];
1453e948693eSPhilip Paeps 
1454e948693eSPhilip Paeps 		EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1455e948693eSPhilip Paeps 		eep->ee_stat[id] = 0;
1456e948693eSPhilip Paeps 	}
1457e948693eSPhilip Paeps }
1458e948693eSPhilip Paeps #endif	/* EFSYS_OPT_QSTATS */
1459e948693eSPhilip Paeps 
14603c838a9fSAndrew Rybchenko static		void
14610f981da7SAndrew Rybchenko siena_ev_qdestroy(
1462e948693eSPhilip Paeps 	__in	efx_evq_t *eep)
1463e948693eSPhilip Paeps {
1464e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
1465e948693eSPhilip Paeps 	efx_oword_t oword;
1466e948693eSPhilip Paeps 
1467e948693eSPhilip Paeps 	/* Purge event queue */
1468e948693eSPhilip Paeps 	EFX_ZERO_OWORD(oword);
1469e948693eSPhilip Paeps 
1470e948693eSPhilip Paeps 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
14713c838a9fSAndrew Rybchenko 	    eep->ee_index, &oword, B_TRUE);
1472e948693eSPhilip Paeps 
1473e948693eSPhilip Paeps 	EFX_ZERO_OWORD(oword);
1474a4983a11SAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE);
1475e948693eSPhilip Paeps }
1476e948693eSPhilip Paeps 
14773c838a9fSAndrew Rybchenko static		void
14780f981da7SAndrew Rybchenko siena_ev_fini(
1479e948693eSPhilip Paeps 	__in	efx_nic_t *enp)
1480e948693eSPhilip Paeps {
14813c838a9fSAndrew Rybchenko 	_NOTE(ARGUNUSED(enp))
1482e948693eSPhilip Paeps }
14833c838a9fSAndrew Rybchenko 
1484e75412c9SAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
1485