xref: /freebsd/sys/dev/sfxge/common/efx_ev.c (revision 3c838a9f51e4d2a7753500d167ba7dbbb9827c82)
1e948693eSPhilip Paeps /*-
2*3c838a9fSAndrew Rybchenko  * Copyright (c) 2007-2015 Solarflare Communications Inc.
3*3c838a9fSAndrew Rybchenko  * All rights reserved.
4e948693eSPhilip Paeps  *
5e948693eSPhilip Paeps  * Redistribution and use in source and binary forms, with or without
6*3c838a9fSAndrew Rybchenko  * modification, are permitted provided that the following conditions are met:
7e948693eSPhilip Paeps  *
8*3c838a9fSAndrew Rybchenko  * 1. Redistributions of source code must retain the above copyright notice,
9*3c838a9fSAndrew Rybchenko  *    this list of conditions and the following disclaimer.
10*3c838a9fSAndrew Rybchenko  * 2. Redistributions in binary form must reproduce the above copyright notice,
11*3c838a9fSAndrew Rybchenko  *    this list of conditions and the following disclaimer in the documentation
12*3c838a9fSAndrew Rybchenko  *    and/or other materials provided with the distribution.
13*3c838a9fSAndrew Rybchenko  *
14*3c838a9fSAndrew Rybchenko  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15*3c838a9fSAndrew Rybchenko  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16*3c838a9fSAndrew Rybchenko  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17*3c838a9fSAndrew Rybchenko  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18*3c838a9fSAndrew Rybchenko  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19*3c838a9fSAndrew Rybchenko  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20*3c838a9fSAndrew Rybchenko  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21*3c838a9fSAndrew Rybchenko  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22*3c838a9fSAndrew Rybchenko  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23*3c838a9fSAndrew Rybchenko  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24*3c838a9fSAndrew Rybchenko  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25*3c838a9fSAndrew Rybchenko  *
26*3c838a9fSAndrew Rybchenko  * The views and conclusions contained in the software and documentation are
27*3c838a9fSAndrew Rybchenko  * those of the authors and should not be interpreted as representing official
28*3c838a9fSAndrew Rybchenko  * policies, either expressed or implied, of the FreeBSD Project.
29e948693eSPhilip Paeps  */
30e948693eSPhilip Paeps 
315dee87d7SPhilip Paeps #include <sys/cdefs.h>
325dee87d7SPhilip Paeps __FBSDID("$FreeBSD$");
335dee87d7SPhilip Paeps 
34e948693eSPhilip Paeps #include "efsys.h"
35e948693eSPhilip Paeps #include "efx.h"
36e948693eSPhilip Paeps #include "efx_types.h"
37e948693eSPhilip Paeps #include "efx_regs.h"
38e948693eSPhilip Paeps #include "efx_impl.h"
39*3c838a9fSAndrew Rybchenko #include "mcdi_mon.h"
40e948693eSPhilip Paeps 
41e948693eSPhilip Paeps #if EFSYS_OPT_QSTATS
42e948693eSPhilip Paeps #define	EFX_EV_QSTAT_INCR(_eep, _stat)					\
43e948693eSPhilip Paeps 	do {								\
44e948693eSPhilip Paeps 		(_eep)->ee_stat[_stat]++;				\
45e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
46e948693eSPhilip Paeps 	} while (B_FALSE)
47e948693eSPhilip Paeps #else
48e948693eSPhilip Paeps #define	EFX_EV_QSTAT_INCR(_eep, _stat)
49e948693eSPhilip Paeps #endif
50e948693eSPhilip Paeps 
51*3c838a9fSAndrew Rybchenko #define	EFX_EV_PRESENT(_qword)						\
52*3c838a9fSAndrew Rybchenko 	(EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&	\
53*3c838a9fSAndrew Rybchenko 	EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
54*3c838a9fSAndrew Rybchenko 
55*3c838a9fSAndrew Rybchenko 
56*3c838a9fSAndrew Rybchenko 
57*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
58*3c838a9fSAndrew Rybchenko 
59*3c838a9fSAndrew Rybchenko static	__checkReturn	int
60*3c838a9fSAndrew Rybchenko falconsiena_ev_init(
61*3c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp);
62*3c838a9fSAndrew Rybchenko 
63*3c838a9fSAndrew Rybchenko static			void
64*3c838a9fSAndrew Rybchenko falconsiena_ev_fini(
65*3c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp);
66*3c838a9fSAndrew Rybchenko 
67*3c838a9fSAndrew Rybchenko static	__checkReturn	int
68*3c838a9fSAndrew Rybchenko falconsiena_ev_qcreate(
69*3c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp,
70*3c838a9fSAndrew Rybchenko 	__in		unsigned int index,
71*3c838a9fSAndrew Rybchenko 	__in		efsys_mem_t *esmp,
72*3c838a9fSAndrew Rybchenko 	__in		size_t n,
73*3c838a9fSAndrew Rybchenko 	__in		uint32_t id,
74*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep);
75*3c838a9fSAndrew Rybchenko 
76*3c838a9fSAndrew Rybchenko static			void
77*3c838a9fSAndrew Rybchenko falconsiena_ev_qdestroy(
78*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep);
79*3c838a9fSAndrew Rybchenko 
80*3c838a9fSAndrew Rybchenko static	__checkReturn	int
81*3c838a9fSAndrew Rybchenko falconsiena_ev_qprime(
82*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
83*3c838a9fSAndrew Rybchenko 	__in		unsigned int count);
84*3c838a9fSAndrew Rybchenko 
85*3c838a9fSAndrew Rybchenko static			void
86*3c838a9fSAndrew Rybchenko falconsiena_ev_qpoll(
87*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
88*3c838a9fSAndrew Rybchenko 	__inout		unsigned int *countp,
89*3c838a9fSAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
90*3c838a9fSAndrew Rybchenko 	__in_opt	void *arg);
91*3c838a9fSAndrew Rybchenko 
92*3c838a9fSAndrew Rybchenko static			void
93*3c838a9fSAndrew Rybchenko falconsiena_ev_qpost(
94*3c838a9fSAndrew Rybchenko 	__in	efx_evq_t *eep,
95*3c838a9fSAndrew Rybchenko 	__in	uint16_t data);
96*3c838a9fSAndrew Rybchenko 
97*3c838a9fSAndrew Rybchenko static	__checkReturn	int
98*3c838a9fSAndrew Rybchenko falconsiena_ev_qmoderate(
99*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
100*3c838a9fSAndrew Rybchenko 	__in		unsigned int us);
101*3c838a9fSAndrew Rybchenko 
102*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
103*3c838a9fSAndrew Rybchenko static			void
104*3c838a9fSAndrew Rybchenko falconsiena_ev_qstats_update(
105*3c838a9fSAndrew Rybchenko 	__in				efx_evq_t *eep,
106*3c838a9fSAndrew Rybchenko 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat);
107*3c838a9fSAndrew Rybchenko 
108*3c838a9fSAndrew Rybchenko #endif
109*3c838a9fSAndrew Rybchenko 
110*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
111*3c838a9fSAndrew Rybchenko 
112*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_FALCON
113*3c838a9fSAndrew Rybchenko static efx_ev_ops_t	__efx_ev_falcon_ops = {
114*3c838a9fSAndrew Rybchenko 	falconsiena_ev_init,			/* eevo_init */
115*3c838a9fSAndrew Rybchenko 	falconsiena_ev_fini,			/* eevo_fini */
116*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qcreate,			/* eevo_qcreate */
117*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qdestroy,		/* eevo_qdestroy */
118*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qprime,			/* eevo_qprime */
119*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qpost,			/* eevo_qpost */
120*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qmoderate,		/* eevo_qmoderate */
121*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
122*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qstats_update,		/* eevo_qstats_update */
123*3c838a9fSAndrew Rybchenko #endif
124*3c838a9fSAndrew Rybchenko };
125*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_FALCON */
126*3c838a9fSAndrew Rybchenko 
127*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_SIENA
128*3c838a9fSAndrew Rybchenko static efx_ev_ops_t	__efx_ev_siena_ops = {
129*3c838a9fSAndrew Rybchenko 	falconsiena_ev_init,			/* eevo_init */
130*3c838a9fSAndrew Rybchenko 	falconsiena_ev_fini,			/* eevo_fini */
131*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qcreate,			/* eevo_qcreate */
132*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qdestroy,		/* eevo_qdestroy */
133*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qprime,			/* eevo_qprime */
134*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qpost,			/* eevo_qpost */
135*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qmoderate,		/* eevo_qmoderate */
136*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
137*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qstats_update,		/* eevo_qstats_update */
138*3c838a9fSAndrew Rybchenko #endif
139*3c838a9fSAndrew Rybchenko };
140*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
141*3c838a9fSAndrew Rybchenko 
142*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON
143*3c838a9fSAndrew Rybchenko static efx_ev_ops_t	__efx_ev_hunt_ops = {
144*3c838a9fSAndrew Rybchenko 	hunt_ev_init,				/* eevo_init */
145*3c838a9fSAndrew Rybchenko 	hunt_ev_fini,				/* eevo_fini */
146*3c838a9fSAndrew Rybchenko 	hunt_ev_qcreate,			/* eevo_qcreate */
147*3c838a9fSAndrew Rybchenko 	hunt_ev_qdestroy,			/* eevo_qdestroy */
148*3c838a9fSAndrew Rybchenko 	hunt_ev_qprime,				/* eevo_qprime */
149*3c838a9fSAndrew Rybchenko 	hunt_ev_qpost,				/* eevo_qpost */
150*3c838a9fSAndrew Rybchenko 	hunt_ev_qmoderate,			/* eevo_qmoderate */
151*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
152*3c838a9fSAndrew Rybchenko 	hunt_ev_qstats_update,			/* eevo_qstats_update */
153*3c838a9fSAndrew Rybchenko #endif
154*3c838a9fSAndrew Rybchenko };
155*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON */
156*3c838a9fSAndrew Rybchenko 
157*3c838a9fSAndrew Rybchenko 
158e948693eSPhilip Paeps 	__checkReturn	int
159e948693eSPhilip Paeps efx_ev_init(
160e948693eSPhilip Paeps 	__in		efx_nic_t *enp)
161e948693eSPhilip Paeps {
162*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop;
163e948693eSPhilip Paeps 	int rc;
164e948693eSPhilip Paeps 
165e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
166e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
167e948693eSPhilip Paeps 
168e948693eSPhilip Paeps 	if (enp->en_mod_flags & EFX_MOD_EV) {
169e948693eSPhilip Paeps 		rc = EINVAL;
170e948693eSPhilip Paeps 		goto fail1;
171e948693eSPhilip Paeps 	}
172e948693eSPhilip Paeps 
173*3c838a9fSAndrew Rybchenko 	switch (enp->en_family) {
174*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_FALCON
175*3c838a9fSAndrew Rybchenko 	case EFX_FAMILY_FALCON:
176*3c838a9fSAndrew Rybchenko 		eevop = (efx_ev_ops_t *)&__efx_ev_falcon_ops;
177*3c838a9fSAndrew Rybchenko 		break;
178*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_FALCON */
179*3c838a9fSAndrew Rybchenko 
180*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_SIENA
181*3c838a9fSAndrew Rybchenko 	case EFX_FAMILY_SIENA:
182*3c838a9fSAndrew Rybchenko 		eevop = (efx_ev_ops_t *)&__efx_ev_siena_ops;
183*3c838a9fSAndrew Rybchenko 		break;
184*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_SIENA */
185*3c838a9fSAndrew Rybchenko 
186*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_HUNTINGTON
187*3c838a9fSAndrew Rybchenko 	case EFX_FAMILY_HUNTINGTON:
188*3c838a9fSAndrew Rybchenko 		eevop = (efx_ev_ops_t *)&__efx_ev_hunt_ops;
189*3c838a9fSAndrew Rybchenko 		break;
190*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_HUNTINGTON */
191*3c838a9fSAndrew Rybchenko 
192*3c838a9fSAndrew Rybchenko 	default:
193*3c838a9fSAndrew Rybchenko 		EFSYS_ASSERT(0);
194*3c838a9fSAndrew Rybchenko 		rc = ENOTSUP;
195*3c838a9fSAndrew Rybchenko 		goto fail1;
196*3c838a9fSAndrew Rybchenko 	}
197*3c838a9fSAndrew Rybchenko 
198e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
199e948693eSPhilip Paeps 
200*3c838a9fSAndrew Rybchenko 	if ((rc = eevop->eevo_init(enp)) != 0)
201*3c838a9fSAndrew Rybchenko 		goto fail2;
202*3c838a9fSAndrew Rybchenko 
203*3c838a9fSAndrew Rybchenko 	enp->en_eevop = eevop;
204*3c838a9fSAndrew Rybchenko 	enp->en_mod_flags |= EFX_MOD_EV;
205*3c838a9fSAndrew Rybchenko 	return (0);
206*3c838a9fSAndrew Rybchenko 
207*3c838a9fSAndrew Rybchenko fail2:
208*3c838a9fSAndrew Rybchenko 	EFSYS_PROBE(fail2);
209*3c838a9fSAndrew Rybchenko 
210*3c838a9fSAndrew Rybchenko fail1:
211*3c838a9fSAndrew Rybchenko 	EFSYS_PROBE1(fail1, int, rc);
212*3c838a9fSAndrew Rybchenko 
213*3c838a9fSAndrew Rybchenko 	enp->en_eevop = NULL;
214*3c838a9fSAndrew Rybchenko 	enp->en_mod_flags &= ~EFX_MOD_EV;
215*3c838a9fSAndrew Rybchenko 	return (rc);
216*3c838a9fSAndrew Rybchenko }
217*3c838a9fSAndrew Rybchenko 
218*3c838a9fSAndrew Rybchenko 		void
219*3c838a9fSAndrew Rybchenko efx_ev_fini(
220*3c838a9fSAndrew Rybchenko 	__in	efx_nic_t *enp)
221*3c838a9fSAndrew Rybchenko {
222*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop = enp->en_eevop;
223*3c838a9fSAndrew Rybchenko 
224*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
225*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
226*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
227*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
228*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
229*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
230*3c838a9fSAndrew Rybchenko 
231*3c838a9fSAndrew Rybchenko 	eevop->eevo_fini(enp);
232*3c838a9fSAndrew Rybchenko 
233*3c838a9fSAndrew Rybchenko 	enp->en_eevop = NULL;
234*3c838a9fSAndrew Rybchenko 	enp->en_mod_flags &= ~EFX_MOD_EV;
235*3c838a9fSAndrew Rybchenko }
236*3c838a9fSAndrew Rybchenko 
237*3c838a9fSAndrew Rybchenko 
238*3c838a9fSAndrew Rybchenko 	__checkReturn	int
239*3c838a9fSAndrew Rybchenko efx_ev_qcreate(
240*3c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp,
241*3c838a9fSAndrew Rybchenko 	__in		unsigned int index,
242*3c838a9fSAndrew Rybchenko 	__in		efsys_mem_t *esmp,
243*3c838a9fSAndrew Rybchenko 	__in		size_t n,
244*3c838a9fSAndrew Rybchenko 	__in		uint32_t id,
245*3c838a9fSAndrew Rybchenko 	__deref_out	efx_evq_t **eepp)
246*3c838a9fSAndrew Rybchenko {
247*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop = enp->en_eevop;
248*3c838a9fSAndrew Rybchenko 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
249*3c838a9fSAndrew Rybchenko 	efx_evq_t *eep;
250*3c838a9fSAndrew Rybchenko 	int rc;
251*3c838a9fSAndrew Rybchenko 
252*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
253*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
254*3c838a9fSAndrew Rybchenko 
255*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
256*3c838a9fSAndrew Rybchenko 
257*3c838a9fSAndrew Rybchenko 	/* Allocate an EVQ object */
258*3c838a9fSAndrew Rybchenko 	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
259*3c838a9fSAndrew Rybchenko 	if (eep == NULL) {
260*3c838a9fSAndrew Rybchenko 		rc = ENOMEM;
261*3c838a9fSAndrew Rybchenko 		goto fail1;
262*3c838a9fSAndrew Rybchenko 	}
263*3c838a9fSAndrew Rybchenko 
264*3c838a9fSAndrew Rybchenko 	eep->ee_magic = EFX_EVQ_MAGIC;
265*3c838a9fSAndrew Rybchenko 	eep->ee_enp = enp;
266*3c838a9fSAndrew Rybchenko 	eep->ee_index = index;
267*3c838a9fSAndrew Rybchenko 	eep->ee_mask = n - 1;
268*3c838a9fSAndrew Rybchenko 	eep->ee_esmp = esmp;
269*3c838a9fSAndrew Rybchenko 
270*3c838a9fSAndrew Rybchenko 	if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, eep)) != 0)
271*3c838a9fSAndrew Rybchenko 		goto fail2;
272*3c838a9fSAndrew Rybchenko 
273*3c838a9fSAndrew Rybchenko 	enp->en_ev_qcount++;
274*3c838a9fSAndrew Rybchenko 	*eepp = eep;
275*3c838a9fSAndrew Rybchenko 
276*3c838a9fSAndrew Rybchenko 	return (0);
277*3c838a9fSAndrew Rybchenko 
278*3c838a9fSAndrew Rybchenko fail2:
279*3c838a9fSAndrew Rybchenko 	EFSYS_PROBE(fail2);
280*3c838a9fSAndrew Rybchenko 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
281*3c838a9fSAndrew Rybchenko fail1:
282*3c838a9fSAndrew Rybchenko 	EFSYS_PROBE1(fail1, int, rc);
283*3c838a9fSAndrew Rybchenko 	return (rc);
284*3c838a9fSAndrew Rybchenko }
285*3c838a9fSAndrew Rybchenko 
286*3c838a9fSAndrew Rybchenko 		void
287*3c838a9fSAndrew Rybchenko efx_ev_qdestroy(
288*3c838a9fSAndrew Rybchenko 	__in	efx_evq_t *eep)
289*3c838a9fSAndrew Rybchenko {
290*3c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
291*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop = enp->en_eevop;
292*3c838a9fSAndrew Rybchenko 
293*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
294*3c838a9fSAndrew Rybchenko 
295*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(enp->en_ev_qcount != 0);
296*3c838a9fSAndrew Rybchenko 	--enp->en_ev_qcount;
297*3c838a9fSAndrew Rybchenko 
298*3c838a9fSAndrew Rybchenko 	eevop->eevo_qdestroy(eep);
299*3c838a9fSAndrew Rybchenko 
300*3c838a9fSAndrew Rybchenko 	/* Free the EVQ object */
301*3c838a9fSAndrew Rybchenko 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
302*3c838a9fSAndrew Rybchenko }
303*3c838a9fSAndrew Rybchenko 
304*3c838a9fSAndrew Rybchenko 	__checkReturn	int
305*3c838a9fSAndrew Rybchenko efx_ev_qprime(
306*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
307*3c838a9fSAndrew Rybchenko 	__in		unsigned int count)
308*3c838a9fSAndrew Rybchenko {
309*3c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
310*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop = enp->en_eevop;
311*3c838a9fSAndrew Rybchenko 	int rc;
312*3c838a9fSAndrew Rybchenko 
313*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
314*3c838a9fSAndrew Rybchenko 
315*3c838a9fSAndrew Rybchenko 	if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
316*3c838a9fSAndrew Rybchenko 		rc = EINVAL;
317*3c838a9fSAndrew Rybchenko 		goto fail1;
318*3c838a9fSAndrew Rybchenko 	}
319*3c838a9fSAndrew Rybchenko 
320*3c838a9fSAndrew Rybchenko 	if ((rc = eevop->eevo_qprime(eep, count)) != 0)
321*3c838a9fSAndrew Rybchenko 		goto fail2;
322*3c838a9fSAndrew Rybchenko 
323*3c838a9fSAndrew Rybchenko 	return (0);
324*3c838a9fSAndrew Rybchenko 
325*3c838a9fSAndrew Rybchenko fail2:
326*3c838a9fSAndrew Rybchenko 	EFSYS_PROBE(fail2);
327*3c838a9fSAndrew Rybchenko fail1:
328*3c838a9fSAndrew Rybchenko 	EFSYS_PROBE1(fail1, int, rc);
329*3c838a9fSAndrew Rybchenko 	return (rc);
330*3c838a9fSAndrew Rybchenko }
331*3c838a9fSAndrew Rybchenko 
332*3c838a9fSAndrew Rybchenko 	__checkReturn	boolean_t
333*3c838a9fSAndrew Rybchenko efx_ev_qpending(
334*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
335*3c838a9fSAndrew Rybchenko 	__in		unsigned int count)
336*3c838a9fSAndrew Rybchenko {
337*3c838a9fSAndrew Rybchenko 	size_t offset;
338*3c838a9fSAndrew Rybchenko 	efx_qword_t qword;
339*3c838a9fSAndrew Rybchenko 
340*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
341*3c838a9fSAndrew Rybchenko 
342*3c838a9fSAndrew Rybchenko 	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
343*3c838a9fSAndrew Rybchenko 	EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
344*3c838a9fSAndrew Rybchenko 
345*3c838a9fSAndrew Rybchenko 	return (EFX_EV_PRESENT(qword));
346*3c838a9fSAndrew Rybchenko }
347*3c838a9fSAndrew Rybchenko 
348*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_EV_PREFETCH
349*3c838a9fSAndrew Rybchenko 
350*3c838a9fSAndrew Rybchenko 			void
351*3c838a9fSAndrew Rybchenko efx_ev_qprefetch(
352*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
353*3c838a9fSAndrew Rybchenko 	__in		unsigned int count)
354*3c838a9fSAndrew Rybchenko {
355*3c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
356*3c838a9fSAndrew Rybchenko 	unsigned int offset;
357*3c838a9fSAndrew Rybchenko 
358*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
359*3c838a9fSAndrew Rybchenko 
360*3c838a9fSAndrew Rybchenko 	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
361*3c838a9fSAndrew Rybchenko 	EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
362*3c838a9fSAndrew Rybchenko }
363*3c838a9fSAndrew Rybchenko 
364*3c838a9fSAndrew Rybchenko #endif	/* EFSYS_OPT_EV_PREFETCH */
365*3c838a9fSAndrew Rybchenko 
366*3c838a9fSAndrew Rybchenko 			void
367*3c838a9fSAndrew Rybchenko efx_ev_qpoll(
368*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
369*3c838a9fSAndrew Rybchenko 	__inout		unsigned int *countp,
370*3c838a9fSAndrew Rybchenko 	__in		const efx_ev_callbacks_t *eecp,
371*3c838a9fSAndrew Rybchenko 	__in_opt	void *arg)
372*3c838a9fSAndrew Rybchenko {
373*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
374*3c838a9fSAndrew Rybchenko 
375*3c838a9fSAndrew Rybchenko 	/*
376*3c838a9fSAndrew Rybchenko 	 * FIXME: Huntington will require support for hardware event batching
377*3c838a9fSAndrew Rybchenko 	 * and merging, which will need a different ev_qpoll implementation.
378*3c838a9fSAndrew Rybchenko 	 *
379*3c838a9fSAndrew Rybchenko 	 * Without those features the Falcon/Siena code can be used unchanged.
380*3c838a9fSAndrew Rybchenko 	 */
381*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
382*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
383*3c838a9fSAndrew Rybchenko 
384*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
385*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
386*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
387*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
388*3c838a9fSAndrew Rybchenko 	    FSE_AZ_EV_CODE_DRV_GEN_EV);
389*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_MCDI
390*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
391*3c838a9fSAndrew Rybchenko 	    FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
392*3c838a9fSAndrew Rybchenko #endif
393*3c838a9fSAndrew Rybchenko 	falconsiena_ev_qpoll(eep, countp, eecp, arg);
394*3c838a9fSAndrew Rybchenko }
395*3c838a9fSAndrew Rybchenko 
396*3c838a9fSAndrew Rybchenko 			void
397*3c838a9fSAndrew Rybchenko efx_ev_qpost(
398*3c838a9fSAndrew Rybchenko 	__in	efx_evq_t *eep,
399*3c838a9fSAndrew Rybchenko 	__in	uint16_t data)
400*3c838a9fSAndrew Rybchenko {
401*3c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
402*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop = enp->en_eevop;
403*3c838a9fSAndrew Rybchenko 
404*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
405*3c838a9fSAndrew Rybchenko 
406*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT(eevop != NULL &&
407*3c838a9fSAndrew Rybchenko 	    eevop->eevo_qpost != NULL);
408*3c838a9fSAndrew Rybchenko 
409*3c838a9fSAndrew Rybchenko 	eevop->eevo_qpost(eep, data);
410*3c838a9fSAndrew Rybchenko }
411*3c838a9fSAndrew Rybchenko 
412*3c838a9fSAndrew Rybchenko 	__checkReturn	int
413*3c838a9fSAndrew Rybchenko efx_ev_qmoderate(
414*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep,
415*3c838a9fSAndrew Rybchenko 	__in		unsigned int us)
416*3c838a9fSAndrew Rybchenko {
417*3c838a9fSAndrew Rybchenko 	efx_nic_t *enp = eep->ee_enp;
418*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop = enp->en_eevop;
419*3c838a9fSAndrew Rybchenko 	int rc;
420*3c838a9fSAndrew Rybchenko 
421*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
422*3c838a9fSAndrew Rybchenko 
423*3c838a9fSAndrew Rybchenko 	if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
424*3c838a9fSAndrew Rybchenko 		goto fail1;
425*3c838a9fSAndrew Rybchenko 
426*3c838a9fSAndrew Rybchenko 	return (0);
427*3c838a9fSAndrew Rybchenko 
428*3c838a9fSAndrew Rybchenko fail1:
429*3c838a9fSAndrew Rybchenko 	EFSYS_PROBE1(fail1, int, rc);
430*3c838a9fSAndrew Rybchenko 	return (rc);
431*3c838a9fSAndrew Rybchenko }
432*3c838a9fSAndrew Rybchenko 
433*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_QSTATS
434*3c838a9fSAndrew Rybchenko 					void
435*3c838a9fSAndrew Rybchenko efx_ev_qstats_update(
436*3c838a9fSAndrew Rybchenko 	__in				efx_evq_t *eep,
437*3c838a9fSAndrew Rybchenko 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
438*3c838a9fSAndrew Rybchenko 
439*3c838a9fSAndrew Rybchenko {	efx_nic_t *enp = eep->ee_enp;
440*3c838a9fSAndrew Rybchenko 	efx_ev_ops_t *eevop = enp->en_eevop;
441*3c838a9fSAndrew Rybchenko 
442*3c838a9fSAndrew Rybchenko 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
443*3c838a9fSAndrew Rybchenko 
444*3c838a9fSAndrew Rybchenko 	eevop->eevo_qstats_update(eep, stat);
445*3c838a9fSAndrew Rybchenko }
446*3c838a9fSAndrew Rybchenko 
447*3c838a9fSAndrew Rybchenko #endif	/* EFSYS_OPT_QSTATS */
448*3c838a9fSAndrew Rybchenko 
449*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
450*3c838a9fSAndrew Rybchenko 
451*3c838a9fSAndrew Rybchenko static	__checkReturn	int
452*3c838a9fSAndrew Rybchenko falconsiena_ev_init(
453*3c838a9fSAndrew Rybchenko 	__in		efx_nic_t *enp)
454*3c838a9fSAndrew Rybchenko {
455*3c838a9fSAndrew Rybchenko 	efx_oword_t oword;
456*3c838a9fSAndrew Rybchenko 
457e948693eSPhilip Paeps 	/*
458e948693eSPhilip Paeps 	 * Program the event queue for receive and transmit queue
459e948693eSPhilip Paeps 	 * flush events.
460e948693eSPhilip Paeps 	 */
461e948693eSPhilip Paeps 	EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
462e948693eSPhilip Paeps 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
463e948693eSPhilip Paeps 	EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
464e948693eSPhilip Paeps 
465e948693eSPhilip Paeps 	return (0);
466e948693eSPhilip Paeps 
467e948693eSPhilip Paeps }
468e948693eSPhilip Paeps 
469e948693eSPhilip Paeps static  __checkReturn   boolean_t
470*3c838a9fSAndrew Rybchenko falconsiena_ev_rx_not_ok(
471e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
472e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
473e948693eSPhilip Paeps 	__in		uint32_t label,
474e948693eSPhilip Paeps 	__in		uint32_t id,
475e948693eSPhilip Paeps 	__inout		uint16_t *flagsp)
476e948693eSPhilip Paeps {
477e948693eSPhilip Paeps 	boolean_t ignore = B_FALSE;
478e948693eSPhilip Paeps 
479e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
480e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
481e948693eSPhilip Paeps 		EFSYS_PROBE(tobe_disc);
4820ff23789SAndrew Rybchenko 		/*
4830ff23789SAndrew Rybchenko 		 * Assume this is a unicast address mismatch, unless below
484e948693eSPhilip Paeps 		 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
485e948693eSPhilip Paeps 		 * EV_RX_PAUSE_FRM_ERR is set.
486e948693eSPhilip Paeps 		 */
487e948693eSPhilip Paeps 		(*flagsp) |= EFX_ADDR_MISMATCH;
488e948693eSPhilip Paeps 	}
489e948693eSPhilip Paeps 
490e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
491e948693eSPhilip Paeps 		EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
492e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
493e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
494e948693eSPhilip Paeps 
495e948693eSPhilip Paeps #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
4960ff23789SAndrew Rybchenko 		/*
4970ff23789SAndrew Rybchenko 		 * Lookout for payload queue ran dry errors and ignore them.
498e948693eSPhilip Paeps 		 *
499e948693eSPhilip Paeps 		 * Sadly for the header/data split cases, the descriptor
500e948693eSPhilip Paeps 		 * pointer in this event refers to the header queue and
501e948693eSPhilip Paeps 		 * therefore cannot be easily detected as duplicate.
502e948693eSPhilip Paeps 		 * So we drop these and rely on the receive processing seeing
503e948693eSPhilip Paeps 		 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
504e948693eSPhilip Paeps 		 * the partially received packet.
505e948693eSPhilip Paeps 		 */
506e948693eSPhilip Paeps 		if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
507e948693eSPhilip Paeps 		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
508e948693eSPhilip Paeps 		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
509e948693eSPhilip Paeps 			ignore = B_TRUE;
510e948693eSPhilip Paeps #endif	/* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
511e948693eSPhilip Paeps 	}
512e948693eSPhilip Paeps 
513e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
514e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
515e948693eSPhilip Paeps 		EFSYS_PROBE(crc_err);
516e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_ADDR_MISMATCH;
517e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
518e948693eSPhilip Paeps 	}
519e948693eSPhilip Paeps 
520e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
521e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
522e948693eSPhilip Paeps 		EFSYS_PROBE(pause_frm_err);
523e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_ADDR_MISMATCH;
524e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
525e948693eSPhilip Paeps 	}
526e948693eSPhilip Paeps 
527e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
528e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
529e948693eSPhilip Paeps 		EFSYS_PROBE(owner_id_err);
530e948693eSPhilip Paeps 		(*flagsp) |= EFX_DISCARD;
531e948693eSPhilip Paeps 	}
532e948693eSPhilip Paeps 
533e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
534e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
535e948693eSPhilip Paeps 		EFSYS_PROBE(ipv4_err);
536e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_CKSUM_IPV4;
537e948693eSPhilip Paeps 	}
538e948693eSPhilip Paeps 
539e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
540e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
541e948693eSPhilip Paeps 		EFSYS_PROBE(udp_chk_err);
542e948693eSPhilip Paeps 		(*flagsp) &= ~EFX_CKSUM_TCPUDP;
543e948693eSPhilip Paeps 	}
544e948693eSPhilip Paeps 
545e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
546e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
547e948693eSPhilip Paeps 
548e948693eSPhilip Paeps 		/*
549e948693eSPhilip Paeps 		 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
550e948693eSPhilip Paeps 		 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
551e948693eSPhilip Paeps 		 * condition.
552e948693eSPhilip Paeps 		 */
553e948693eSPhilip Paeps 		(*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
554e948693eSPhilip Paeps 	}
555e948693eSPhilip Paeps 
556e948693eSPhilip Paeps 	return (ignore);
557e948693eSPhilip Paeps }
558e948693eSPhilip Paeps 
559e948693eSPhilip Paeps static	__checkReturn	boolean_t
560*3c838a9fSAndrew Rybchenko falconsiena_ev_rx(
561e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
562e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
563e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
564e948693eSPhilip Paeps 	__in_opt	void *arg)
565e948693eSPhilip Paeps {
566e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
567e948693eSPhilip Paeps 	uint32_t id;
568e948693eSPhilip Paeps 	uint32_t size;
569e948693eSPhilip Paeps 	uint32_t label;
570e948693eSPhilip Paeps 	boolean_t ok;
571e948693eSPhilip Paeps #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
572e948693eSPhilip Paeps 	boolean_t sop;
573e948693eSPhilip Paeps 	boolean_t jumbo_cont;
574e948693eSPhilip Paeps #endif	/* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
575e948693eSPhilip Paeps 	uint32_t hdr_type;
576e948693eSPhilip Paeps 	boolean_t is_v6;
577e948693eSPhilip Paeps 	uint16_t flags;
578e948693eSPhilip Paeps 	boolean_t ignore;
579e948693eSPhilip Paeps 	boolean_t should_abort;
580e948693eSPhilip Paeps 
581e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_RX);
582e948693eSPhilip Paeps 
583e948693eSPhilip Paeps 	/* Basic packet information */
584e948693eSPhilip Paeps 	id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
585e948693eSPhilip Paeps 	size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
586e948693eSPhilip Paeps 	label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
587e948693eSPhilip Paeps 	ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
588e948693eSPhilip Paeps 
589e948693eSPhilip Paeps #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
590e948693eSPhilip Paeps 	sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
591e948693eSPhilip Paeps 	jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
592e948693eSPhilip Paeps #endif	/* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
593e948693eSPhilip Paeps 
594e948693eSPhilip Paeps 	hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
595e948693eSPhilip Paeps 
596e948693eSPhilip Paeps 	is_v6 = (enp->en_family != EFX_FAMILY_FALCON &&
597e948693eSPhilip Paeps 		    EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
598e948693eSPhilip Paeps 
599e948693eSPhilip Paeps 	/*
600e948693eSPhilip Paeps 	 * If packet is marked as OK and packet type is TCP/IP or
601e948693eSPhilip Paeps 	 * UDP/IP or other IP, then we can rely on the hardware checksums.
602e948693eSPhilip Paeps 	 */
603e948693eSPhilip Paeps 	switch (hdr_type) {
604e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
605e948693eSPhilip Paeps 		flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
606e948693eSPhilip Paeps 		if (is_v6) {
607e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
608e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV6;
609e948693eSPhilip Paeps 		} else {
610e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
611e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
612e948693eSPhilip Paeps 		}
613e948693eSPhilip Paeps 		break;
614e948693eSPhilip Paeps 
615e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
616e948693eSPhilip Paeps 		flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
617e948693eSPhilip Paeps 		if (is_v6) {
618e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
619e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV6;
620e948693eSPhilip Paeps 		} else {
621e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
622e948693eSPhilip Paeps 			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
623e948693eSPhilip Paeps 		}
624e948693eSPhilip Paeps 		break;
625e948693eSPhilip Paeps 
626e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
627e948693eSPhilip Paeps 		if (is_v6) {
628e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
629e948693eSPhilip Paeps 			flags = EFX_PKT_IPV6;
630e948693eSPhilip Paeps 		} else {
631e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
632e948693eSPhilip Paeps 			flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
633e948693eSPhilip Paeps 		}
634e948693eSPhilip Paeps 		break;
635e948693eSPhilip Paeps 
636e948693eSPhilip Paeps 	case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
637e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
638e948693eSPhilip Paeps 		flags = 0;
639e948693eSPhilip Paeps 		break;
640e948693eSPhilip Paeps 
641e948693eSPhilip Paeps 	default:
642e948693eSPhilip Paeps 		EFSYS_ASSERT(B_FALSE);
643e948693eSPhilip Paeps 		flags = 0;
644e948693eSPhilip Paeps 		break;
645e948693eSPhilip Paeps 	}
646e948693eSPhilip Paeps 
647e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT
648e948693eSPhilip Paeps 	/* Report scatter and header/lookahead split buffer flags */
649e948693eSPhilip Paeps 	if (sop)
650e948693eSPhilip Paeps 		flags |= EFX_PKT_START;
651e948693eSPhilip Paeps 	if (jumbo_cont)
652e948693eSPhilip Paeps 		flags |= EFX_PKT_CONT;
653e948693eSPhilip Paeps #endif	/* EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT */
654e948693eSPhilip Paeps 
655e948693eSPhilip Paeps 	/* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
656e948693eSPhilip Paeps 	if (!ok) {
657*3c838a9fSAndrew Rybchenko 		ignore = falconsiena_ev_rx_not_ok(eep, eqp, label, id, &flags);
658e948693eSPhilip Paeps 		if (ignore) {
659e948693eSPhilip Paeps 			EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
660e948693eSPhilip Paeps 			    uint32_t, size, uint16_t, flags);
661e948693eSPhilip Paeps 
662e948693eSPhilip Paeps 			return (B_FALSE);
663e948693eSPhilip Paeps 		}
664e948693eSPhilip Paeps 	}
665e948693eSPhilip Paeps 
666e948693eSPhilip Paeps 	/* If we're not discarding the packet then it is ok */
667e948693eSPhilip Paeps 	if (~flags & EFX_DISCARD)
668e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
669e948693eSPhilip Paeps 
670e948693eSPhilip Paeps 	/* Detect multicast packets that didn't match the filter */
671e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
672e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
673e948693eSPhilip Paeps 
674e948693eSPhilip Paeps 		if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
675e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
676e948693eSPhilip Paeps 		} else {
677e948693eSPhilip Paeps 			EFSYS_PROBE(mcast_mismatch);
678e948693eSPhilip Paeps 			flags |= EFX_ADDR_MISMATCH;
679e948693eSPhilip Paeps 		}
680e948693eSPhilip Paeps 	} else {
681e948693eSPhilip Paeps 		flags |= EFX_PKT_UNICAST;
682e948693eSPhilip Paeps 	}
683e948693eSPhilip Paeps 
684e948693eSPhilip Paeps 	/*
685e948693eSPhilip Paeps 	 * The packet parser in Siena can abort parsing packets under
686e948693eSPhilip Paeps 	 * certain error conditions, setting the PKT_NOT_PARSED bit
687e948693eSPhilip Paeps 	 * (which clears PKT_OK). If this is set, then don't trust
688e948693eSPhilip Paeps 	 * the PKT_TYPE field.
689e948693eSPhilip Paeps 	 */
690e948693eSPhilip Paeps 	if (enp->en_family != EFX_FAMILY_FALCON && !ok) {
691e948693eSPhilip Paeps 		uint32_t parse_err;
692e948693eSPhilip Paeps 
693e948693eSPhilip Paeps 		parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
694e948693eSPhilip Paeps 		if (parse_err != 0)
695e948693eSPhilip Paeps 			flags |= EFX_CHECK_VLAN;
696e948693eSPhilip Paeps 	}
697e948693eSPhilip Paeps 
698e948693eSPhilip Paeps 	if (~flags & EFX_CHECK_VLAN) {
699e948693eSPhilip Paeps 		uint32_t pkt_type;
700e948693eSPhilip Paeps 
701e948693eSPhilip Paeps 		pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
702e948693eSPhilip Paeps 		if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
703e948693eSPhilip Paeps 			flags |= EFX_PKT_VLAN_TAGGED;
704e948693eSPhilip Paeps 	}
705e948693eSPhilip Paeps 
706e948693eSPhilip Paeps 	EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
707e948693eSPhilip Paeps 	    uint32_t, size, uint16_t, flags);
708e948693eSPhilip Paeps 
709e948693eSPhilip Paeps 	EFSYS_ASSERT(eecp->eec_rx != NULL);
710e948693eSPhilip Paeps 	should_abort = eecp->eec_rx(arg, label, id, size, flags);
711e948693eSPhilip Paeps 
712e948693eSPhilip Paeps 	return (should_abort);
713e948693eSPhilip Paeps }
714e948693eSPhilip Paeps 
715e948693eSPhilip Paeps static	__checkReturn	boolean_t
716*3c838a9fSAndrew Rybchenko falconsiena_ev_tx(
717e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
718e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
719e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
720e948693eSPhilip Paeps 	__in_opt	void *arg)
721e948693eSPhilip Paeps {
722e948693eSPhilip Paeps 	uint32_t id;
723e948693eSPhilip Paeps 	uint32_t label;
724e948693eSPhilip Paeps 	boolean_t should_abort;
725e948693eSPhilip Paeps 
726e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_TX);
727e948693eSPhilip Paeps 
728e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
729e948693eSPhilip Paeps 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
730e948693eSPhilip Paeps 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
731e948693eSPhilip Paeps 	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
732e948693eSPhilip Paeps 
733e948693eSPhilip Paeps 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
734e948693eSPhilip Paeps 		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
735e948693eSPhilip Paeps 
736e948693eSPhilip Paeps 		EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
737e948693eSPhilip Paeps 
738e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_tx != NULL);
739e948693eSPhilip Paeps 		should_abort = eecp->eec_tx(arg, label, id);
740e948693eSPhilip Paeps 
741e948693eSPhilip Paeps 		return (should_abort);
742e948693eSPhilip Paeps 	}
743e948693eSPhilip Paeps 
744e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
745e948693eSPhilip Paeps 		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
746e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
747e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
748e948693eSPhilip Paeps 
749e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
750e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
751e948693eSPhilip Paeps 
752e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
753e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
754e948693eSPhilip Paeps 
755e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
756e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
757e948693eSPhilip Paeps 
758e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
759e948693eSPhilip Paeps 	return (B_FALSE);
760e948693eSPhilip Paeps }
761e948693eSPhilip Paeps 
762e948693eSPhilip Paeps static	__checkReturn	boolean_t
763*3c838a9fSAndrew Rybchenko falconsiena_ev_global(
764e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
765e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
766e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
767e948693eSPhilip Paeps 	__in_opt	void *arg)
768e948693eSPhilip Paeps {
769e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
770e948693eSPhilip Paeps 	efx_port_t *epp = &(enp->en_port);
771e948693eSPhilip Paeps 	boolean_t should_abort;
772e948693eSPhilip Paeps 
773e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
774e948693eSPhilip Paeps 	should_abort = B_FALSE;
775e948693eSPhilip Paeps 
776e948693eSPhilip Paeps 	/* Check for a link management event */
777e948693eSPhilip Paeps 	if (EFX_QWORD_FIELD(*eqp, FSF_BZ_GLB_EV_XG_MNT_INTR) != 0) {
778e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_GLOBAL_MNT);
779e948693eSPhilip Paeps 
780e948693eSPhilip Paeps 		EFSYS_PROBE(xg_mgt);
781e948693eSPhilip Paeps 
782e948693eSPhilip Paeps 		epp->ep_mac_poll_needed = B_TRUE;
783e948693eSPhilip Paeps 	}
784e948693eSPhilip Paeps 
785e948693eSPhilip Paeps 	return (should_abort);
786e948693eSPhilip Paeps }
787e948693eSPhilip Paeps 
788e948693eSPhilip Paeps static	__checkReturn	boolean_t
789*3c838a9fSAndrew Rybchenko falconsiena_ev_driver(
790e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
791e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
792e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
793e948693eSPhilip Paeps 	__in_opt	void *arg)
794e948693eSPhilip Paeps {
795e948693eSPhilip Paeps 	boolean_t should_abort;
796e948693eSPhilip Paeps 
797e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
798e948693eSPhilip Paeps 	should_abort = B_FALSE;
799e948693eSPhilip Paeps 
800e948693eSPhilip Paeps 	switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
801e948693eSPhilip Paeps 	case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
802cf07c70dSGeorge V. Neville-Neil 		uint32_t txq_index;
803e948693eSPhilip Paeps 
804e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
805e948693eSPhilip Paeps 
806cf07c70dSGeorge V. Neville-Neil 		txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
807e948693eSPhilip Paeps 
808cf07c70dSGeorge V. Neville-Neil 		EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
809e948693eSPhilip Paeps 
810e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
811cf07c70dSGeorge V. Neville-Neil 		should_abort = eecp->eec_txq_flush_done(arg, txq_index);
812e948693eSPhilip Paeps 
813e948693eSPhilip Paeps 		break;
814e948693eSPhilip Paeps 	}
815e948693eSPhilip Paeps 	case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
816cf07c70dSGeorge V. Neville-Neil 		uint32_t rxq_index;
817e948693eSPhilip Paeps 		uint32_t failed;
818e948693eSPhilip Paeps 
819cf07c70dSGeorge V. Neville-Neil 		rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
820e948693eSPhilip Paeps 		failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
821e948693eSPhilip Paeps 
822e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
823e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
824e948693eSPhilip Paeps 
825e948693eSPhilip Paeps 		if (failed) {
826e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
827e948693eSPhilip Paeps 
828cf07c70dSGeorge V. Neville-Neil 			EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
829e948693eSPhilip Paeps 
830*3c838a9fSAndrew Rybchenko 			should_abort = eecp->eec_rxq_flush_failed(arg,
831*3c838a9fSAndrew Rybchenko 								    rxq_index);
832e948693eSPhilip Paeps 		} else {
833e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
834e948693eSPhilip Paeps 
835cf07c70dSGeorge V. Neville-Neil 			EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
836e948693eSPhilip Paeps 
837cf07c70dSGeorge V. Neville-Neil 			should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
838e948693eSPhilip Paeps 		}
839e948693eSPhilip Paeps 
840e948693eSPhilip Paeps 		break;
841e948693eSPhilip Paeps 	}
842e948693eSPhilip Paeps 	case FSE_AZ_EVQ_INIT_DONE_EV:
843e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_initialized != NULL);
844e948693eSPhilip Paeps 		should_abort = eecp->eec_initialized(arg);
845e948693eSPhilip Paeps 
846e948693eSPhilip Paeps 		break;
847e948693eSPhilip Paeps 
848e948693eSPhilip Paeps 	case FSE_AZ_EVQ_NOT_EN_EV:
849e948693eSPhilip Paeps 		EFSYS_PROBE(evq_not_en);
850e948693eSPhilip Paeps 		break;
851e948693eSPhilip Paeps 
852e948693eSPhilip Paeps 	case FSE_AZ_SRM_UPD_DONE_EV: {
853e948693eSPhilip Paeps 		uint32_t code;
854e948693eSPhilip Paeps 
855e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
856e948693eSPhilip Paeps 
857e948693eSPhilip Paeps 		code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
858e948693eSPhilip Paeps 
859e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_sram != NULL);
860e948693eSPhilip Paeps 		should_abort = eecp->eec_sram(arg, code);
861e948693eSPhilip Paeps 
862e948693eSPhilip Paeps 		break;
863e948693eSPhilip Paeps 	}
864e948693eSPhilip Paeps 	case FSE_AZ_WAKE_UP_EV: {
865e948693eSPhilip Paeps 		uint32_t id;
866e948693eSPhilip Paeps 
867e948693eSPhilip Paeps 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
868e948693eSPhilip Paeps 
869e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_wake_up != NULL);
870e948693eSPhilip Paeps 		should_abort = eecp->eec_wake_up(arg, id);
871e948693eSPhilip Paeps 
872e948693eSPhilip Paeps 		break;
873e948693eSPhilip Paeps 	}
874e948693eSPhilip Paeps 	case FSE_AZ_TX_PKT_NON_TCP_UDP:
875e948693eSPhilip Paeps 		EFSYS_PROBE(tx_pkt_non_tcp_udp);
876e948693eSPhilip Paeps 		break;
877e948693eSPhilip Paeps 
878e948693eSPhilip Paeps 	case FSE_AZ_TIMER_EV: {
879e948693eSPhilip Paeps 		uint32_t id;
880e948693eSPhilip Paeps 
881e948693eSPhilip Paeps 		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
882e948693eSPhilip Paeps 
883e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_timer != NULL);
884e948693eSPhilip Paeps 		should_abort = eecp->eec_timer(arg, id);
885e948693eSPhilip Paeps 
886e948693eSPhilip Paeps 		break;
887e948693eSPhilip Paeps 	}
888e948693eSPhilip Paeps 	case FSE_AZ_RX_DSC_ERROR_EV:
889e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
890e948693eSPhilip Paeps 
891e948693eSPhilip Paeps 		EFSYS_PROBE(rx_dsc_error);
892e948693eSPhilip Paeps 
893e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_exception != NULL);
894e948693eSPhilip Paeps 		should_abort = eecp->eec_exception(arg,
895e948693eSPhilip Paeps 			EFX_EXCEPTION_RX_DSC_ERROR, 0);
896e948693eSPhilip Paeps 
897e948693eSPhilip Paeps 		break;
898e948693eSPhilip Paeps 
899e948693eSPhilip Paeps 	case FSE_AZ_TX_DSC_ERROR_EV:
900e948693eSPhilip Paeps 		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
901e948693eSPhilip Paeps 
902e948693eSPhilip Paeps 		EFSYS_PROBE(tx_dsc_error);
903e948693eSPhilip Paeps 
904e948693eSPhilip Paeps 		EFSYS_ASSERT(eecp->eec_exception != NULL);
905e948693eSPhilip Paeps 		should_abort = eecp->eec_exception(arg,
906e948693eSPhilip Paeps 			EFX_EXCEPTION_TX_DSC_ERROR, 0);
907e948693eSPhilip Paeps 
908e948693eSPhilip Paeps 		break;
909e948693eSPhilip Paeps 
910e948693eSPhilip Paeps 	default:
911e948693eSPhilip Paeps 		break;
912e948693eSPhilip Paeps 	}
913e948693eSPhilip Paeps 
914e948693eSPhilip Paeps 	return (should_abort);
915e948693eSPhilip Paeps }
916e948693eSPhilip Paeps 
917e948693eSPhilip Paeps static	__checkReturn	boolean_t
918*3c838a9fSAndrew Rybchenko falconsiena_ev_drv_gen(
919e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
920e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
921e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
922e948693eSPhilip Paeps 	__in_opt	void *arg)
923e948693eSPhilip Paeps {
924e948693eSPhilip Paeps 	uint32_t data;
925e948693eSPhilip Paeps 	boolean_t should_abort;
926e948693eSPhilip Paeps 
927e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
928e948693eSPhilip Paeps 
929e948693eSPhilip Paeps 	data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
930e948693eSPhilip Paeps 	if (data >= ((uint32_t)1 << 16)) {
931e948693eSPhilip Paeps 		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
932e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
933e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
934e948693eSPhilip Paeps 		return (B_TRUE);
935e948693eSPhilip Paeps 	}
936e948693eSPhilip Paeps 
937e948693eSPhilip Paeps 	EFSYS_ASSERT(eecp->eec_software != NULL);
938e948693eSPhilip Paeps 	should_abort = eecp->eec_software(arg, (uint16_t)data);
939e948693eSPhilip Paeps 
940e948693eSPhilip Paeps 	return (should_abort);
941e948693eSPhilip Paeps }
942e948693eSPhilip Paeps 
943e948693eSPhilip Paeps #if EFSYS_OPT_MCDI
944e948693eSPhilip Paeps 
945e948693eSPhilip Paeps static	__checkReturn	boolean_t
946*3c838a9fSAndrew Rybchenko falconsiena_ev_mcdi(
947e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
948e948693eSPhilip Paeps 	__in		efx_qword_t *eqp,
949e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
950e948693eSPhilip Paeps 	__in_opt	void *arg)
951e948693eSPhilip Paeps {
952e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
953e948693eSPhilip Paeps 	unsigned code;
954e948693eSPhilip Paeps 	boolean_t should_abort = B_FALSE;
955e948693eSPhilip Paeps 
956e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
957e948693eSPhilip Paeps 
958e948693eSPhilip Paeps 	if (enp->en_family != EFX_FAMILY_SIENA)
959e948693eSPhilip Paeps 		goto out;
960e948693eSPhilip Paeps 
9616da0beb5SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_link_change != NULL);
9626da0beb5SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_exception != NULL);
9636da0beb5SAndrew Rybchenko #if EFSYS_OPT_MON_STATS
9646da0beb5SAndrew Rybchenko 	EFSYS_ASSERT(eecp->eec_monitor != NULL);
9656da0beb5SAndrew Rybchenko #endif
9666da0beb5SAndrew Rybchenko 
967e948693eSPhilip Paeps 	EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
968e948693eSPhilip Paeps 
969e948693eSPhilip Paeps 	code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
970e948693eSPhilip Paeps 	switch (code) {
971e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_BADSSERT:
972e948693eSPhilip Paeps 		efx_mcdi_ev_death(enp, EINTR);
973e948693eSPhilip Paeps 		break;
974e948693eSPhilip Paeps 
975e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_CMDDONE:
976e948693eSPhilip Paeps 		efx_mcdi_ev_cpl(enp,
977d8574f4fSAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
978d8574f4fSAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
979d8574f4fSAndrew Rybchenko 		    MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
980e948693eSPhilip Paeps 		break;
981e948693eSPhilip Paeps 
982e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_LINKCHANGE: {
983e948693eSPhilip Paeps 		efx_link_mode_t link_mode;
984e948693eSPhilip Paeps 
985e948693eSPhilip Paeps 		siena_phy_link_ev(enp, eqp, &link_mode);
986e948693eSPhilip Paeps 		should_abort = eecp->eec_link_change(arg, link_mode);
987e948693eSPhilip Paeps 		break;
988e948693eSPhilip Paeps 	}
989e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_SENSOREVT: {
990e948693eSPhilip Paeps #if EFSYS_OPT_MON_STATS
991e948693eSPhilip Paeps 		efx_mon_stat_t id;
992e948693eSPhilip Paeps 		efx_mon_stat_value_t value;
993e948693eSPhilip Paeps 		int rc;
994e948693eSPhilip Paeps 
995*3c838a9fSAndrew Rybchenko 		if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
996e948693eSPhilip Paeps 			should_abort = eecp->eec_monitor(arg, id, value);
997e948693eSPhilip Paeps 		else if (rc == ENOTSUP) {
998e948693eSPhilip Paeps 			should_abort = eecp->eec_exception(arg,
999e948693eSPhilip Paeps 				EFX_EXCEPTION_UNKNOWN_SENSOREVT,
1000e948693eSPhilip Paeps 				MCDI_EV_FIELD(eqp, DATA));
1001e948693eSPhilip Paeps 		} else
1002e948693eSPhilip Paeps 			EFSYS_ASSERT(rc == ENODEV);	/* Wrong port */
1003e948693eSPhilip Paeps #else
1004e948693eSPhilip Paeps 		should_abort = B_FALSE;
1005e948693eSPhilip Paeps #endif
1006e948693eSPhilip Paeps 		break;
1007e948693eSPhilip Paeps 	}
1008e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_SCHEDERR:
1009e948693eSPhilip Paeps 		/* Informational only */
1010e948693eSPhilip Paeps 		break;
1011e948693eSPhilip Paeps 
1012e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_REBOOT:
1013e948693eSPhilip Paeps 		efx_mcdi_ev_death(enp, EIO);
1014e948693eSPhilip Paeps 		break;
1015e948693eSPhilip Paeps 
1016e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_MAC_STATS_DMA:
1017e948693eSPhilip Paeps #if EFSYS_OPT_MAC_STATS
1018e948693eSPhilip Paeps 		if (eecp->eec_mac_stats != NULL) {
1019e948693eSPhilip Paeps 			eecp->eec_mac_stats(arg,
1020e948693eSPhilip Paeps 			    MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
1021e948693eSPhilip Paeps 		}
1022e948693eSPhilip Paeps #endif
1023e948693eSPhilip Paeps 		break;
1024e948693eSPhilip Paeps 
1025e948693eSPhilip Paeps 	case MCDI_EVENT_CODE_FWALERT: {
1026e948693eSPhilip Paeps 		uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
1027e948693eSPhilip Paeps 
1028e948693eSPhilip Paeps 		if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
1029e948693eSPhilip Paeps 			should_abort = eecp->eec_exception(arg,
1030e948693eSPhilip Paeps 				EFX_EXCEPTION_FWALERT_SRAM,
1031e948693eSPhilip Paeps 				MCDI_EV_FIELD(eqp, FWALERT_DATA));
1032e948693eSPhilip Paeps 		else
1033e948693eSPhilip Paeps 			should_abort = eecp->eec_exception(arg,
1034e948693eSPhilip Paeps 				EFX_EXCEPTION_UNKNOWN_FWALERT,
1035e948693eSPhilip Paeps 				MCDI_EV_FIELD(eqp, DATA));
1036e948693eSPhilip Paeps 		break;
1037e948693eSPhilip Paeps 	}
1038e948693eSPhilip Paeps 
1039e948693eSPhilip Paeps 	default:
1040e948693eSPhilip Paeps 		EFSYS_PROBE1(mc_pcol_error, int, code);
1041e948693eSPhilip Paeps 		break;
1042e948693eSPhilip Paeps 	}
1043e948693eSPhilip Paeps 
1044e948693eSPhilip Paeps out:
1045e948693eSPhilip Paeps 	return (should_abort);
1046e948693eSPhilip Paeps }
1047e948693eSPhilip Paeps 
1048c071447aSAndrew Rybchenko #endif	/* EFSYS_OPT_MCDI */
1049e948693eSPhilip Paeps 
1050*3c838a9fSAndrew Rybchenko static	__checkReturn	int
1051*3c838a9fSAndrew Rybchenko falconsiena_ev_qprime(
1052e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
1053e948693eSPhilip Paeps 	__in		unsigned int count)
1054e948693eSPhilip Paeps {
1055e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
1056e948693eSPhilip Paeps 	uint32_t rptr;
1057e948693eSPhilip Paeps 	efx_dword_t dword;
1058e948693eSPhilip Paeps 
1059e948693eSPhilip Paeps 	rptr = count & eep->ee_mask;
1060e948693eSPhilip Paeps 
1061e948693eSPhilip Paeps 	EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
1062e948693eSPhilip Paeps 
1063e948693eSPhilip Paeps 	EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
1064e948693eSPhilip Paeps 			    &dword, B_FALSE);
1065e948693eSPhilip Paeps 
1066e948693eSPhilip Paeps 	return (0);
1067e948693eSPhilip Paeps }
1068e948693eSPhilip Paeps 
1069e948693eSPhilip Paeps #define	EFX_EV_BATCH	8
1070e948693eSPhilip Paeps 
1071*3c838a9fSAndrew Rybchenko static			void
1072*3c838a9fSAndrew Rybchenko falconsiena_ev_qpoll(
1073e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
1074e948693eSPhilip Paeps 	__inout		unsigned int *countp,
1075e948693eSPhilip Paeps 	__in		const efx_ev_callbacks_t *eecp,
1076e948693eSPhilip Paeps 	__in_opt	void *arg)
1077e948693eSPhilip Paeps {
1078e948693eSPhilip Paeps 	efx_qword_t ev[EFX_EV_BATCH];
1079e948693eSPhilip Paeps 	unsigned int batch;
1080e948693eSPhilip Paeps 	unsigned int total;
1081e948693eSPhilip Paeps 	unsigned int count;
1082e948693eSPhilip Paeps 	unsigned int index;
1083e948693eSPhilip Paeps 	size_t offset;
1084e948693eSPhilip Paeps 
1085e948693eSPhilip Paeps 	EFSYS_ASSERT(countp != NULL);
1086e948693eSPhilip Paeps 	EFSYS_ASSERT(eecp != NULL);
1087e948693eSPhilip Paeps 
1088e948693eSPhilip Paeps 	count = *countp;
1089e948693eSPhilip Paeps 	do {
1090e948693eSPhilip Paeps 		/* Read up until the end of the batch period */
1091e948693eSPhilip Paeps 		batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
1092e948693eSPhilip Paeps 		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1093e948693eSPhilip Paeps 		for (total = 0; total < batch; ++total) {
1094e948693eSPhilip Paeps 			EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
1095e948693eSPhilip Paeps 
1096e948693eSPhilip Paeps 			if (!EFX_EV_PRESENT(ev[total]))
1097e948693eSPhilip Paeps 				break;
1098e948693eSPhilip Paeps 
1099e948693eSPhilip Paeps 			EFSYS_PROBE3(event, unsigned int, eep->ee_index,
1100e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
1101e948693eSPhilip Paeps 			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
1102e948693eSPhilip Paeps 
1103e948693eSPhilip Paeps 			offset += sizeof (efx_qword_t);
1104e948693eSPhilip Paeps 		}
1105e948693eSPhilip Paeps 
1106e948693eSPhilip Paeps #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
1107e948693eSPhilip Paeps 		/*
1108e948693eSPhilip Paeps 		 * Prefetch the next batch when we get within PREFETCH_PERIOD
1109e948693eSPhilip Paeps 		 * of a completed batch. If the batch is smaller, then prefetch
1110e948693eSPhilip Paeps 		 * immediately.
1111e948693eSPhilip Paeps 		 */
1112e948693eSPhilip Paeps 		if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
1113e948693eSPhilip Paeps 			EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1114e948693eSPhilip Paeps #endif	/* EFSYS_OPT_EV_PREFETCH */
1115e948693eSPhilip Paeps 
1116e948693eSPhilip Paeps 		/* Process the batch of events */
1117e948693eSPhilip Paeps 		for (index = 0; index < total; ++index) {
1118e948693eSPhilip Paeps 			boolean_t should_abort;
1119e948693eSPhilip Paeps 			uint32_t code;
1120e948693eSPhilip Paeps 
1121e948693eSPhilip Paeps #if EFSYS_OPT_EV_PREFETCH
1122e948693eSPhilip Paeps 			/* Prefetch if we've now reached the batch period */
1123e948693eSPhilip Paeps 			if (total == batch &&
1124e948693eSPhilip Paeps 			    index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
1125e948693eSPhilip Paeps 				offset = (count + batch) & eep->ee_mask;
1126e948693eSPhilip Paeps 				offset *= sizeof (efx_qword_t);
1127e948693eSPhilip Paeps 
1128e948693eSPhilip Paeps 				EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1129e948693eSPhilip Paeps 			}
1130e948693eSPhilip Paeps #endif	/* EFSYS_OPT_EV_PREFETCH */
1131e948693eSPhilip Paeps 
1132e948693eSPhilip Paeps 			EFX_EV_QSTAT_INCR(eep, EV_ALL);
1133e948693eSPhilip Paeps 
1134e948693eSPhilip Paeps 			code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
1135*3c838a9fSAndrew Rybchenko 			switch (code) {
1136*3c838a9fSAndrew Rybchenko 			case FSE_AZ_EV_CODE_RX_EV:
1137*3c838a9fSAndrew Rybchenko 				should_abort = eep->ee_rx(eep,
1138*3c838a9fSAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1139*3c838a9fSAndrew Rybchenko 				break;
1140*3c838a9fSAndrew Rybchenko 			case FSE_AZ_EV_CODE_TX_EV:
1141*3c838a9fSAndrew Rybchenko 				should_abort = eep->ee_tx(eep,
1142*3c838a9fSAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1143*3c838a9fSAndrew Rybchenko 				break;
1144*3c838a9fSAndrew Rybchenko 			case FSE_AZ_EV_CODE_DRIVER_EV:
1145*3c838a9fSAndrew Rybchenko 				should_abort = eep->ee_driver(eep,
1146*3c838a9fSAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1147*3c838a9fSAndrew Rybchenko 				break;
1148*3c838a9fSAndrew Rybchenko 			case FSE_AZ_EV_CODE_DRV_GEN_EV:
1149*3c838a9fSAndrew Rybchenko 				should_abort = eep->ee_drv_gen(eep,
1150*3c838a9fSAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1151*3c838a9fSAndrew Rybchenko 				break;
1152*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_MCDI
1153*3c838a9fSAndrew Rybchenko 			case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
1154*3c838a9fSAndrew Rybchenko 				should_abort = eep->ee_mcdi(eep,
1155*3c838a9fSAndrew Rybchenko 				    &(ev[index]), eecp, arg);
1156*3c838a9fSAndrew Rybchenko 				break;
1157*3c838a9fSAndrew Rybchenko #endif
1158*3c838a9fSAndrew Rybchenko 			case FSE_AZ_EV_CODE_GLOBAL_EV:
1159*3c838a9fSAndrew Rybchenko 				if (eep->ee_global) {
1160*3c838a9fSAndrew Rybchenko 					should_abort = eep->ee_global(eep,
1161*3c838a9fSAndrew Rybchenko 					    &(ev[index]), eecp, arg);
1162*3c838a9fSAndrew Rybchenko 					break;
1163*3c838a9fSAndrew Rybchenko 				}
1164*3c838a9fSAndrew Rybchenko 				/* else fallthrough */
1165*3c838a9fSAndrew Rybchenko 			default:
1166*3c838a9fSAndrew Rybchenko 				EFSYS_PROBE3(bad_event,
1167*3c838a9fSAndrew Rybchenko 				    unsigned int, eep->ee_index,
1168*3c838a9fSAndrew Rybchenko 				    uint32_t,
1169*3c838a9fSAndrew Rybchenko 				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
1170*3c838a9fSAndrew Rybchenko 				    uint32_t,
1171*3c838a9fSAndrew Rybchenko 				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
1172*3c838a9fSAndrew Rybchenko 
1173*3c838a9fSAndrew Rybchenko 				EFSYS_ASSERT(eecp->eec_exception != NULL);
1174*3c838a9fSAndrew Rybchenko 				(void) eecp->eec_exception(arg,
1175*3c838a9fSAndrew Rybchenko 					EFX_EXCEPTION_EV_ERROR, code);
1176*3c838a9fSAndrew Rybchenko 				should_abort = B_TRUE;
1177*3c838a9fSAndrew Rybchenko 			}
1178e948693eSPhilip Paeps 			if (should_abort) {
1179e948693eSPhilip Paeps 				/* Ignore subsequent events */
1180e948693eSPhilip Paeps 				total = index + 1;
1181e948693eSPhilip Paeps 				break;
1182e948693eSPhilip Paeps 			}
1183e948693eSPhilip Paeps 		}
1184e948693eSPhilip Paeps 
1185e948693eSPhilip Paeps 		/*
1186e948693eSPhilip Paeps 		 * Now that the hardware has most likely moved onto dma'ing
1187e948693eSPhilip Paeps 		 * into the next cache line, clear the processed events. Take
1188e948693eSPhilip Paeps 		 * care to only clear out events that we've processed
1189e948693eSPhilip Paeps 		 */
1190e948693eSPhilip Paeps 		EFX_SET_QWORD(ev[0]);
1191e948693eSPhilip Paeps 		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1192e948693eSPhilip Paeps 		for (index = 0; index < total; ++index) {
1193e948693eSPhilip Paeps 			EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
1194e948693eSPhilip Paeps 			offset += sizeof (efx_qword_t);
1195e948693eSPhilip Paeps 		}
1196e948693eSPhilip Paeps 
1197e948693eSPhilip Paeps 		count += total;
1198e948693eSPhilip Paeps 
1199e948693eSPhilip Paeps 	} while (total == batch);
1200e948693eSPhilip Paeps 
1201e948693eSPhilip Paeps 	*countp = count;
1202e948693eSPhilip Paeps }
1203e948693eSPhilip Paeps 
1204*3c838a9fSAndrew Rybchenko static		void
1205*3c838a9fSAndrew Rybchenko falconsiena_ev_qpost(
1206e948693eSPhilip Paeps 	__in	efx_evq_t *eep,
1207e948693eSPhilip Paeps 	__in	uint16_t data)
1208e948693eSPhilip Paeps {
1209e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
1210e948693eSPhilip Paeps 	efx_qword_t ev;
1211e948693eSPhilip Paeps 	efx_oword_t oword;
1212e948693eSPhilip Paeps 
1213e948693eSPhilip Paeps 	EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
1214e948693eSPhilip Paeps 	    FSF_AZ_EV_DATA_DW0, (uint32_t)data);
1215e948693eSPhilip Paeps 
1216e948693eSPhilip Paeps 	EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
1217e948693eSPhilip Paeps 	    EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
1218e948693eSPhilip Paeps 	    EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
1219e948693eSPhilip Paeps 
1220e948693eSPhilip Paeps 	EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
1221e948693eSPhilip Paeps }
1222e948693eSPhilip Paeps 
1223*3c838a9fSAndrew Rybchenko static	__checkReturn	int
1224*3c838a9fSAndrew Rybchenko falconsiena_ev_qmoderate(
1225e948693eSPhilip Paeps 	__in		efx_evq_t *eep,
1226e948693eSPhilip Paeps 	__in		unsigned int us)
1227e948693eSPhilip Paeps {
1228e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
12298bf93a04SAndrew Rybchenko 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1230e948693eSPhilip Paeps 	unsigned int locked;
1231e948693eSPhilip Paeps 	efx_dword_t dword;
1232e948693eSPhilip Paeps 	int rc;
1233e948693eSPhilip Paeps 
1234af9078c3SAndrew Rybchenko 	if (us > encp->enc_evq_timer_max_us) {
1235e948693eSPhilip Paeps 		rc = EINVAL;
1236e948693eSPhilip Paeps 		goto fail1;
1237e948693eSPhilip Paeps 	}
1238e948693eSPhilip Paeps 
1239e948693eSPhilip Paeps 	/* If the value is zero then disable the timer */
1240e948693eSPhilip Paeps 	if (us == 0) {
1241e948693eSPhilip Paeps 		if (enp->en_family == EFX_FAMILY_FALCON)
1242e948693eSPhilip Paeps 			EFX_POPULATE_DWORD_2(dword,
1243e948693eSPhilip Paeps 			    FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_DIS,
1244e948693eSPhilip Paeps 			    FRF_AB_TC_TIMER_VAL, 0);
1245e948693eSPhilip Paeps 		else
1246e948693eSPhilip Paeps 			EFX_POPULATE_DWORD_2(dword,
1247e948693eSPhilip Paeps 			    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
1248e948693eSPhilip Paeps 			    FRF_CZ_TC_TIMER_VAL, 0);
1249e948693eSPhilip Paeps 	} else {
1250e948693eSPhilip Paeps 		uint32_t timer_val;
1251e948693eSPhilip Paeps 
1252e948693eSPhilip Paeps 		/* Calculate the timer value in quanta */
1253af9078c3SAndrew Rybchenko 		timer_val = us * 1000 / encp->enc_evq_timer_quantum_ns;
1254e948693eSPhilip Paeps 
1255e948693eSPhilip Paeps 		/* Moderation value is base 0 so we need to deduct 1 */
12568bf93a04SAndrew Rybchenko 		if (timer_val > 0)
12578bf93a04SAndrew Rybchenko 			timer_val--;
12588bf93a04SAndrew Rybchenko 
1259e948693eSPhilip Paeps 		if (enp->en_family == EFX_FAMILY_FALCON)
1260e948693eSPhilip Paeps 			EFX_POPULATE_DWORD_2(dword,
1261e948693eSPhilip Paeps 			    FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_INT_HLDOFF,
12628bf93a04SAndrew Rybchenko 			    FRF_AB_TIMER_VAL, timer_val);
1263e948693eSPhilip Paeps 		else
1264e948693eSPhilip Paeps 			EFX_POPULATE_DWORD_2(dword,
1265e948693eSPhilip Paeps 			    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
12668bf93a04SAndrew Rybchenko 			    FRF_CZ_TC_TIMER_VAL, timer_val);
1267e948693eSPhilip Paeps 	}
1268e948693eSPhilip Paeps 
1269e948693eSPhilip Paeps 	locked = (eep->ee_index == 0) ? 1 : 0;
1270e948693eSPhilip Paeps 
1271e948693eSPhilip Paeps 	EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
1272e948693eSPhilip Paeps 	    eep->ee_index, &dword, locked);
1273e948693eSPhilip Paeps 
1274e948693eSPhilip Paeps 	return (0);
1275e948693eSPhilip Paeps 
1276e948693eSPhilip Paeps fail1:
1277e948693eSPhilip Paeps 	EFSYS_PROBE1(fail1, int, rc);
1278e948693eSPhilip Paeps 
1279e948693eSPhilip Paeps 	return (rc);
1280e948693eSPhilip Paeps }
1281e948693eSPhilip Paeps 
1282*3c838a9fSAndrew Rybchenko static	__checkReturn	int
1283*3c838a9fSAndrew Rybchenko falconsiena_ev_qcreate(
1284e948693eSPhilip Paeps 	__in		efx_nic_t *enp,
1285e948693eSPhilip Paeps 	__in		unsigned int index,
1286e948693eSPhilip Paeps 	__in		efsys_mem_t *esmp,
1287e948693eSPhilip Paeps 	__in		size_t n,
1288e948693eSPhilip Paeps 	__in		uint32_t id,
1289*3c838a9fSAndrew Rybchenko 	__in		efx_evq_t *eep)
1290e948693eSPhilip Paeps {
1291e948693eSPhilip Paeps 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1292e948693eSPhilip Paeps 	uint32_t size;
1293e948693eSPhilip Paeps 	efx_oword_t oword;
1294e948693eSPhilip Paeps 	int rc;
1295e948693eSPhilip Paeps 
1296*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS));
1297*3c838a9fSAndrew Rybchenko 	EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS));
1298e948693eSPhilip Paeps 
1299*3c838a9fSAndrew Rybchenko 	if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) {
1300e948693eSPhilip Paeps 		rc = EINVAL;
1301e948693eSPhilip Paeps 		goto fail1;
1302e948693eSPhilip Paeps 	}
1303e948693eSPhilip Paeps 	if (index >= encp->enc_evq_limit) {
1304e948693eSPhilip Paeps 		rc = EINVAL;
1305e948693eSPhilip Paeps 		goto fail2;
1306e948693eSPhilip Paeps 	}
1307e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCALE
1308e948693eSPhilip Paeps 	if (enp->en_intr.ei_type == EFX_INTR_LINE &&
1309e948693eSPhilip Paeps 	    index >= EFX_MAXRSS_LEGACY) {
1310e948693eSPhilip Paeps 		rc = EINVAL;
1311e948693eSPhilip Paeps 		goto fail3;
1312e948693eSPhilip Paeps 	}
1313e948693eSPhilip Paeps #endif
1314e948693eSPhilip Paeps 	for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
1315e948693eSPhilip Paeps 	    size++)
1316e948693eSPhilip Paeps 		if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
1317e948693eSPhilip Paeps 			break;
1318e948693eSPhilip Paeps 	if (id + (1 << size) >= encp->enc_buftbl_limit) {
1319e948693eSPhilip Paeps 		rc = EINVAL;
1320e948693eSPhilip Paeps 		goto fail4;
1321e948693eSPhilip Paeps 	}
1322e948693eSPhilip Paeps 
1323e948693eSPhilip Paeps 	/* Set up the handler table */
1324*3c838a9fSAndrew Rybchenko 	eep->ee_rx	= falconsiena_ev_rx;
1325*3c838a9fSAndrew Rybchenko 	eep->ee_tx	= falconsiena_ev_tx;
1326*3c838a9fSAndrew Rybchenko 	eep->ee_driver	= falconsiena_ev_driver;
1327*3c838a9fSAndrew Rybchenko 	eep->ee_global	= falconsiena_ev_global;
1328*3c838a9fSAndrew Rybchenko 	eep->ee_drv_gen	= falconsiena_ev_drv_gen;
1329e948693eSPhilip Paeps #if EFSYS_OPT_MCDI
1330*3c838a9fSAndrew Rybchenko 	eep->ee_mcdi	= falconsiena_ev_mcdi;
1331c071447aSAndrew Rybchenko #endif	/* EFSYS_OPT_MCDI */
1332e948693eSPhilip Paeps 
1333e948693eSPhilip Paeps 	/* Set up the new event queue */
1334e948693eSPhilip Paeps 	if (enp->en_family != EFX_FAMILY_FALCON) {
1335e948693eSPhilip Paeps 		EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1);
1336*3c838a9fSAndrew Rybchenko 		EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
1337e948693eSPhilip Paeps 	}
1338e948693eSPhilip Paeps 
1339e948693eSPhilip Paeps 	EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
1340e948693eSPhilip Paeps 	    FRF_AZ_EVQ_BUF_BASE_ID, id);
1341e948693eSPhilip Paeps 
1342*3c838a9fSAndrew Rybchenko 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
1343e948693eSPhilip Paeps 
1344e948693eSPhilip Paeps 	return (0);
1345e948693eSPhilip Paeps 
1346e948693eSPhilip Paeps fail4:
1347e948693eSPhilip Paeps 	EFSYS_PROBE(fail4);
1348e948693eSPhilip Paeps #if EFSYS_OPT_RX_SCALE
1349e948693eSPhilip Paeps fail3:
1350e948693eSPhilip Paeps 	EFSYS_PROBE(fail3);
1351e948693eSPhilip Paeps #endif
1352e948693eSPhilip Paeps fail2:
1353e948693eSPhilip Paeps 	EFSYS_PROBE(fail2);
1354e948693eSPhilip Paeps fail1:
1355e948693eSPhilip Paeps 	EFSYS_PROBE1(fail1, int, rc);
1356e948693eSPhilip Paeps 
1357e948693eSPhilip Paeps 	return (rc);
1358e948693eSPhilip Paeps }
1359e948693eSPhilip Paeps 
1360*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1361*3c838a9fSAndrew Rybchenko 
1362fe900081SAndrew Rybchenko #if EFSYS_OPT_QSTATS
1363e948693eSPhilip Paeps #if EFSYS_OPT_NAMES
1364*3c838a9fSAndrew Rybchenko /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock b693ddf85aee1bfd */
1365*3c838a9fSAndrew Rybchenko static const char 	*__efx_ev_qstat_name[] = {
1366e948693eSPhilip Paeps 	"all",
1367e948693eSPhilip Paeps 	"rx",
1368e948693eSPhilip Paeps 	"rx_ok",
1369e948693eSPhilip Paeps 	"rx_frm_trunc",
1370e948693eSPhilip Paeps 	"rx_tobe_disc",
1371e948693eSPhilip Paeps 	"rx_pause_frm_err",
1372e948693eSPhilip Paeps 	"rx_buf_owner_id_err",
1373e948693eSPhilip Paeps 	"rx_ipv4_hdr_chksum_err",
1374e948693eSPhilip Paeps 	"rx_tcp_udp_chksum_err",
1375e948693eSPhilip Paeps 	"rx_eth_crc_err",
1376e948693eSPhilip Paeps 	"rx_ip_frag_err",
1377e948693eSPhilip Paeps 	"rx_mcast_pkt",
1378e948693eSPhilip Paeps 	"rx_mcast_hash_match",
1379e948693eSPhilip Paeps 	"rx_tcp_ipv4",
1380e948693eSPhilip Paeps 	"rx_tcp_ipv6",
1381e948693eSPhilip Paeps 	"rx_udp_ipv4",
1382e948693eSPhilip Paeps 	"rx_udp_ipv6",
1383e948693eSPhilip Paeps 	"rx_other_ipv4",
1384e948693eSPhilip Paeps 	"rx_other_ipv6",
1385e948693eSPhilip Paeps 	"rx_non_ip",
1386*3c838a9fSAndrew Rybchenko 	"rx_batch",
1387e948693eSPhilip Paeps 	"tx",
1388e948693eSPhilip Paeps 	"tx_wq_ff_full",
1389e948693eSPhilip Paeps 	"tx_pkt_err",
1390e948693eSPhilip Paeps 	"tx_pkt_too_big",
1391e948693eSPhilip Paeps 	"tx_unexpected",
1392e948693eSPhilip Paeps 	"global",
1393e948693eSPhilip Paeps 	"global_mnt",
1394e948693eSPhilip Paeps 	"driver",
1395e948693eSPhilip Paeps 	"driver_srm_upd_done",
1396e948693eSPhilip Paeps 	"driver_tx_descq_fls_done",
1397e948693eSPhilip Paeps 	"driver_rx_descq_fls_done",
1398e948693eSPhilip Paeps 	"driver_rx_descq_fls_failed",
1399e948693eSPhilip Paeps 	"driver_rx_dsc_error",
1400e948693eSPhilip Paeps 	"driver_tx_dsc_error",
1401e948693eSPhilip Paeps 	"drv_gen",
1402e948693eSPhilip Paeps 	"mcdi_response",
1403e948693eSPhilip Paeps };
1404e948693eSPhilip Paeps /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1405e948693eSPhilip Paeps 
1406*3c838a9fSAndrew Rybchenko 		const char *
1407e948693eSPhilip Paeps efx_ev_qstat_name(
1408e948693eSPhilip Paeps 	__in	efx_nic_t *enp,
1409e948693eSPhilip Paeps 	__in	unsigned int id)
1410e948693eSPhilip Paeps {
1411e948693eSPhilip Paeps 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1412e948693eSPhilip Paeps 	EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1413e948693eSPhilip Paeps 
1414e948693eSPhilip Paeps 	return (__efx_ev_qstat_name[id]);
1415e948693eSPhilip Paeps }
1416e948693eSPhilip Paeps #endif	/* EFSYS_OPT_NAMES */
1417fe900081SAndrew Rybchenko #endif	/* EFSYS_OPT_QSTATS */
1418e948693eSPhilip Paeps 
1419*3c838a9fSAndrew Rybchenko #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
1420*3c838a9fSAndrew Rybchenko 
1421e948693eSPhilip Paeps #if EFSYS_OPT_QSTATS
1422*3c838a9fSAndrew Rybchenko static					void
1423*3c838a9fSAndrew Rybchenko falconsiena_ev_qstats_update(
1424e948693eSPhilip Paeps 	__in				efx_evq_t *eep,
1425e948693eSPhilip Paeps 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
1426e948693eSPhilip Paeps {
1427e948693eSPhilip Paeps 	unsigned int id;
1428e948693eSPhilip Paeps 
1429e948693eSPhilip Paeps 	for (id = 0; id < EV_NQSTATS; id++) {
1430e948693eSPhilip Paeps 		efsys_stat_t *essp = &stat[id];
1431e948693eSPhilip Paeps 
1432e948693eSPhilip Paeps 		EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1433e948693eSPhilip Paeps 		eep->ee_stat[id] = 0;
1434e948693eSPhilip Paeps 	}
1435e948693eSPhilip Paeps }
1436e948693eSPhilip Paeps #endif	/* EFSYS_OPT_QSTATS */
1437e948693eSPhilip Paeps 
1438*3c838a9fSAndrew Rybchenko static		void
1439*3c838a9fSAndrew Rybchenko falconsiena_ev_qdestroy(
1440e948693eSPhilip Paeps 	__in	efx_evq_t *eep)
1441e948693eSPhilip Paeps {
1442e948693eSPhilip Paeps 	efx_nic_t *enp = eep->ee_enp;
1443e948693eSPhilip Paeps 	efx_oword_t oword;
1444e948693eSPhilip Paeps 
1445e948693eSPhilip Paeps 	/* Purge event queue */
1446e948693eSPhilip Paeps 	EFX_ZERO_OWORD(oword);
1447e948693eSPhilip Paeps 
1448e948693eSPhilip Paeps 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1449*3c838a9fSAndrew Rybchenko 	    eep->ee_index, &oword, B_TRUE);
1450e948693eSPhilip Paeps 
1451e948693eSPhilip Paeps 	if (enp->en_family != EFX_FAMILY_FALCON) {
1452e948693eSPhilip Paeps 		EFX_ZERO_OWORD(oword);
1453e948693eSPhilip Paeps 		EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL,
1454*3c838a9fSAndrew Rybchenko 		    eep->ee_index, &oword, B_TRUE);
1455*3c838a9fSAndrew Rybchenko 	}
1456e948693eSPhilip Paeps }
1457e948693eSPhilip Paeps 
1458*3c838a9fSAndrew Rybchenko static		void
1459*3c838a9fSAndrew Rybchenko falconsiena_ev_fini(
1460e948693eSPhilip Paeps 	__in	efx_nic_t *enp)
1461e948693eSPhilip Paeps {
1462*3c838a9fSAndrew Rybchenko 	_NOTE(ARGUNUSED(enp))
1463e948693eSPhilip Paeps }
1464*3c838a9fSAndrew Rybchenko 
1465*3c838a9fSAndrew Rybchenko #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1466