xref: /freebsd/sys/dev/sfxge/common/ef10_mac.c (revision f5ef5f675d9d9eb6e35ed9142d70ecf774456ad4)
1 /*-
2  * Copyright (c) 2012-2016 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  * 1. Redistributions of source code must retain the above copyright notice,
9  *    this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright notice,
11  *    this list of conditions and the following disclaimer in the documentation
12  *    and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are
27  * those of the authors and should not be interpreted as representing official
28  * policies, either expressed or implied, of the FreeBSD Project.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "efx.h"
35 #include "efx_impl.h"
36 
37 
38 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
39 
40 	__checkReturn	efx_rc_t
41 ef10_mac_poll(
42 	__in		efx_nic_t *enp,
43 	__out		efx_link_mode_t *link_modep)
44 {
45 	efx_port_t *epp = &(enp->en_port);
46 	ef10_link_state_t els;
47 	efx_rc_t rc;
48 
49 	if ((rc = ef10_phy_get_link(enp, &els)) != 0)
50 		goto fail1;
51 
52 	epp->ep_adv_cap_mask = els.epls.epls_adv_cap_mask;
53 	epp->ep_fcntl = els.epls.epls_fcntl;
54 
55 	*link_modep = els.epls.epls_link_mode;
56 
57 	return (0);
58 
59 fail1:
60 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
61 
62 	*link_modep = EFX_LINK_UNKNOWN;
63 
64 	return (rc);
65 }
66 
67 	__checkReturn	efx_rc_t
68 ef10_mac_up(
69 	__in		efx_nic_t *enp,
70 	__out		boolean_t *mac_upp)
71 {
72 	ef10_link_state_t els;
73 	efx_rc_t rc;
74 
75 	/*
76 	 * Because EF10 doesn't *require* polling, we can't rely on
77 	 * ef10_mac_poll() being executed to populate epp->ep_mac_up.
78 	 */
79 	if ((rc = ef10_phy_get_link(enp, &els)) != 0)
80 		goto fail1;
81 
82 	*mac_upp = els.els_mac_up;
83 
84 	return (0);
85 
86 fail1:
87 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
88 
89 	return (rc);
90 }
91 
92 /*
93  * EF10 adapters use MC_CMD_VADAPTOR_SET_MAC to set the
94  * MAC address; the address field in MC_CMD_SET_MAC has no
95  * effect.
96  * MC_CMD_VADAPTOR_SET_MAC requires mac-spoofing privilege and
97  * the port to have no filters or queues active.
98  */
99 static	__checkReturn	efx_rc_t
100 efx_mcdi_vadapter_set_mac(
101 	__in		efx_nic_t *enp)
102 {
103 	efx_port_t *epp = &(enp->en_port);
104 	efx_mcdi_req_t req;
105 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_VADAPTOR_SET_MAC_IN_LEN,
106 		MC_CMD_VADAPTOR_SET_MAC_OUT_LEN);
107 	efx_rc_t rc;
108 
109 	req.emr_cmd = MC_CMD_VADAPTOR_SET_MAC;
110 	req.emr_in_buf = payload;
111 	req.emr_in_length = MC_CMD_VADAPTOR_SET_MAC_IN_LEN;
112 	req.emr_out_buf = payload;
113 	req.emr_out_length = MC_CMD_VADAPTOR_SET_MAC_OUT_LEN;
114 
115 	MCDI_IN_SET_DWORD(req, VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID,
116 	    enp->en_vport_id);
117 	EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, VADAPTOR_SET_MAC_IN_MACADDR),
118 	    epp->ep_mac_addr);
119 
120 	efx_mcdi_execute(enp, &req);
121 
122 	if (req.emr_rc != 0) {
123 		rc = req.emr_rc;
124 		goto fail1;
125 	}
126 
127 	return (0);
128 
129 fail1:
130 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
131 
132 	return (rc);
133 }
134 
135 	__checkReturn	efx_rc_t
136 ef10_mac_addr_set(
137 	__in		efx_nic_t *enp)
138 {
139 	efx_rc_t rc;
140 
141 	if ((rc = efx_mcdi_vadapter_set_mac(enp)) != 0) {
142 		if (rc != ENOTSUP)
143 			goto fail1;
144 
145 		/*
146 		 * Fallback for older Huntington firmware without Vadapter
147 		 * support.
148 		 */
149 		if ((rc = ef10_mac_reconfigure(enp)) != 0)
150 			goto fail2;
151 	}
152 
153 	return (0);
154 
155 fail2:
156 	EFSYS_PROBE(fail2);
157 
158 fail1:
159 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
160 
161 	return (rc);
162 }
163 
164 static	__checkReturn	efx_rc_t
165 efx_mcdi_mtu_set(
166 	__in		efx_nic_t *enp,
167 	__in		uint32_t mtu)
168 {
169 	efx_mcdi_req_t req;
170 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_EXT_IN_LEN,
171 		MC_CMD_SET_MAC_OUT_LEN);
172 	efx_rc_t rc;
173 
174 	req.emr_cmd = MC_CMD_SET_MAC;
175 	req.emr_in_buf = payload;
176 	req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN;
177 	req.emr_out_buf = payload;
178 	req.emr_out_length = MC_CMD_SET_MAC_OUT_LEN;
179 
180 	/* Only configure the MTU in this call to MC_CMD_SET_MAC */
181 	MCDI_IN_SET_DWORD(req, SET_MAC_EXT_IN_MTU, mtu);
182 	MCDI_IN_POPULATE_DWORD_1(req, SET_MAC_EXT_IN_CONTROL,
183 			    SET_MAC_EXT_IN_CFG_MTU, 1);
184 
185 	efx_mcdi_execute(enp, &req);
186 
187 	if (req.emr_rc != 0) {
188 		rc = req.emr_rc;
189 		goto fail1;
190 	}
191 
192 	return (0);
193 
194 fail1:
195 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
196 
197 	return (rc);
198 }
199 
200 static	__checkReturn		efx_rc_t
201 efx_mcdi_mtu_get(
202 	__in		efx_nic_t *enp,
203 	__out		size_t *mtu)
204 {
205 	efx_mcdi_req_t req;
206 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_EXT_IN_LEN,
207 		MC_CMD_SET_MAC_V2_OUT_LEN);
208 	efx_rc_t rc;
209 
210 	req.emr_cmd = MC_CMD_SET_MAC;
211 	req.emr_in_buf = payload;
212 	req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN;
213 	req.emr_out_buf = payload;
214 	req.emr_out_length = MC_CMD_SET_MAC_V2_OUT_LEN;
215 
216 	/*
217 	 * With MC_CMD_SET_MAC_EXT_IN_CONTROL set to 0, this just queries the
218 	 * MTU.  This should always be supported on Medford, but it is not
219 	 * supported on older Huntington firmware.
220 	 */
221 	MCDI_IN_SET_DWORD(req, SET_MAC_EXT_IN_CONTROL, 0);
222 
223 	efx_mcdi_execute(enp, &req);
224 
225 	if (req.emr_rc != 0) {
226 		rc = req.emr_rc;
227 		goto fail1;
228 	}
229 	if (req.emr_out_length_used < MC_CMD_SET_MAC_V2_OUT_MTU_OFST + 4) {
230 		rc = EMSGSIZE;
231 		goto fail2;
232 	}
233 
234 	*mtu = MCDI_OUT_DWORD(req, SET_MAC_V2_OUT_MTU);
235 
236 	return (0);
237 
238 fail2:
239 	EFSYS_PROBE(fail2);
240 fail1:
241 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
242 
243 	return (rc);
244 }
245 
246 	__checkReturn	efx_rc_t
247 ef10_mac_pdu_set(
248 	__in		efx_nic_t *enp)
249 {
250 	efx_port_t *epp = &(enp->en_port);
251 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
252 	efx_rc_t rc;
253 
254 	if (encp->enc_enhanced_set_mac_supported) {
255 		if ((rc = efx_mcdi_mtu_set(enp, epp->ep_mac_pdu)) != 0)
256 			goto fail1;
257 	} else {
258 		/*
259 		 * Fallback for older Huntington firmware, which always
260 		 * configure all of the parameters to MC_CMD_SET_MAC. This isn't
261 		 * suitable for setting the MTU on unpriviliged functions.
262 		 */
263 		if ((rc = ef10_mac_reconfigure(enp)) != 0)
264 			goto fail2;
265 	}
266 
267 	return (0);
268 
269 fail2:
270 	EFSYS_PROBE(fail2);
271 fail1:
272 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
273 
274 	return (rc);
275 }
276 
277 	__checkReturn		efx_rc_t
278 ef10_mac_pdu_get(
279 	__in		efx_nic_t *enp,
280 	__out		size_t *pdu)
281 {
282 	efx_rc_t rc;
283 
284 	if ((rc = efx_mcdi_mtu_get(enp, pdu)) != 0)
285 		goto fail1;
286 
287 	return (0);
288 
289 fail1:
290 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
291 
292 	return (rc);
293 }
294 
295 __checkReturn	efx_rc_t
296 ef10_mac_reconfigure(
297 	__in		efx_nic_t *enp)
298 {
299 	efx_port_t *epp = &(enp->en_port);
300 	efx_mcdi_req_t req;
301 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_IN_LEN,
302 		MC_CMD_SET_MAC_OUT_LEN);
303 	efx_rc_t rc;
304 
305 	req.emr_cmd = MC_CMD_SET_MAC;
306 	req.emr_in_buf = payload;
307 	req.emr_in_length = MC_CMD_SET_MAC_IN_LEN;
308 	req.emr_out_buf = payload;
309 	req.emr_out_length = MC_CMD_SET_MAC_OUT_LEN;
310 
311 	MCDI_IN_SET_DWORD(req, SET_MAC_IN_MTU, epp->ep_mac_pdu);
312 	MCDI_IN_SET_DWORD(req, SET_MAC_IN_DRAIN, epp->ep_mac_drain ? 1 : 0);
313 	EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, SET_MAC_IN_ADDR),
314 			    epp->ep_mac_addr);
315 
316 	/*
317 	 * Note: The Huntington MAC does not support REJECT_BRDCST.
318 	 * The REJECT_UNCST flag will also prevent multicast traffic
319 	 * from reaching the filters. As Huntington filters drop any
320 	 * traffic that does not match a filter it is ok to leave the
321 	 * MAC running in promiscuous mode. See bug41141.
322 	 *
323 	 * FIXME: Does REJECT_UNCST behave the same way on Medford?
324 	 */
325 	MCDI_IN_POPULATE_DWORD_2(req, SET_MAC_IN_REJECT,
326 				    SET_MAC_IN_REJECT_UNCST, 0,
327 				    SET_MAC_IN_REJECT_BRDCST, 0);
328 
329 	/*
330 	 * Flow control, whether it is auto-negotiated or not,
331 	 * is set via the PHY advertised capabilities.  When set to
332 	 * automatic the MAC will use the PHY settings to determine
333 	 * the flow control settings.
334 	 */
335 	MCDI_IN_SET_DWORD(req, SET_MAC_IN_FCNTL, MC_CMD_FCNTL_AUTO);
336 
337 	/* Do not include the Ethernet frame checksum in RX packets */
338 	MCDI_IN_POPULATE_DWORD_1(req, SET_MAC_IN_FLAGS,
339 				    SET_MAC_IN_FLAG_INCLUDE_FCS, 0);
340 
341 	efx_mcdi_execute_quiet(enp, &req);
342 
343 	if (req.emr_rc != 0) {
344 		/*
345 		 * Unprivileged functions cannot control link state,
346 		 * but still need to configure filters.
347 		 */
348 		if (req.emr_rc != EACCES) {
349 			rc = req.emr_rc;
350 			goto fail1;
351 		}
352 	}
353 
354 	/*
355 	 * Apply the filters for the MAC configuration.
356 	 * If the NIC isn't ready to accept filters this may
357 	 * return success without setting anything.
358 	 */
359 	rc = efx_filter_reconfigure(enp, epp->ep_mac_addr,
360 				    epp->ep_all_unicst, epp->ep_mulcst,
361 				    epp->ep_all_mulcst, epp->ep_brdcst,
362 				    epp->ep_mulcst_addr_list,
363 				    epp->ep_mulcst_addr_count);
364 
365 	return (0);
366 
367 fail1:
368 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
369 
370 	return (rc);
371 }
372 
373 	__checkReturn			efx_rc_t
374 ef10_mac_multicast_list_set(
375 	__in				efx_nic_t *enp)
376 {
377 	efx_port_t *epp = &(enp->en_port);
378 	const efx_mac_ops_t *emop = epp->ep_emop;
379 	efx_rc_t rc;
380 
381 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
382 	    enp->en_family == EFX_FAMILY_MEDFORD ||
383 	    enp->en_family == EFX_FAMILY_MEDFORD2);
384 
385 	if ((rc = emop->emo_reconfigure(enp)) != 0)
386 		goto fail1;
387 
388 	return (0);
389 
390 fail1:
391 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
392 
393 	return (rc);
394 }
395 
396 	__checkReturn	efx_rc_t
397 ef10_mac_filter_default_rxq_set(
398 	__in		efx_nic_t *enp,
399 	__in		efx_rxq_t *erp,
400 	__in		boolean_t using_rss)
401 {
402 	efx_port_t *epp = &(enp->en_port);
403 	efx_rxq_t *old_rxq;
404 	boolean_t old_using_rss;
405 	efx_rc_t rc;
406 
407 	ef10_filter_get_default_rxq(enp, &old_rxq, &old_using_rss);
408 
409 	ef10_filter_default_rxq_set(enp, erp, using_rss);
410 
411 	rc = efx_filter_reconfigure(enp, epp->ep_mac_addr,
412 				    epp->ep_all_unicst, epp->ep_mulcst,
413 				    epp->ep_all_mulcst, epp->ep_brdcst,
414 				    epp->ep_mulcst_addr_list,
415 				    epp->ep_mulcst_addr_count);
416 
417 	if (rc != 0)
418 		goto fail1;
419 
420 	return (0);
421 
422 fail1:
423 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
424 
425 	ef10_filter_default_rxq_set(enp, old_rxq, old_using_rss);
426 
427 	return (rc);
428 }
429 
430 			void
431 ef10_mac_filter_default_rxq_clear(
432 	__in		efx_nic_t *enp)
433 {
434 	efx_port_t *epp = &(enp->en_port);
435 
436 	ef10_filter_default_rxq_clear(enp);
437 
438 	(void) efx_filter_reconfigure(enp, epp->ep_mac_addr,
439 				    epp->ep_all_unicst, epp->ep_mulcst,
440 				    epp->ep_all_mulcst, epp->ep_brdcst,
441 				    epp->ep_mulcst_addr_list,
442 				    epp->ep_mulcst_addr_count);
443 }
444 
445 
446 #if EFSYS_OPT_LOOPBACK
447 
448 	__checkReturn	efx_rc_t
449 ef10_mac_loopback_set(
450 	__in		efx_nic_t *enp,
451 	__in		efx_link_mode_t link_mode,
452 	__in		efx_loopback_type_t loopback_type)
453 {
454 	efx_port_t *epp = &(enp->en_port);
455 	const efx_phy_ops_t *epop = epp->ep_epop;
456 	efx_loopback_type_t old_loopback_type;
457 	efx_link_mode_t old_loopback_link_mode;
458 	efx_rc_t rc;
459 
460 	/* The PHY object handles this on EF10 */
461 	old_loopback_type = epp->ep_loopback_type;
462 	old_loopback_link_mode = epp->ep_loopback_link_mode;
463 	epp->ep_loopback_type = loopback_type;
464 	epp->ep_loopback_link_mode = link_mode;
465 
466 	if ((rc = epop->epo_reconfigure(enp)) != 0)
467 		goto fail1;
468 
469 	return (0);
470 
471 fail1:
472 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
473 
474 	epp->ep_loopback_type = old_loopback_type;
475 	epp->ep_loopback_link_mode = old_loopback_link_mode;
476 
477 	return (rc);
478 }
479 
480 #endif	/* EFSYS_OPT_LOOPBACK */
481 
482 #if EFSYS_OPT_MAC_STATS
483 
484 	__checkReturn			efx_rc_t
485 ef10_mac_stats_get_mask(
486 	__in				efx_nic_t *enp,
487 	__inout_bcount(mask_size)	uint32_t *maskp,
488 	__in				size_t mask_size)
489 {
490 	const struct efx_mac_stats_range ef10_common[] = {
491 		{ EFX_MAC_RX_OCTETS, EFX_MAC_RX_GE_15XX_PKTS },
492 		{ EFX_MAC_RX_FCS_ERRORS, EFX_MAC_RX_DROP_EVENTS },
493 		{ EFX_MAC_RX_JABBER_PKTS, EFX_MAC_RX_JABBER_PKTS },
494 		{ EFX_MAC_RX_NODESC_DROP_CNT, EFX_MAC_TX_PAUSE_PKTS },
495 	};
496 	const struct efx_mac_stats_range ef10_tx_size_bins[] = {
497 		{ EFX_MAC_TX_LE_64_PKTS, EFX_MAC_TX_GE_15XX_PKTS },
498 	};
499 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
500 	efx_port_t *epp = &(enp->en_port);
501 	efx_rc_t rc;
502 
503 	if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
504 	    ef10_common, EFX_ARRAY_SIZE(ef10_common))) != 0)
505 		goto fail1;
506 
507 	if (epp->ep_phy_cap_mask & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) {
508 		const struct efx_mac_stats_range ef10_40g_extra[] = {
509 			{ EFX_MAC_RX_ALIGN_ERRORS, EFX_MAC_RX_ALIGN_ERRORS },
510 		};
511 
512 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
513 		    ef10_40g_extra, EFX_ARRAY_SIZE(ef10_40g_extra))) != 0)
514 			goto fail2;
515 
516 		if (encp->enc_mac_stats_40g_tx_size_bins) {
517 			if ((rc = efx_mac_stats_mask_add_ranges(maskp,
518 			    mask_size, ef10_tx_size_bins,
519 			    EFX_ARRAY_SIZE(ef10_tx_size_bins))) != 0)
520 				goto fail3;
521 		}
522 	} else {
523 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
524 		    ef10_tx_size_bins, EFX_ARRAY_SIZE(ef10_tx_size_bins))) != 0)
525 			goto fail4;
526 	}
527 
528 	if (encp->enc_pm_and_rxdp_counters) {
529 		const struct efx_mac_stats_range ef10_pm_and_rxdp[] = {
530 			{ EFX_MAC_PM_TRUNC_BB_OVERFLOW, EFX_MAC_RXDP_HLB_WAIT },
531 		};
532 
533 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
534 		    ef10_pm_and_rxdp, EFX_ARRAY_SIZE(ef10_pm_and_rxdp))) != 0)
535 			goto fail5;
536 	}
537 
538 	if (encp->enc_datapath_cap_evb) {
539 		const struct efx_mac_stats_range ef10_vadaptor[] = {
540 			{ EFX_MAC_VADAPTER_RX_UNICAST_PACKETS,
541 			    EFX_MAC_VADAPTER_TX_OVERFLOW },
542 		};
543 
544 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
545 		    ef10_vadaptor, EFX_ARRAY_SIZE(ef10_vadaptor))) != 0)
546 			goto fail6;
547 	}
548 
549 	if (encp->enc_fec_counters) {
550 		const struct efx_mac_stats_range ef10_fec[] = {
551 			{ EFX_MAC_FEC_UNCORRECTED_ERRORS,
552 			    EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE3 },
553 		};
554 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
555 		    ef10_fec, EFX_ARRAY_SIZE(ef10_fec))) != 0)
556 			goto fail7;
557 	}
558 
559 	if (encp->enc_mac_stats_nstats >= MC_CMD_MAC_NSTATS_V4) {
560 		const struct efx_mac_stats_range ef10_rxdp_sdt[] = {
561 			{ EFX_MAC_RXDP_SCATTER_DISABLED_TRUNC,
562 			    EFX_MAC_RXDP_SCATTER_DISABLED_TRUNC },
563 		};
564 
565 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
566 		    ef10_rxdp_sdt, EFX_ARRAY_SIZE(ef10_rxdp_sdt))) != 0)
567 			goto fail8;
568 	}
569 
570 	if (encp->enc_hlb_counters) {
571 		const struct efx_mac_stats_range ef10_hlb[] = {
572 			{ EFX_MAC_RXDP_HLB_IDLE, EFX_MAC_RXDP_HLB_TIMEOUT },
573 		};
574 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
575 		    ef10_hlb, EFX_ARRAY_SIZE(ef10_hlb))) != 0)
576 			goto fail9;
577 	}
578 
579 	return (0);
580 
581 fail9:
582 	EFSYS_PROBE(fail9);
583 fail8:
584 	EFSYS_PROBE(fail8);
585 fail7:
586 	EFSYS_PROBE(fail7);
587 fail6:
588 	EFSYS_PROBE(fail6);
589 fail5:
590 	EFSYS_PROBE(fail5);
591 fail4:
592 	EFSYS_PROBE(fail4);
593 fail3:
594 	EFSYS_PROBE(fail3);
595 fail2:
596 	EFSYS_PROBE(fail2);
597 fail1:
598 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
599 
600 	return (rc);
601 }
602 
603 #define	EF10_MAC_STAT_READ(_esmp, _field, _eqp)			\
604 	EFSYS_MEM_READQ((_esmp), (_field) * sizeof (efx_qword_t), _eqp)
605 
606 
607 	__checkReturn			efx_rc_t
608 ef10_mac_stats_update(
609 	__in				efx_nic_t *enp,
610 	__in				efsys_mem_t *esmp,
611 	__inout_ecount(EFX_MAC_NSTATS)	efsys_stat_t *stat,
612 	__inout_opt			uint32_t *generationp)
613 {
614 	const efx_nic_cfg_t *encp = &enp->en_nic_cfg;
615 	efx_qword_t generation_start;
616 	efx_qword_t generation_end;
617 	efx_qword_t value;
618 	efx_rc_t rc;
619 
620 	/*
621 	 * The MAC_STATS contain start and end generation counters used to
622 	 * detect when the DMA buffer has been updated during stats decode.
623 	 * All stats counters are 64bit unsigned values.
624 	 *
625 	 * Siena-compatible MAC stats contain MC_CMD_MAC_NSTATS 64bit counters.
626 	 * The generation end counter is at index MC_CMD_MAC_GENERATION_END
627 	 * (same as MC_CMD_MAC_NSTATS-1).
628 	 *
629 	 * Medford2 and later use a larger DMA buffer: MAC_STATS_NUM_STATS from
630 	 * MC_CMD_GET_CAPABILITIES_V4_OUT reports the number of 64bit counters.
631 	 *
632 	 * Firmware writes the generation end counter as the last counter in the
633 	 * DMA buffer. Do not use MC_CMD_MAC_GENERATION_END, as that is only
634 	 * correct for legacy Siena-compatible MAC stats.
635 	 */
636 
637 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS) {
638 		/* MAC stats count too small for legacy MAC stats */
639 		rc = ENOSPC;
640 		goto fail1;
641 	}
642 	if (EFSYS_MEM_SIZE(esmp) <
643 	    (encp->enc_mac_stats_nstats * sizeof (efx_qword_t))) {
644 		/* DMA buffer too small */
645 		rc = ENOSPC;
646 		goto fail2;
647 	}
648 
649 	/* Read END first so we don't race with the MC */
650 	EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFSYS_MEM_SIZE(esmp));
651 	EF10_MAC_STAT_READ(esmp, (encp->enc_mac_stats_nstats - 1),
652 	    &generation_end);
653 	EFSYS_MEM_READ_BARRIER();
654 
655 	/* TX */
656 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PKTS, &value);
657 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value);
658 
659 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_CONTROL_PKTS, &value);
660 	EFSYS_STAT_SUBR_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value);
661 
662 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PAUSE_PKTS, &value);
663 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PAUSE_PKTS]), &value);
664 
665 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_UNICAST_PKTS, &value);
666 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_UNICST_PKTS]), &value);
667 
668 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTICAST_PKTS, &value);
669 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULTICST_PKTS]), &value);
670 
671 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BROADCAST_PKTS, &value);
672 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_BRDCST_PKTS]), &value);
673 
674 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BYTES, &value);
675 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_OCTETS]), &value);
676 
677 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LT64_PKTS, &value);
678 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
679 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_64_PKTS, &value);
680 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
681 
682 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_65_TO_127_PKTS, &value);
683 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_65_TO_127_PKTS]), &value);
684 
685 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_128_TO_255_PKTS, &value);
686 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_128_TO_255_PKTS]), &value);
687 
688 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_256_TO_511_PKTS, &value);
689 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_256_TO_511_PKTS]), &value);
690 
691 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_512_TO_1023_PKTS, &value);
692 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_512_TO_1023_PKTS]), &value);
693 
694 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_1024_TO_15XX_PKTS, &value);
695 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_1024_TO_15XX_PKTS]), &value);
696 
697 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_15XX_TO_JUMBO_PKTS, &value);
698 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value);
699 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_GTJUMBO_PKTS, &value);
700 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value);
701 
702 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BAD_FCS_PKTS, &value);
703 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_ERRORS]), &value);
704 
705 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_SINGLE_COLLISION_PKTS, &value);
706 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_SGL_COL_PKTS]), &value);
707 
708 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTIPLE_COLLISION_PKTS,
709 			    &value);
710 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULT_COL_PKTS]), &value);
711 
712 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_COLLISION_PKTS,
713 			    &value);
714 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_COL_PKTS]), &value);
715 
716 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LATE_COLLISION_PKTS, &value);
717 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LATE_COL_PKTS]), &value);
718 
719 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_DEFERRED_PKTS, &value);
720 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_DEF_PKTS]), &value);
721 
722 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_DEFERRED_PKTS,
723 	    &value);
724 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_DEF_PKTS]), &value);
725 
726 	/* RX */
727 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BYTES, &value);
728 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_OCTETS]), &value);
729 
730 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PKTS, &value);
731 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PKTS]), &value);
732 
733 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNICAST_PKTS, &value);
734 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_UNICST_PKTS]), &value);
735 
736 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MULTICAST_PKTS, &value);
737 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MULTICST_PKTS]), &value);
738 
739 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BROADCAST_PKTS, &value);
740 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_BRDCST_PKTS]), &value);
741 
742 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PAUSE_PKTS, &value);
743 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PAUSE_PKTS]), &value);
744 
745 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNDERSIZE_PKTS, &value);
746 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value);
747 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_64_PKTS, &value);
748 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value);
749 
750 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_65_TO_127_PKTS, &value);
751 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_65_TO_127_PKTS]), &value);
752 
753 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_128_TO_255_PKTS, &value);
754 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_128_TO_255_PKTS]), &value);
755 
756 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_256_TO_511_PKTS, &value);
757 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_256_TO_511_PKTS]), &value);
758 
759 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_512_TO_1023_PKTS, &value);
760 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_512_TO_1023_PKTS]), &value);
761 
762 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_1024_TO_15XX_PKTS, &value);
763 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_1024_TO_15XX_PKTS]), &value);
764 
765 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_15XX_TO_JUMBO_PKTS, &value);
766 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value);
767 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_GTJUMBO_PKTS, &value);
768 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value);
769 
770 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BAD_FCS_PKTS, &value);
771 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FCS_ERRORS]), &value);
772 
773 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_OVERFLOW_PKTS, &value);
774 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_DROP_EVENTS]), &value);
775 
776 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_FALSE_CARRIER_PKTS, &value);
777 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FALSE_CARRIER_ERRORS]), &value);
778 
779 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_SYMBOL_ERROR_PKTS, &value);
780 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_SYMBOL_ERRORS]), &value);
781 
782 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_ALIGN_ERROR_PKTS, &value);
783 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_ALIGN_ERRORS]), &value);
784 
785 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_INTERNAL_ERROR_PKTS, &value);
786 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_INTERNAL_ERRORS]), &value);
787 
788 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_JABBER_PKTS, &value);
789 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_JABBER_PKTS]), &value);
790 
791 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_CHAR_ERR, &value);
792 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_CHAR_ERR]),
793 			    &(value.eq_dword[0]));
794 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_CHAR_ERR]),
795 			    &(value.eq_dword[1]));
796 
797 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_CHAR_ERR, &value);
798 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_CHAR_ERR]),
799 			    &(value.eq_dword[0]));
800 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_CHAR_ERR]),
801 			    &(value.eq_dword[1]));
802 
803 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_DISP_ERR, &value);
804 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_DISP_ERR]),
805 			    &(value.eq_dword[0]));
806 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_DISP_ERR]),
807 			    &(value.eq_dword[1]));
808 
809 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_DISP_ERR, &value);
810 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_DISP_ERR]),
811 			    &(value.eq_dword[0]));
812 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_DISP_ERR]),
813 			    &(value.eq_dword[1]));
814 
815 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MATCH_FAULT, &value);
816 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MATCH_FAULT]), &value);
817 
818 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_NODESC_DROPS, &value);
819 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_NODESC_DROP_CNT]), &value);
820 
821 	/* Packet memory (EF10 only) */
822 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_BB_OVERFLOW, &value);
823 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_BB_OVERFLOW]), &value);
824 
825 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_BB_OVERFLOW, &value);
826 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_BB_OVERFLOW]), &value);
827 
828 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_VFIFO_FULL, &value);
829 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_VFIFO_FULL]), &value);
830 
831 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_VFIFO_FULL, &value);
832 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_VFIFO_FULL]), &value);
833 
834 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_QBB, &value);
835 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_QBB]), &value);
836 
837 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_QBB, &value);
838 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_QBB]), &value);
839 
840 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_MAPPING, &value);
841 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_MAPPING]), &value);
842 
843 	/* RX datapath */
844 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_Q_DISABLED_PKTS, &value);
845 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_Q_DISABLED_PKTS]), &value);
846 
847 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_DI_DROPPED_PKTS, &value);
848 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_DI_DROPPED_PKTS]), &value);
849 
850 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_STREAMING_PKTS, &value);
851 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_STREAMING_PKTS]), &value);
852 
853 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_FETCH_CONDITIONS, &value);
854 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_FETCH]), &value);
855 
856 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_WAIT_CONDITIONS, &value);
857 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_WAIT]), &value);
858 
859 
860 	/* VADAPTER RX */
861 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_UNICAST_PACKETS,
862 	    &value);
863 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS]),
864 	    &value);
865 
866 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_UNICAST_BYTES,
867 	    &value);
868 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_UNICAST_BYTES]),
869 	    &value);
870 
871 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_MULTICAST_PACKETS,
872 	    &value);
873 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS]),
874 	    &value);
875 
876 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_MULTICAST_BYTES,
877 	    &value);
878 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES]),
879 	    &value);
880 
881 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BROADCAST_PACKETS,
882 	    &value);
883 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS]),
884 	    &value);
885 
886 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BROADCAST_BYTES,
887 	    &value);
888 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES]),
889 	    &value);
890 
891 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BAD_PACKETS,
892 	    &value);
893 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BAD_PACKETS]),
894 	    &value);
895 
896 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BAD_BYTES, &value);
897 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BAD_BYTES]), &value);
898 
899 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_OVERFLOW, &value);
900 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_OVERFLOW]), &value);
901 
902 	/* VADAPTER TX */
903 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_UNICAST_PACKETS,
904 	    &value);
905 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS]),
906 	    &value);
907 
908 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_UNICAST_BYTES,
909 	    &value);
910 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_UNICAST_BYTES]),
911 	    &value);
912 
913 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_MULTICAST_PACKETS,
914 	    &value);
915 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS]),
916 	    &value);
917 
918 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_MULTICAST_BYTES,
919 	    &value);
920 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES]),
921 	    &value);
922 
923 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BROADCAST_PACKETS,
924 	    &value);
925 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS]),
926 	    &value);
927 
928 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BROADCAST_BYTES,
929 	    &value);
930 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES]),
931 	    &value);
932 
933 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BAD_PACKETS, &value);
934 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BAD_PACKETS]), &value);
935 
936 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BAD_BYTES, &value);
937 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BAD_BYTES]), &value);
938 
939 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_OVERFLOW, &value);
940 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_OVERFLOW]), &value);
941 
942 
943 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS_V2)
944 		goto done;
945 
946 	/* FEC */
947 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_UNCORRECTED_ERRORS, &value);
948 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_UNCORRECTED_ERRORS]), &value);
949 
950 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_ERRORS, &value);
951 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_ERRORS]), &value);
952 
953 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE0,
954 	    &value);
955 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE0]),
956 	    &value);
957 
958 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE1,
959 	    &value);
960 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE1]),
961 	    &value);
962 
963 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE2,
964 	    &value);
965 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE2]),
966 	    &value);
967 
968 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE3,
969 	    &value);
970 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE3]),
971 	    &value);
972 
973 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS_V3)
974 		goto done;
975 
976 	/* CTPIO exceptions */
977 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_VI_BUSY_FALLBACK, &value);
978 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_VI_BUSY_FALLBACK]), &value);
979 
980 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_LONG_WRITE_SUCCESS, &value);
981 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_LONG_WRITE_SUCCESS]), &value);
982 
983 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_MISSING_DBELL_FAIL, &value);
984 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_MISSING_DBELL_FAIL]), &value);
985 
986 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_OVERFLOW_FAIL, &value);
987 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_OVERFLOW_FAIL]), &value);
988 
989 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_UNDERFLOW_FAIL, &value);
990 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_UNDERFLOW_FAIL]), &value);
991 
992 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_TIMEOUT_FAIL, &value);
993 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_TIMEOUT_FAIL]), &value);
994 
995 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_NONCONTIG_WR_FAIL, &value);
996 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_NONCONTIG_WR_FAIL]), &value);
997 
998 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_FRM_CLOBBER_FAIL, &value);
999 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_FRM_CLOBBER_FAIL]), &value);
1000 
1001 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_INVALID_WR_FAIL, &value);
1002 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_INVALID_WR_FAIL]), &value);
1003 
1004 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_VI_CLOBBER_FALLBACK, &value);
1005 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_VI_CLOBBER_FALLBACK]),
1006 	    &value);
1007 
1008 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_UNQUALIFIED_FALLBACK, &value);
1009 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_UNQUALIFIED_FALLBACK]),
1010 	    &value);
1011 
1012 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_RUNT_FALLBACK, &value);
1013 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_RUNT_FALLBACK]), &value);
1014 
1015 	/* CTPIO per-port stats */
1016 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_SUCCESS, &value);
1017 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_SUCCESS]), &value);
1018 
1019 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_FALLBACK, &value);
1020 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_FALLBACK]), &value);
1021 
1022 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_POISON, &value);
1023 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_POISON]), &value);
1024 
1025 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_ERASE, &value);
1026 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_ERASE]), &value);
1027 
1028 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS_V4)
1029 		goto done;
1030 
1031 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_SCATTER_DISABLED_TRUNC,
1032 	    &value);
1033 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_SCATTER_DISABLED_TRUNC]),
1034 	    &value);
1035 
1036 	/* Head-of-line blocking */
1037 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_IDLE, &value);
1038 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_IDLE]), &value);
1039 
1040 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_TIMEOUT, &value);
1041 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_TIMEOUT]), &value);
1042 
1043 done:
1044 	/* Read START generation counter */
1045 	EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFSYS_MEM_SIZE(esmp));
1046 	EFSYS_MEM_READ_BARRIER();
1047 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_START,
1048 			    &generation_start);
1049 
1050 	/* Check that we didn't read the stats in the middle of a DMA */
1051 	/* Not a good enough check ? */
1052 	if (memcmp(&generation_start, &generation_end,
1053 	    sizeof (generation_start)))
1054 		return (EAGAIN);
1055 
1056 	if (generationp)
1057 		*generationp = EFX_QWORD_FIELD(generation_start, EFX_DWORD_0);
1058 
1059 	return (0);
1060 
1061 fail2:
1062 	EFSYS_PROBE(fail2);
1063 fail1:
1064 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1065 
1066 	return (rc);
1067 }
1068 
1069 #endif	/* EFSYS_OPT_MAC_STATS */
1070 
1071 #endif	/* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
1072