xref: /freebsd/sys/dev/sfxge/common/ef10_mac.c (revision d4eeb02986980bf33dd56c41ceb9fc5f180c0d47)
1 /*-
2  * Copyright (c) 2012-2016 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  * 1. Redistributions of source code must retain the above copyright notice,
9  *    this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright notice,
11  *    this list of conditions and the following disclaimer in the documentation
12  *    and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are
27  * those of the authors and should not be interpreted as representing official
28  * policies, either expressed or implied, of the FreeBSD Project.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "efx.h"
35 #include "efx_impl.h"
36 
37 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
38 
39 	__checkReturn	efx_rc_t
40 ef10_mac_poll(
41 	__in		efx_nic_t *enp,
42 	__out		efx_link_mode_t *link_modep)
43 {
44 	efx_port_t *epp = &(enp->en_port);
45 	ef10_link_state_t els;
46 	efx_rc_t rc;
47 
48 	if ((rc = ef10_phy_get_link(enp, &els)) != 0)
49 		goto fail1;
50 
51 	epp->ep_adv_cap_mask = els.epls.epls_adv_cap_mask;
52 	epp->ep_fcntl = els.epls.epls_fcntl;
53 
54 	*link_modep = els.epls.epls_link_mode;
55 
56 	return (0);
57 
58 fail1:
59 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
60 
61 	*link_modep = EFX_LINK_UNKNOWN;
62 
63 	return (rc);
64 }
65 
66 	__checkReturn	efx_rc_t
67 ef10_mac_up(
68 	__in		efx_nic_t *enp,
69 	__out		boolean_t *mac_upp)
70 {
71 	ef10_link_state_t els;
72 	efx_rc_t rc;
73 
74 	/*
75 	 * Because EF10 doesn't *require* polling, we can't rely on
76 	 * ef10_mac_poll() being executed to populate epp->ep_mac_up.
77 	 */
78 	if ((rc = ef10_phy_get_link(enp, &els)) != 0)
79 		goto fail1;
80 
81 	*mac_upp = els.els_mac_up;
82 
83 	return (0);
84 
85 fail1:
86 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
87 
88 	return (rc);
89 }
90 
91 /*
92  * EF10 adapters use MC_CMD_VADAPTOR_SET_MAC to set the
93  * MAC address; the address field in MC_CMD_SET_MAC has no
94  * effect.
95  * MC_CMD_VADAPTOR_SET_MAC requires mac-spoofing privilege and
96  * the port to have no filters or queues active.
97  */
98 static	__checkReturn	efx_rc_t
99 efx_mcdi_vadapter_set_mac(
100 	__in		efx_nic_t *enp)
101 {
102 	efx_port_t *epp = &(enp->en_port);
103 	efx_mcdi_req_t req;
104 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_VADAPTOR_SET_MAC_IN_LEN,
105 		MC_CMD_VADAPTOR_SET_MAC_OUT_LEN);
106 	efx_rc_t rc;
107 
108 	req.emr_cmd = MC_CMD_VADAPTOR_SET_MAC;
109 	req.emr_in_buf = payload;
110 	req.emr_in_length = MC_CMD_VADAPTOR_SET_MAC_IN_LEN;
111 	req.emr_out_buf = payload;
112 	req.emr_out_length = MC_CMD_VADAPTOR_SET_MAC_OUT_LEN;
113 
114 	MCDI_IN_SET_DWORD(req, VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID,
115 	    enp->en_vport_id);
116 	EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, VADAPTOR_SET_MAC_IN_MACADDR),
117 	    epp->ep_mac_addr);
118 
119 	efx_mcdi_execute(enp, &req);
120 
121 	if (req.emr_rc != 0) {
122 		rc = req.emr_rc;
123 		goto fail1;
124 	}
125 
126 	return (0);
127 
128 fail1:
129 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
130 
131 	return (rc);
132 }
133 
134 	__checkReturn	efx_rc_t
135 ef10_mac_addr_set(
136 	__in		efx_nic_t *enp)
137 {
138 	efx_rc_t rc;
139 
140 	if ((rc = efx_mcdi_vadapter_set_mac(enp)) != 0) {
141 		if (rc != ENOTSUP)
142 			goto fail1;
143 
144 		/*
145 		 * Fallback for older Huntington firmware without Vadapter
146 		 * support.
147 		 */
148 		if ((rc = ef10_mac_reconfigure(enp)) != 0)
149 			goto fail2;
150 	}
151 
152 	return (0);
153 
154 fail2:
155 	EFSYS_PROBE(fail2);
156 
157 fail1:
158 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
159 
160 	return (rc);
161 }
162 
163 static	__checkReturn	efx_rc_t
164 efx_mcdi_mtu_set(
165 	__in		efx_nic_t *enp,
166 	__in		uint32_t mtu)
167 {
168 	efx_mcdi_req_t req;
169 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_EXT_IN_LEN,
170 		MC_CMD_SET_MAC_OUT_LEN);
171 	efx_rc_t rc;
172 
173 	req.emr_cmd = MC_CMD_SET_MAC;
174 	req.emr_in_buf = payload;
175 	req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN;
176 	req.emr_out_buf = payload;
177 	req.emr_out_length = MC_CMD_SET_MAC_OUT_LEN;
178 
179 	/* Only configure the MTU in this call to MC_CMD_SET_MAC */
180 	MCDI_IN_SET_DWORD(req, SET_MAC_EXT_IN_MTU, mtu);
181 	MCDI_IN_POPULATE_DWORD_1(req, SET_MAC_EXT_IN_CONTROL,
182 			    SET_MAC_EXT_IN_CFG_MTU, 1);
183 
184 	efx_mcdi_execute(enp, &req);
185 
186 	if (req.emr_rc != 0) {
187 		rc = req.emr_rc;
188 		goto fail1;
189 	}
190 
191 	return (0);
192 
193 fail1:
194 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
195 
196 	return (rc);
197 }
198 
199 static	__checkReturn		efx_rc_t
200 efx_mcdi_mtu_get(
201 	__in		efx_nic_t *enp,
202 	__out		size_t *mtu)
203 {
204 	efx_mcdi_req_t req;
205 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_EXT_IN_LEN,
206 		MC_CMD_SET_MAC_V2_OUT_LEN);
207 	efx_rc_t rc;
208 
209 	req.emr_cmd = MC_CMD_SET_MAC;
210 	req.emr_in_buf = payload;
211 	req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN;
212 	req.emr_out_buf = payload;
213 	req.emr_out_length = MC_CMD_SET_MAC_V2_OUT_LEN;
214 
215 	/*
216 	 * With MC_CMD_SET_MAC_EXT_IN_CONTROL set to 0, this just queries the
217 	 * MTU.  This should always be supported on Medford, but it is not
218 	 * supported on older Huntington firmware.
219 	 */
220 	MCDI_IN_SET_DWORD(req, SET_MAC_EXT_IN_CONTROL, 0);
221 
222 	efx_mcdi_execute(enp, &req);
223 
224 	if (req.emr_rc != 0) {
225 		rc = req.emr_rc;
226 		goto fail1;
227 	}
228 	if (req.emr_out_length_used < MC_CMD_SET_MAC_V2_OUT_MTU_OFST + 4) {
229 		rc = EMSGSIZE;
230 		goto fail2;
231 	}
232 
233 	*mtu = MCDI_OUT_DWORD(req, SET_MAC_V2_OUT_MTU);
234 
235 	return (0);
236 
237 fail2:
238 	EFSYS_PROBE(fail2);
239 fail1:
240 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
241 
242 	return (rc);
243 }
244 
245 	__checkReturn	efx_rc_t
246 ef10_mac_pdu_set(
247 	__in		efx_nic_t *enp)
248 {
249 	efx_port_t *epp = &(enp->en_port);
250 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
251 	efx_rc_t rc;
252 
253 	if (encp->enc_enhanced_set_mac_supported) {
254 		if ((rc = efx_mcdi_mtu_set(enp, epp->ep_mac_pdu)) != 0)
255 			goto fail1;
256 	} else {
257 		/*
258 		 * Fallback for older Huntington firmware, which always
259 		 * configure all of the parameters to MC_CMD_SET_MAC. This isn't
260 		 * suitable for setting the MTU on unpriviliged functions.
261 		 */
262 		if ((rc = ef10_mac_reconfigure(enp)) != 0)
263 			goto fail2;
264 	}
265 
266 	return (0);
267 
268 fail2:
269 	EFSYS_PROBE(fail2);
270 fail1:
271 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
272 
273 	return (rc);
274 }
275 
276 	__checkReturn		efx_rc_t
277 ef10_mac_pdu_get(
278 	__in		efx_nic_t *enp,
279 	__out		size_t *pdu)
280 {
281 	efx_rc_t rc;
282 
283 	if ((rc = efx_mcdi_mtu_get(enp, pdu)) != 0)
284 		goto fail1;
285 
286 	return (0);
287 
288 fail1:
289 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
290 
291 	return (rc);
292 }
293 
294 __checkReturn	efx_rc_t
295 ef10_mac_reconfigure(
296 	__in		efx_nic_t *enp)
297 {
298 	efx_port_t *epp = &(enp->en_port);
299 	efx_mcdi_req_t req;
300 	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_IN_LEN,
301 		MC_CMD_SET_MAC_OUT_LEN);
302 	efx_rc_t rc;
303 
304 	req.emr_cmd = MC_CMD_SET_MAC;
305 	req.emr_in_buf = payload;
306 	req.emr_in_length = MC_CMD_SET_MAC_IN_LEN;
307 	req.emr_out_buf = payload;
308 	req.emr_out_length = MC_CMD_SET_MAC_OUT_LEN;
309 
310 	MCDI_IN_SET_DWORD(req, SET_MAC_IN_MTU, epp->ep_mac_pdu);
311 	MCDI_IN_SET_DWORD(req, SET_MAC_IN_DRAIN, epp->ep_mac_drain ? 1 : 0);
312 	EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, SET_MAC_IN_ADDR),
313 			    epp->ep_mac_addr);
314 
315 	/*
316 	 * Note: The Huntington MAC does not support REJECT_BRDCST.
317 	 * The REJECT_UNCST flag will also prevent multicast traffic
318 	 * from reaching the filters. As Huntington filters drop any
319 	 * traffic that does not match a filter it is ok to leave the
320 	 * MAC running in promiscuous mode. See bug41141.
321 	 *
322 	 * FIXME: Does REJECT_UNCST behave the same way on Medford?
323 	 */
324 	MCDI_IN_POPULATE_DWORD_2(req, SET_MAC_IN_REJECT,
325 				    SET_MAC_IN_REJECT_UNCST, 0,
326 				    SET_MAC_IN_REJECT_BRDCST, 0);
327 
328 	/*
329 	 * Flow control, whether it is auto-negotiated or not,
330 	 * is set via the PHY advertised capabilities.  When set to
331 	 * automatic the MAC will use the PHY settings to determine
332 	 * the flow control settings.
333 	 */
334 	MCDI_IN_SET_DWORD(req, SET_MAC_IN_FCNTL, MC_CMD_FCNTL_AUTO);
335 
336 	/* Do not include the Ethernet frame checksum in RX packets */
337 	MCDI_IN_POPULATE_DWORD_1(req, SET_MAC_IN_FLAGS,
338 				    SET_MAC_IN_FLAG_INCLUDE_FCS, 0);
339 
340 	efx_mcdi_execute_quiet(enp, &req);
341 
342 	if (req.emr_rc != 0) {
343 		/*
344 		 * Unprivileged functions cannot control link state,
345 		 * but still need to configure filters.
346 		 */
347 		if (req.emr_rc != EACCES) {
348 			rc = req.emr_rc;
349 			goto fail1;
350 		}
351 	}
352 
353 	/*
354 	 * Apply the filters for the MAC configuration.
355 	 * If the NIC isn't ready to accept filters this may
356 	 * return success without setting anything.
357 	 */
358 	rc = efx_filter_reconfigure(enp, epp->ep_mac_addr,
359 				    epp->ep_all_unicst, epp->ep_mulcst,
360 				    epp->ep_all_mulcst, epp->ep_brdcst,
361 				    epp->ep_mulcst_addr_list,
362 				    epp->ep_mulcst_addr_count);
363 
364 	return (0);
365 
366 fail1:
367 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
368 
369 	return (rc);
370 }
371 
372 	__checkReturn			efx_rc_t
373 ef10_mac_multicast_list_set(
374 	__in				efx_nic_t *enp)
375 {
376 	efx_port_t *epp = &(enp->en_port);
377 	const efx_mac_ops_t *emop = epp->ep_emop;
378 	efx_rc_t rc;
379 
380 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
381 	    enp->en_family == EFX_FAMILY_MEDFORD ||
382 	    enp->en_family == EFX_FAMILY_MEDFORD2);
383 
384 	if ((rc = emop->emo_reconfigure(enp)) != 0)
385 		goto fail1;
386 
387 	return (0);
388 
389 fail1:
390 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
391 
392 	return (rc);
393 }
394 
395 	__checkReturn	efx_rc_t
396 ef10_mac_filter_default_rxq_set(
397 	__in		efx_nic_t *enp,
398 	__in		efx_rxq_t *erp,
399 	__in		boolean_t using_rss)
400 {
401 	efx_port_t *epp = &(enp->en_port);
402 	efx_rxq_t *old_rxq;
403 	boolean_t old_using_rss;
404 	efx_rc_t rc;
405 
406 	ef10_filter_get_default_rxq(enp, &old_rxq, &old_using_rss);
407 
408 	ef10_filter_default_rxq_set(enp, erp, using_rss);
409 
410 	rc = efx_filter_reconfigure(enp, epp->ep_mac_addr,
411 				    epp->ep_all_unicst, epp->ep_mulcst,
412 				    epp->ep_all_mulcst, epp->ep_brdcst,
413 				    epp->ep_mulcst_addr_list,
414 				    epp->ep_mulcst_addr_count);
415 
416 	if (rc != 0)
417 		goto fail1;
418 
419 	return (0);
420 
421 fail1:
422 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
423 
424 	ef10_filter_default_rxq_set(enp, old_rxq, old_using_rss);
425 
426 	return (rc);
427 }
428 
429 			void
430 ef10_mac_filter_default_rxq_clear(
431 	__in		efx_nic_t *enp)
432 {
433 	efx_port_t *epp = &(enp->en_port);
434 
435 	ef10_filter_default_rxq_clear(enp);
436 
437 	(void) efx_filter_reconfigure(enp, epp->ep_mac_addr,
438 				    epp->ep_all_unicst, epp->ep_mulcst,
439 				    epp->ep_all_mulcst, epp->ep_brdcst,
440 				    epp->ep_mulcst_addr_list,
441 				    epp->ep_mulcst_addr_count);
442 }
443 
444 #if EFSYS_OPT_LOOPBACK
445 
446 	__checkReturn	efx_rc_t
447 ef10_mac_loopback_set(
448 	__in		efx_nic_t *enp,
449 	__in		efx_link_mode_t link_mode,
450 	__in		efx_loopback_type_t loopback_type)
451 {
452 	efx_port_t *epp = &(enp->en_port);
453 	const efx_phy_ops_t *epop = epp->ep_epop;
454 	efx_loopback_type_t old_loopback_type;
455 	efx_link_mode_t old_loopback_link_mode;
456 	efx_rc_t rc;
457 
458 	/* The PHY object handles this on EF10 */
459 	old_loopback_type = epp->ep_loopback_type;
460 	old_loopback_link_mode = epp->ep_loopback_link_mode;
461 	epp->ep_loopback_type = loopback_type;
462 	epp->ep_loopback_link_mode = link_mode;
463 
464 	if ((rc = epop->epo_reconfigure(enp)) != 0)
465 		goto fail1;
466 
467 	return (0);
468 
469 fail1:
470 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
471 
472 	epp->ep_loopback_type = old_loopback_type;
473 	epp->ep_loopback_link_mode = old_loopback_link_mode;
474 
475 	return (rc);
476 }
477 
478 #endif	/* EFSYS_OPT_LOOPBACK */
479 
480 #if EFSYS_OPT_MAC_STATS
481 
482 	__checkReturn			efx_rc_t
483 ef10_mac_stats_get_mask(
484 	__in				efx_nic_t *enp,
485 	__inout_bcount(mask_size)	uint32_t *maskp,
486 	__in				size_t mask_size)
487 {
488 	const struct efx_mac_stats_range ef10_common[] = {
489 		{ EFX_MAC_RX_OCTETS, EFX_MAC_RX_GE_15XX_PKTS },
490 		{ EFX_MAC_RX_FCS_ERRORS, EFX_MAC_RX_DROP_EVENTS },
491 		{ EFX_MAC_RX_JABBER_PKTS, EFX_MAC_RX_JABBER_PKTS },
492 		{ EFX_MAC_RX_NODESC_DROP_CNT, EFX_MAC_TX_PAUSE_PKTS },
493 	};
494 	const struct efx_mac_stats_range ef10_tx_size_bins[] = {
495 		{ EFX_MAC_TX_LE_64_PKTS, EFX_MAC_TX_GE_15XX_PKTS },
496 	};
497 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
498 	efx_port_t *epp = &(enp->en_port);
499 	efx_rc_t rc;
500 
501 	if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
502 	    ef10_common, EFX_ARRAY_SIZE(ef10_common))) != 0)
503 		goto fail1;
504 
505 	if (epp->ep_phy_cap_mask & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) {
506 		const struct efx_mac_stats_range ef10_40g_extra[] = {
507 			{ EFX_MAC_RX_ALIGN_ERRORS, EFX_MAC_RX_ALIGN_ERRORS },
508 		};
509 
510 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
511 		    ef10_40g_extra, EFX_ARRAY_SIZE(ef10_40g_extra))) != 0)
512 			goto fail2;
513 
514 		if (encp->enc_mac_stats_40g_tx_size_bins) {
515 			if ((rc = efx_mac_stats_mask_add_ranges(maskp,
516 			    mask_size, ef10_tx_size_bins,
517 			    EFX_ARRAY_SIZE(ef10_tx_size_bins))) != 0)
518 				goto fail3;
519 		}
520 	} else {
521 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
522 		    ef10_tx_size_bins, EFX_ARRAY_SIZE(ef10_tx_size_bins))) != 0)
523 			goto fail4;
524 	}
525 
526 	if (encp->enc_pm_and_rxdp_counters) {
527 		const struct efx_mac_stats_range ef10_pm_and_rxdp[] = {
528 			{ EFX_MAC_PM_TRUNC_BB_OVERFLOW, EFX_MAC_RXDP_HLB_WAIT },
529 		};
530 
531 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
532 		    ef10_pm_and_rxdp, EFX_ARRAY_SIZE(ef10_pm_and_rxdp))) != 0)
533 			goto fail5;
534 	}
535 
536 	if (encp->enc_datapath_cap_evb) {
537 		const struct efx_mac_stats_range ef10_vadaptor[] = {
538 			{ EFX_MAC_VADAPTER_RX_UNICAST_PACKETS,
539 			    EFX_MAC_VADAPTER_TX_OVERFLOW },
540 		};
541 
542 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
543 		    ef10_vadaptor, EFX_ARRAY_SIZE(ef10_vadaptor))) != 0)
544 			goto fail6;
545 	}
546 
547 	if (encp->enc_fec_counters) {
548 		const struct efx_mac_stats_range ef10_fec[] = {
549 			{ EFX_MAC_FEC_UNCORRECTED_ERRORS,
550 			    EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE3 },
551 		};
552 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
553 		    ef10_fec, EFX_ARRAY_SIZE(ef10_fec))) != 0)
554 			goto fail7;
555 	}
556 
557 	if (encp->enc_mac_stats_nstats >= MC_CMD_MAC_NSTATS_V4) {
558 		const struct efx_mac_stats_range ef10_rxdp_sdt[] = {
559 			{ EFX_MAC_RXDP_SCATTER_DISABLED_TRUNC,
560 			    EFX_MAC_RXDP_SCATTER_DISABLED_TRUNC },
561 		};
562 
563 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
564 		    ef10_rxdp_sdt, EFX_ARRAY_SIZE(ef10_rxdp_sdt))) != 0)
565 			goto fail8;
566 	}
567 
568 	if (encp->enc_hlb_counters) {
569 		const struct efx_mac_stats_range ef10_hlb[] = {
570 			{ EFX_MAC_RXDP_HLB_IDLE, EFX_MAC_RXDP_HLB_TIMEOUT },
571 		};
572 		if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size,
573 		    ef10_hlb, EFX_ARRAY_SIZE(ef10_hlb))) != 0)
574 			goto fail9;
575 	}
576 
577 	return (0);
578 
579 fail9:
580 	EFSYS_PROBE(fail9);
581 fail8:
582 	EFSYS_PROBE(fail8);
583 fail7:
584 	EFSYS_PROBE(fail7);
585 fail6:
586 	EFSYS_PROBE(fail6);
587 fail5:
588 	EFSYS_PROBE(fail5);
589 fail4:
590 	EFSYS_PROBE(fail4);
591 fail3:
592 	EFSYS_PROBE(fail3);
593 fail2:
594 	EFSYS_PROBE(fail2);
595 fail1:
596 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
597 
598 	return (rc);
599 }
600 
601 #define	EF10_MAC_STAT_READ(_esmp, _field, _eqp)			\
602 	EFSYS_MEM_READQ((_esmp), (_field) * sizeof (efx_qword_t), _eqp)
603 
604 	__checkReturn			efx_rc_t
605 ef10_mac_stats_update(
606 	__in				efx_nic_t *enp,
607 	__in				efsys_mem_t *esmp,
608 	__inout_ecount(EFX_MAC_NSTATS)	efsys_stat_t *stat,
609 	__inout_opt			uint32_t *generationp)
610 {
611 	const efx_nic_cfg_t *encp = &enp->en_nic_cfg;
612 	efx_qword_t generation_start;
613 	efx_qword_t generation_end;
614 	efx_qword_t value;
615 	efx_rc_t rc;
616 
617 	/*
618 	 * The MAC_STATS contain start and end generation counters used to
619 	 * detect when the DMA buffer has been updated during stats decode.
620 	 * All stats counters are 64bit unsigned values.
621 	 *
622 	 * Siena-compatible MAC stats contain MC_CMD_MAC_NSTATS 64bit counters.
623 	 * The generation end counter is at index MC_CMD_MAC_GENERATION_END
624 	 * (same as MC_CMD_MAC_NSTATS-1).
625 	 *
626 	 * Medford2 and later use a larger DMA buffer: MAC_STATS_NUM_STATS from
627 	 * MC_CMD_GET_CAPABILITIES_V4_OUT reports the number of 64bit counters.
628 	 *
629 	 * Firmware writes the generation end counter as the last counter in the
630 	 * DMA buffer. Do not use MC_CMD_MAC_GENERATION_END, as that is only
631 	 * correct for legacy Siena-compatible MAC stats.
632 	 */
633 
634 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS) {
635 		/* MAC stats count too small for legacy MAC stats */
636 		rc = ENOSPC;
637 		goto fail1;
638 	}
639 	if (EFSYS_MEM_SIZE(esmp) <
640 	    (encp->enc_mac_stats_nstats * sizeof (efx_qword_t))) {
641 		/* DMA buffer too small */
642 		rc = ENOSPC;
643 		goto fail2;
644 	}
645 
646 	/* Read END first so we don't race with the MC */
647 	EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFSYS_MEM_SIZE(esmp));
648 	EF10_MAC_STAT_READ(esmp, (encp->enc_mac_stats_nstats - 1),
649 	    &generation_end);
650 	EFSYS_MEM_READ_BARRIER();
651 
652 	/* TX */
653 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PKTS, &value);
654 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value);
655 
656 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_CONTROL_PKTS, &value);
657 	EFSYS_STAT_SUBR_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value);
658 
659 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PAUSE_PKTS, &value);
660 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PAUSE_PKTS]), &value);
661 
662 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_UNICAST_PKTS, &value);
663 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_UNICST_PKTS]), &value);
664 
665 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTICAST_PKTS, &value);
666 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULTICST_PKTS]), &value);
667 
668 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BROADCAST_PKTS, &value);
669 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_BRDCST_PKTS]), &value);
670 
671 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BYTES, &value);
672 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_OCTETS]), &value);
673 
674 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LT64_PKTS, &value);
675 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
676 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_64_PKTS, &value);
677 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
678 
679 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_65_TO_127_PKTS, &value);
680 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_65_TO_127_PKTS]), &value);
681 
682 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_128_TO_255_PKTS, &value);
683 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_128_TO_255_PKTS]), &value);
684 
685 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_256_TO_511_PKTS, &value);
686 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_256_TO_511_PKTS]), &value);
687 
688 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_512_TO_1023_PKTS, &value);
689 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_512_TO_1023_PKTS]), &value);
690 
691 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_1024_TO_15XX_PKTS, &value);
692 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_1024_TO_15XX_PKTS]), &value);
693 
694 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_15XX_TO_JUMBO_PKTS, &value);
695 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value);
696 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_GTJUMBO_PKTS, &value);
697 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value);
698 
699 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BAD_FCS_PKTS, &value);
700 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_ERRORS]), &value);
701 
702 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_SINGLE_COLLISION_PKTS, &value);
703 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_SGL_COL_PKTS]), &value);
704 
705 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTIPLE_COLLISION_PKTS,
706 			    &value);
707 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULT_COL_PKTS]), &value);
708 
709 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_COLLISION_PKTS,
710 			    &value);
711 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_COL_PKTS]), &value);
712 
713 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LATE_COLLISION_PKTS, &value);
714 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LATE_COL_PKTS]), &value);
715 
716 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_DEFERRED_PKTS, &value);
717 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_DEF_PKTS]), &value);
718 
719 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_DEFERRED_PKTS,
720 	    &value);
721 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_DEF_PKTS]), &value);
722 
723 	/* RX */
724 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BYTES, &value);
725 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_OCTETS]), &value);
726 
727 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PKTS, &value);
728 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PKTS]), &value);
729 
730 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNICAST_PKTS, &value);
731 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_UNICST_PKTS]), &value);
732 
733 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MULTICAST_PKTS, &value);
734 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MULTICST_PKTS]), &value);
735 
736 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BROADCAST_PKTS, &value);
737 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_BRDCST_PKTS]), &value);
738 
739 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PAUSE_PKTS, &value);
740 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PAUSE_PKTS]), &value);
741 
742 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNDERSIZE_PKTS, &value);
743 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value);
744 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_64_PKTS, &value);
745 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value);
746 
747 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_65_TO_127_PKTS, &value);
748 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_65_TO_127_PKTS]), &value);
749 
750 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_128_TO_255_PKTS, &value);
751 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_128_TO_255_PKTS]), &value);
752 
753 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_256_TO_511_PKTS, &value);
754 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_256_TO_511_PKTS]), &value);
755 
756 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_512_TO_1023_PKTS, &value);
757 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_512_TO_1023_PKTS]), &value);
758 
759 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_1024_TO_15XX_PKTS, &value);
760 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_1024_TO_15XX_PKTS]), &value);
761 
762 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_15XX_TO_JUMBO_PKTS, &value);
763 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value);
764 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_GTJUMBO_PKTS, &value);
765 	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value);
766 
767 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BAD_FCS_PKTS, &value);
768 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FCS_ERRORS]), &value);
769 
770 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_OVERFLOW_PKTS, &value);
771 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_DROP_EVENTS]), &value);
772 
773 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_FALSE_CARRIER_PKTS, &value);
774 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FALSE_CARRIER_ERRORS]), &value);
775 
776 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_SYMBOL_ERROR_PKTS, &value);
777 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_SYMBOL_ERRORS]), &value);
778 
779 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_ALIGN_ERROR_PKTS, &value);
780 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_ALIGN_ERRORS]), &value);
781 
782 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_INTERNAL_ERROR_PKTS, &value);
783 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_INTERNAL_ERRORS]), &value);
784 
785 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_JABBER_PKTS, &value);
786 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_JABBER_PKTS]), &value);
787 
788 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_CHAR_ERR, &value);
789 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_CHAR_ERR]),
790 			    &(value.eq_dword[0]));
791 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_CHAR_ERR]),
792 			    &(value.eq_dword[1]));
793 
794 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_CHAR_ERR, &value);
795 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_CHAR_ERR]),
796 			    &(value.eq_dword[0]));
797 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_CHAR_ERR]),
798 			    &(value.eq_dword[1]));
799 
800 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_DISP_ERR, &value);
801 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_DISP_ERR]),
802 			    &(value.eq_dword[0]));
803 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_DISP_ERR]),
804 			    &(value.eq_dword[1]));
805 
806 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_DISP_ERR, &value);
807 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_DISP_ERR]),
808 			    &(value.eq_dword[0]));
809 	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_DISP_ERR]),
810 			    &(value.eq_dword[1]));
811 
812 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MATCH_FAULT, &value);
813 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MATCH_FAULT]), &value);
814 
815 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_NODESC_DROPS, &value);
816 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_NODESC_DROP_CNT]), &value);
817 
818 	/* Packet memory (EF10 only) */
819 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_BB_OVERFLOW, &value);
820 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_BB_OVERFLOW]), &value);
821 
822 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_BB_OVERFLOW, &value);
823 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_BB_OVERFLOW]), &value);
824 
825 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_VFIFO_FULL, &value);
826 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_VFIFO_FULL]), &value);
827 
828 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_VFIFO_FULL, &value);
829 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_VFIFO_FULL]), &value);
830 
831 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_QBB, &value);
832 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_QBB]), &value);
833 
834 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_QBB, &value);
835 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_QBB]), &value);
836 
837 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_MAPPING, &value);
838 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_MAPPING]), &value);
839 
840 	/* RX datapath */
841 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_Q_DISABLED_PKTS, &value);
842 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_Q_DISABLED_PKTS]), &value);
843 
844 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_DI_DROPPED_PKTS, &value);
845 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_DI_DROPPED_PKTS]), &value);
846 
847 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_STREAMING_PKTS, &value);
848 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_STREAMING_PKTS]), &value);
849 
850 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_FETCH_CONDITIONS, &value);
851 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_FETCH]), &value);
852 
853 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_WAIT_CONDITIONS, &value);
854 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_WAIT]), &value);
855 
856 	/* VADAPTER RX */
857 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_UNICAST_PACKETS,
858 	    &value);
859 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS]),
860 	    &value);
861 
862 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_UNICAST_BYTES,
863 	    &value);
864 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_UNICAST_BYTES]),
865 	    &value);
866 
867 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_MULTICAST_PACKETS,
868 	    &value);
869 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS]),
870 	    &value);
871 
872 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_MULTICAST_BYTES,
873 	    &value);
874 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES]),
875 	    &value);
876 
877 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BROADCAST_PACKETS,
878 	    &value);
879 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS]),
880 	    &value);
881 
882 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BROADCAST_BYTES,
883 	    &value);
884 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES]),
885 	    &value);
886 
887 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BAD_PACKETS,
888 	    &value);
889 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BAD_PACKETS]),
890 	    &value);
891 
892 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BAD_BYTES, &value);
893 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BAD_BYTES]), &value);
894 
895 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_OVERFLOW, &value);
896 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_OVERFLOW]), &value);
897 
898 	/* VADAPTER TX */
899 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_UNICAST_PACKETS,
900 	    &value);
901 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS]),
902 	    &value);
903 
904 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_UNICAST_BYTES,
905 	    &value);
906 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_UNICAST_BYTES]),
907 	    &value);
908 
909 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_MULTICAST_PACKETS,
910 	    &value);
911 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS]),
912 	    &value);
913 
914 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_MULTICAST_BYTES,
915 	    &value);
916 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES]),
917 	    &value);
918 
919 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BROADCAST_PACKETS,
920 	    &value);
921 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS]),
922 	    &value);
923 
924 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BROADCAST_BYTES,
925 	    &value);
926 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES]),
927 	    &value);
928 
929 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BAD_PACKETS, &value);
930 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BAD_PACKETS]), &value);
931 
932 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BAD_BYTES, &value);
933 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BAD_BYTES]), &value);
934 
935 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_OVERFLOW, &value);
936 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_OVERFLOW]), &value);
937 
938 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS_V2)
939 		goto done;
940 
941 	/* FEC */
942 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_UNCORRECTED_ERRORS, &value);
943 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_UNCORRECTED_ERRORS]), &value);
944 
945 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_ERRORS, &value);
946 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_ERRORS]), &value);
947 
948 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE0,
949 	    &value);
950 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE0]),
951 	    &value);
952 
953 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE1,
954 	    &value);
955 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE1]),
956 	    &value);
957 
958 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE2,
959 	    &value);
960 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE2]),
961 	    &value);
962 
963 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE3,
964 	    &value);
965 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_FEC_CORRECTED_SYMBOLS_LANE3]),
966 	    &value);
967 
968 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS_V3)
969 		goto done;
970 
971 	/* CTPIO exceptions */
972 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_VI_BUSY_FALLBACK, &value);
973 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_VI_BUSY_FALLBACK]), &value);
974 
975 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_LONG_WRITE_SUCCESS, &value);
976 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_LONG_WRITE_SUCCESS]), &value);
977 
978 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_MISSING_DBELL_FAIL, &value);
979 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_MISSING_DBELL_FAIL]), &value);
980 
981 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_OVERFLOW_FAIL, &value);
982 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_OVERFLOW_FAIL]), &value);
983 
984 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_UNDERFLOW_FAIL, &value);
985 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_UNDERFLOW_FAIL]), &value);
986 
987 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_TIMEOUT_FAIL, &value);
988 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_TIMEOUT_FAIL]), &value);
989 
990 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_NONCONTIG_WR_FAIL, &value);
991 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_NONCONTIG_WR_FAIL]), &value);
992 
993 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_FRM_CLOBBER_FAIL, &value);
994 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_FRM_CLOBBER_FAIL]), &value);
995 
996 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_INVALID_WR_FAIL, &value);
997 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_INVALID_WR_FAIL]), &value);
998 
999 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_VI_CLOBBER_FALLBACK, &value);
1000 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_VI_CLOBBER_FALLBACK]),
1001 	    &value);
1002 
1003 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_UNQUALIFIED_FALLBACK, &value);
1004 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_UNQUALIFIED_FALLBACK]),
1005 	    &value);
1006 
1007 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_RUNT_FALLBACK, &value);
1008 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_RUNT_FALLBACK]), &value);
1009 
1010 	/* CTPIO per-port stats */
1011 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_SUCCESS, &value);
1012 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_SUCCESS]), &value);
1013 
1014 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_FALLBACK, &value);
1015 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_FALLBACK]), &value);
1016 
1017 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_POISON, &value);
1018 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_POISON]), &value);
1019 
1020 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_CTPIO_ERASE, &value);
1021 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_CTPIO_ERASE]), &value);
1022 
1023 	if (encp->enc_mac_stats_nstats < MC_CMD_MAC_NSTATS_V4)
1024 		goto done;
1025 
1026 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_SCATTER_DISABLED_TRUNC,
1027 	    &value);
1028 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_SCATTER_DISABLED_TRUNC]),
1029 	    &value);
1030 
1031 	/* Head-of-line blocking */
1032 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_IDLE, &value);
1033 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_IDLE]), &value);
1034 
1035 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_TIMEOUT, &value);
1036 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_TIMEOUT]), &value);
1037 
1038 done:
1039 	/* Read START generation counter */
1040 	EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFSYS_MEM_SIZE(esmp));
1041 	EFSYS_MEM_READ_BARRIER();
1042 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_START,
1043 			    &generation_start);
1044 
1045 	/* Check that we didn't read the stats in the middle of a DMA */
1046 	/* Not a good enough check ? */
1047 	if (memcmp(&generation_start, &generation_end,
1048 	    sizeof (generation_start)))
1049 		return (EAGAIN);
1050 
1051 	if (generationp)
1052 		*generationp = EFX_QWORD_FIELD(generation_start, EFX_DWORD_0);
1053 
1054 	return (0);
1055 
1056 fail2:
1057 	EFSYS_PROBE(fail2);
1058 fail1:
1059 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1060 
1061 	return (rc);
1062 }
1063 
1064 #endif	/* EFSYS_OPT_MAC_STATS */
1065 
1066 #endif	/* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
1067