xref: /freebsd/sys/dev/sfxge/common/efx_nic.c (revision af6a5351a1fdb1130f18be6c782c4d48916eb971)
1 /*-
2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  * 1. Redistributions of source code must retain the above copyright notice,
9  *    this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright notice,
11  *    this list of conditions and the following disclaimer in the documentation
12  *    and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are
27  * those of the authors and should not be interpreted as representing official
28  * policies, either expressed or implied, of the FreeBSD Project.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "efx.h"
35 #include "efx_impl.h"
36 
37 	__checkReturn	efx_rc_t
38 efx_family(
39 	__in		uint16_t venid,
40 	__in		uint16_t devid,
41 	__out		efx_family_t *efp)
42 {
43 	if (venid == EFX_PCI_VENID_SFC) {
44 		switch (devid) {
45 #if EFSYS_OPT_SIENA
46 		case EFX_PCI_DEVID_SIENA_F1_UNINIT:
47 			/*
48 			 * Hardware default for PF0 of uninitialised Siena.
49 			 * manftest must be able to cope with this device id.
50 			 */
51 			*efp = EFX_FAMILY_SIENA;
52 			return (0);
53 
54 		case EFX_PCI_DEVID_BETHPAGE:
55 		case EFX_PCI_DEVID_SIENA:
56 			*efp = EFX_FAMILY_SIENA;
57 			return (0);
58 #endif /* EFSYS_OPT_SIENA */
59 
60 #if EFSYS_OPT_HUNTINGTON
61 		case EFX_PCI_DEVID_HUNTINGTON_PF_UNINIT:
62 			/*
63 			 * Hardware default for PF0 of uninitialised Huntington.
64 			 * manftest must be able to cope with this device id.
65 			 */
66 			*efp = EFX_FAMILY_HUNTINGTON;
67 			return (0);
68 
69 		case EFX_PCI_DEVID_FARMINGDALE:
70 		case EFX_PCI_DEVID_GREENPORT:
71 			*efp = EFX_FAMILY_HUNTINGTON;
72 			return (0);
73 
74 		case EFX_PCI_DEVID_FARMINGDALE_VF:
75 		case EFX_PCI_DEVID_GREENPORT_VF:
76 			*efp = EFX_FAMILY_HUNTINGTON;
77 			return (0);
78 #endif /* EFSYS_OPT_HUNTINGTON */
79 
80 #if EFSYS_OPT_MEDFORD
81 		case EFX_PCI_DEVID_MEDFORD_PF_UNINIT:
82 			/*
83 			 * Hardware default for PF0 of uninitialised Medford.
84 			 * manftest must be able to cope with this device id.
85 			 */
86 			*efp = EFX_FAMILY_MEDFORD;
87 			return (0);
88 
89 		case EFX_PCI_DEVID_MEDFORD:
90 			*efp = EFX_FAMILY_MEDFORD;
91 			return (0);
92 
93 		case EFX_PCI_DEVID_MEDFORD_VF:
94 			*efp = EFX_FAMILY_MEDFORD;
95 			return (0);
96 #endif /* EFSYS_OPT_MEDFORD */
97 
98 		case EFX_PCI_DEVID_FALCON:	/* Obsolete, not supported */
99 		default:
100 			break;
101 		}
102 	}
103 
104 	*efp = EFX_FAMILY_INVALID;
105 	return (ENOTSUP);
106 }
107 
108 
109 #define	EFX_BIU_MAGIC0	0x01234567
110 #define	EFX_BIU_MAGIC1	0xfedcba98
111 
112 	__checkReturn	efx_rc_t
113 efx_nic_biu_test(
114 	__in		efx_nic_t *enp)
115 {
116 	efx_oword_t oword;
117 	efx_rc_t rc;
118 
119 	/*
120 	 * Write magic values to scratch registers 0 and 1, then
121 	 * verify that the values were written correctly.  Interleave
122 	 * the accesses to ensure that the BIU is not just reading
123 	 * back the cached value that was last written.
124 	 */
125 	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
126 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE);
127 
128 	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
129 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE);
130 
131 	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE);
132 	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
133 		rc = EIO;
134 		goto fail1;
135 	}
136 
137 	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE);
138 	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
139 		rc = EIO;
140 		goto fail2;
141 	}
142 
143 	/*
144 	 * Perform the same test, with the values swapped.  This
145 	 * ensures that subsequent tests don't start with the correct
146 	 * values already written into the scratch registers.
147 	 */
148 	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
149 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE);
150 
151 	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
152 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE);
153 
154 	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE);
155 	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
156 		rc = EIO;
157 		goto fail3;
158 	}
159 
160 	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE);
161 	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
162 		rc = EIO;
163 		goto fail4;
164 	}
165 
166 	return (0);
167 
168 fail4:
169 	EFSYS_PROBE(fail4);
170 fail3:
171 	EFSYS_PROBE(fail3);
172 fail2:
173 	EFSYS_PROBE(fail2);
174 fail1:
175 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
176 
177 	return (rc);
178 }
179 
180 #if EFSYS_OPT_SIENA
181 
182 static const efx_nic_ops_t	__efx_nic_siena_ops = {
183 	siena_nic_probe,		/* eno_probe */
184 	NULL,				/* eno_board_cfg */
185 	NULL,				/* eno_set_drv_limits */
186 	siena_nic_reset,		/* eno_reset */
187 	siena_nic_init,			/* eno_init */
188 	NULL,				/* eno_get_vi_pool */
189 	NULL,				/* eno_get_bar_region */
190 #if EFSYS_OPT_DIAG
191 	siena_nic_register_test,	/* eno_register_test */
192 #endif	/* EFSYS_OPT_DIAG */
193 	siena_nic_fini,			/* eno_fini */
194 	siena_nic_unprobe,		/* eno_unprobe */
195 };
196 
197 #endif	/* EFSYS_OPT_SIENA */
198 
199 #if EFSYS_OPT_HUNTINGTON
200 
201 static const efx_nic_ops_t	__efx_nic_hunt_ops = {
202 	ef10_nic_probe,			/* eno_probe */
203 	hunt_board_cfg,			/* eno_board_cfg */
204 	ef10_nic_set_drv_limits,	/* eno_set_drv_limits */
205 	ef10_nic_reset,			/* eno_reset */
206 	ef10_nic_init,			/* eno_init */
207 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
208 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
209 #if EFSYS_OPT_DIAG
210 	ef10_nic_register_test,		/* eno_register_test */
211 #endif	/* EFSYS_OPT_DIAG */
212 	ef10_nic_fini,			/* eno_fini */
213 	ef10_nic_unprobe,		/* eno_unprobe */
214 };
215 
216 #endif	/* EFSYS_OPT_HUNTINGTON */
217 
218 #if EFSYS_OPT_MEDFORD
219 
220 static const efx_nic_ops_t	__efx_nic_medford_ops = {
221 	ef10_nic_probe,			/* eno_probe */
222 	medford_board_cfg,		/* eno_board_cfg */
223 	ef10_nic_set_drv_limits,	/* eno_set_drv_limits */
224 	ef10_nic_reset,			/* eno_reset */
225 	ef10_nic_init,			/* eno_init */
226 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
227 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
228 #if EFSYS_OPT_DIAG
229 	ef10_nic_register_test,		/* eno_register_test */
230 #endif	/* EFSYS_OPT_DIAG */
231 	ef10_nic_fini,			/* eno_fini */
232 	ef10_nic_unprobe,		/* eno_unprobe */
233 };
234 
235 #endif	/* EFSYS_OPT_MEDFORD */
236 
237 
238 	__checkReturn	efx_rc_t
239 efx_nic_create(
240 	__in		efx_family_t family,
241 	__in		efsys_identifier_t *esip,
242 	__in		efsys_bar_t *esbp,
243 	__in		efsys_lock_t *eslp,
244 	__deref_out	efx_nic_t **enpp)
245 {
246 	efx_nic_t *enp;
247 	efx_rc_t rc;
248 
249 	EFSYS_ASSERT3U(family, >, EFX_FAMILY_INVALID);
250 	EFSYS_ASSERT3U(family, <, EFX_FAMILY_NTYPES);
251 
252 	/* Allocate a NIC object */
253 	EFSYS_KMEM_ALLOC(esip, sizeof (efx_nic_t), enp);
254 
255 	if (enp == NULL) {
256 		rc = ENOMEM;
257 		goto fail1;
258 	}
259 
260 	enp->en_magic = EFX_NIC_MAGIC;
261 
262 	switch (family) {
263 #if EFSYS_OPT_SIENA
264 	case EFX_FAMILY_SIENA:
265 		enp->en_enop = &__efx_nic_siena_ops;
266 		enp->en_features =
267 		    EFX_FEATURE_IPV6 |
268 		    EFX_FEATURE_LFSR_HASH_INSERT |
269 		    EFX_FEATURE_LINK_EVENTS |
270 		    EFX_FEATURE_PERIODIC_MAC_STATS |
271 		    EFX_FEATURE_MCDI |
272 		    EFX_FEATURE_LOOKAHEAD_SPLIT |
273 		    EFX_FEATURE_MAC_HEADER_FILTERS |
274 		    EFX_FEATURE_TX_SRC_FILTERS;
275 		break;
276 #endif	/* EFSYS_OPT_SIENA */
277 
278 #if EFSYS_OPT_HUNTINGTON
279 	case EFX_FAMILY_HUNTINGTON:
280 		enp->en_enop = &__efx_nic_hunt_ops;
281 		enp->en_features =
282 		    EFX_FEATURE_IPV6 |
283 		    EFX_FEATURE_LINK_EVENTS |
284 		    EFX_FEATURE_PERIODIC_MAC_STATS |
285 		    EFX_FEATURE_MCDI |
286 		    EFX_FEATURE_MAC_HEADER_FILTERS |
287 		    EFX_FEATURE_MCDI_DMA |
288 		    EFX_FEATURE_PIO_BUFFERS |
289 		    EFX_FEATURE_FW_ASSISTED_TSO |
290 		    EFX_FEATURE_FW_ASSISTED_TSO_V2;
291 		break;
292 #endif	/* EFSYS_OPT_HUNTINGTON */
293 
294 #if EFSYS_OPT_MEDFORD
295 	case EFX_FAMILY_MEDFORD:
296 		enp->en_enop = &__efx_nic_medford_ops;
297 		/*
298 		 * FW_ASSISTED_TSO omitted as Medford only supports firmware
299 		 * assisted TSO version 2, not the v1 scheme used on Huntington.
300 		 */
301 		enp->en_features =
302 		    EFX_FEATURE_IPV6 |
303 		    EFX_FEATURE_LINK_EVENTS |
304 		    EFX_FEATURE_PERIODIC_MAC_STATS |
305 		    EFX_FEATURE_MCDI |
306 		    EFX_FEATURE_MAC_HEADER_FILTERS |
307 		    EFX_FEATURE_MCDI_DMA |
308 		    EFX_FEATURE_PIO_BUFFERS |
309 		    EFX_FEATURE_FW_ASSISTED_TSO_V2;
310 		break;
311 #endif	/* EFSYS_OPT_MEDFORD */
312 
313 	default:
314 		rc = ENOTSUP;
315 		goto fail2;
316 	}
317 
318 	enp->en_family = family;
319 	enp->en_esip = esip;
320 	enp->en_esbp = esbp;
321 	enp->en_eslp = eslp;
322 
323 	*enpp = enp;
324 
325 	return (0);
326 
327 fail2:
328 	EFSYS_PROBE(fail2);
329 
330 	enp->en_magic = 0;
331 
332 	/* Free the NIC object */
333 	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
334 
335 fail1:
336 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
337 
338 	return (rc);
339 }
340 
341 	__checkReturn	efx_rc_t
342 efx_nic_probe(
343 	__in		efx_nic_t *enp)
344 {
345 	const efx_nic_ops_t *enop;
346 	efx_rc_t rc;
347 
348 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
349 #if EFSYS_OPT_MCDI
350 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
351 #endif	/* EFSYS_OPT_MCDI */
352 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_PROBE));
353 
354 	enop = enp->en_enop;
355 	if ((rc = enop->eno_probe(enp)) != 0)
356 		goto fail1;
357 
358 	if ((rc = efx_phy_probe(enp)) != 0)
359 		goto fail2;
360 
361 	enp->en_mod_flags |= EFX_MOD_PROBE;
362 
363 	return (0);
364 
365 fail2:
366 	EFSYS_PROBE(fail2);
367 
368 	enop->eno_unprobe(enp);
369 
370 fail1:
371 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
372 
373 	return (rc);
374 }
375 
376 	__checkReturn	efx_rc_t
377 efx_nic_set_drv_limits(
378 	__inout		efx_nic_t *enp,
379 	__in		efx_drv_limits_t *edlp)
380 {
381 	const efx_nic_ops_t *enop = enp->en_enop;
382 	efx_rc_t rc;
383 
384 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
385 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
386 
387 	if (enop->eno_set_drv_limits != NULL) {
388 		if ((rc = enop->eno_set_drv_limits(enp, edlp)) != 0)
389 			goto fail1;
390 	}
391 
392 	return (0);
393 
394 fail1:
395 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
396 
397 	return (rc);
398 }
399 
400 	__checkReturn	efx_rc_t
401 efx_nic_get_bar_region(
402 	__in		efx_nic_t *enp,
403 	__in		efx_nic_region_t region,
404 	__out		uint32_t *offsetp,
405 	__out		size_t *sizep)
406 {
407 	const efx_nic_ops_t *enop = enp->en_enop;
408 	efx_rc_t rc;
409 
410 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
411 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
412 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
413 
414 	if (enop->eno_get_bar_region == NULL) {
415 		rc = ENOTSUP;
416 		goto fail1;
417 	}
418 	if ((rc = (enop->eno_get_bar_region)(enp,
419 		    region, offsetp, sizep)) != 0) {
420 		goto fail2;
421 	}
422 
423 	return (0);
424 
425 fail2:
426 	EFSYS_PROBE(fail2);
427 
428 fail1:
429 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
430 
431 	return (rc);
432 }
433 
434 
435 	__checkReturn	efx_rc_t
436 efx_nic_get_vi_pool(
437 	__in		efx_nic_t *enp,
438 	__out		uint32_t *evq_countp,
439 	__out		uint32_t *rxq_countp,
440 	__out		uint32_t *txq_countp)
441 {
442 	const efx_nic_ops_t *enop = enp->en_enop;
443 	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
444 	efx_rc_t rc;
445 
446 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
447 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
448 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
449 
450 	if (enop->eno_get_vi_pool != NULL) {
451 		uint32_t vi_count = 0;
452 
453 		if ((rc = (enop->eno_get_vi_pool)(enp, &vi_count)) != 0)
454 			goto fail1;
455 
456 		*evq_countp = vi_count;
457 		*rxq_countp = vi_count;
458 		*txq_countp = vi_count;
459 	} else {
460 		/* Use NIC limits as default value */
461 		*evq_countp = encp->enc_evq_limit;
462 		*rxq_countp = encp->enc_rxq_limit;
463 		*txq_countp = encp->enc_txq_limit;
464 	}
465 
466 	return (0);
467 
468 fail1:
469 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
470 
471 	return (rc);
472 }
473 
474 
475 	__checkReturn	efx_rc_t
476 efx_nic_init(
477 	__in		efx_nic_t *enp)
478 {
479 	const efx_nic_ops_t *enop = enp->en_enop;
480 	efx_rc_t rc;
481 
482 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
483 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
484 
485 	if (enp->en_mod_flags & EFX_MOD_NIC) {
486 		rc = EINVAL;
487 		goto fail1;
488 	}
489 
490 	if ((rc = enop->eno_init(enp)) != 0)
491 		goto fail2;
492 
493 	enp->en_mod_flags |= EFX_MOD_NIC;
494 
495 	return (0);
496 
497 fail2:
498 	EFSYS_PROBE(fail2);
499 fail1:
500 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
501 
502 	return (rc);
503 }
504 
505 			void
506 efx_nic_fini(
507 	__in		efx_nic_t *enp)
508 {
509 	const efx_nic_ops_t *enop = enp->en_enop;
510 
511 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
512 	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
513 	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_NIC);
514 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
515 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
516 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
517 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
518 
519 	enop->eno_fini(enp);
520 
521 	enp->en_mod_flags &= ~EFX_MOD_NIC;
522 }
523 
524 			void
525 efx_nic_unprobe(
526 	__in		efx_nic_t *enp)
527 {
528 	const efx_nic_ops_t *enop = enp->en_enop;
529 
530 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
531 #if EFSYS_OPT_MCDI
532 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
533 #endif	/* EFSYS_OPT_MCDI */
534 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
535 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
536 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
537 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
538 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
539 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
540 
541 	efx_phy_unprobe(enp);
542 
543 	enop->eno_unprobe(enp);
544 
545 	enp->en_mod_flags &= ~EFX_MOD_PROBE;
546 }
547 
548 			void
549 efx_nic_destroy(
550 	__in	efx_nic_t *enp)
551 {
552 	efsys_identifier_t *esip = enp->en_esip;
553 
554 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
555 	EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0);
556 
557 	enp->en_family = EFX_FAMILY_INVALID;
558 	enp->en_esip = NULL;
559 	enp->en_esbp = NULL;
560 	enp->en_eslp = NULL;
561 
562 	enp->en_enop = NULL;
563 
564 	enp->en_magic = 0;
565 
566 	/* Free the NIC object */
567 	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
568 }
569 
570 	__checkReturn	efx_rc_t
571 efx_nic_reset(
572 	__in		efx_nic_t *enp)
573 {
574 	const efx_nic_ops_t *enop = enp->en_enop;
575 	unsigned int mod_flags;
576 	efx_rc_t rc;
577 
578 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
579 	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
580 	/*
581 	 * All modules except the MCDI, PROBE, NVRAM, VPD, MON
582 	 * (which we do not reset here) must have been shut down or never
583 	 * initialized.
584 	 *
585 	 * A rule of thumb here is: If the controller or MC reboots, is *any*
586 	 * state lost. If it's lost and needs reapplying, then the module
587 	 * *must* not be initialised during the reset.
588 	 */
589 	mod_flags = enp->en_mod_flags;
590 	mod_flags &= ~(EFX_MOD_MCDI | EFX_MOD_PROBE | EFX_MOD_NVRAM |
591 		    EFX_MOD_VPD | EFX_MOD_MON);
592 	EFSYS_ASSERT3U(mod_flags, ==, 0);
593 	if (mod_flags != 0) {
594 		rc = EINVAL;
595 		goto fail1;
596 	}
597 
598 	if ((rc = enop->eno_reset(enp)) != 0)
599 		goto fail2;
600 
601 	return (0);
602 
603 fail2:
604 	EFSYS_PROBE(fail2);
605 fail1:
606 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
607 
608 	return (rc);
609 }
610 
611 			const efx_nic_cfg_t *
612 efx_nic_cfg_get(
613 	__in		efx_nic_t *enp)
614 {
615 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
616 
617 	return (&(enp->en_nic_cfg));
618 }
619 
620 #if EFSYS_OPT_DIAG
621 
622 	__checkReturn	efx_rc_t
623 efx_nic_register_test(
624 	__in		efx_nic_t *enp)
625 {
626 	const efx_nic_ops_t *enop = enp->en_enop;
627 	efx_rc_t rc;
628 
629 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
630 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
631 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
632 
633 	if ((rc = enop->eno_register_test(enp)) != 0)
634 		goto fail1;
635 
636 	return (0);
637 
638 fail1:
639 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
640 
641 	return (rc);
642 }
643 
644 	__checkReturn	efx_rc_t
645 efx_nic_test_registers(
646 	__in		efx_nic_t *enp,
647 	__in		efx_register_set_t *rsp,
648 	__in		size_t count)
649 {
650 	unsigned int bit;
651 	efx_oword_t original;
652 	efx_oword_t reg;
653 	efx_oword_t buf;
654 	efx_rc_t rc;
655 
656 	while (count > 0) {
657 		/* This function is only suitable for registers */
658 		EFSYS_ASSERT(rsp->rows == 1);
659 
660 		/* bit sweep on and off */
661 		EFSYS_BAR_READO(enp->en_esbp, rsp->address, &original,
662 			    B_TRUE);
663 		for (bit = 0; bit < 128; bit++) {
664 			/* Is this bit in the mask? */
665 			if (~(rsp->mask.eo_u32[bit >> 5]) & (1 << bit))
666 				continue;
667 
668 			/* Test this bit can be set in isolation */
669 			reg = original;
670 			EFX_AND_OWORD(reg, rsp->mask);
671 			EFX_SET_OWORD_BIT(reg, bit);
672 
673 			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
674 				    B_TRUE);
675 			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
676 				    B_TRUE);
677 
678 			EFX_AND_OWORD(buf, rsp->mask);
679 			if (memcmp(&reg, &buf, sizeof (reg))) {
680 				rc = EIO;
681 				goto fail1;
682 			}
683 
684 			/* Test this bit can be cleared in isolation */
685 			EFX_OR_OWORD(reg, rsp->mask);
686 			EFX_CLEAR_OWORD_BIT(reg, bit);
687 
688 			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
689 				    B_TRUE);
690 			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
691 				    B_TRUE);
692 
693 			EFX_AND_OWORD(buf, rsp->mask);
694 			if (memcmp(&reg, &buf, sizeof (reg))) {
695 				rc = EIO;
696 				goto fail2;
697 			}
698 		}
699 
700 		/* Restore the old value */
701 		EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original,
702 			    B_TRUE);
703 
704 		--count;
705 		++rsp;
706 	}
707 
708 	return (0);
709 
710 fail2:
711 	EFSYS_PROBE(fail2);
712 fail1:
713 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
714 
715 	/* Restore the old value */
716 	EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original, B_TRUE);
717 
718 	return (rc);
719 }
720 
721 	__checkReturn	efx_rc_t
722 efx_nic_test_tables(
723 	__in		efx_nic_t *enp,
724 	__in		efx_register_set_t *rsp,
725 	__in		efx_pattern_type_t pattern,
726 	__in		size_t count)
727 {
728 	efx_sram_pattern_fn_t func;
729 	unsigned int index;
730 	unsigned int address;
731 	efx_oword_t reg;
732 	efx_oword_t buf;
733 	efx_rc_t rc;
734 
735 	EFSYS_ASSERT(pattern < EFX_PATTERN_NTYPES);
736 	func = __efx_sram_pattern_fns[pattern];
737 
738 	while (count > 0) {
739 		/* Write */
740 		address = rsp->address;
741 		for (index = 0; index < rsp->rows; ++index) {
742 			func(2 * index + 0, B_FALSE, &reg.eo_qword[0]);
743 			func(2 * index + 1, B_FALSE, &reg.eo_qword[1]);
744 			EFX_AND_OWORD(reg, rsp->mask);
745 			EFSYS_BAR_WRITEO(enp->en_esbp, address, &reg, B_TRUE);
746 
747 			address += rsp->step;
748 		}
749 
750 		/* Read */
751 		address = rsp->address;
752 		for (index = 0; index < rsp->rows; ++index) {
753 			func(2 * index + 0, B_FALSE, &reg.eo_qword[0]);
754 			func(2 * index + 1, B_FALSE, &reg.eo_qword[1]);
755 			EFX_AND_OWORD(reg, rsp->mask);
756 			EFSYS_BAR_READO(enp->en_esbp, address, &buf, B_TRUE);
757 			if (memcmp(&reg, &buf, sizeof (reg))) {
758 				rc = EIO;
759 				goto fail1;
760 			}
761 
762 			address += rsp->step;
763 		}
764 
765 		++rsp;
766 		--count;
767 	}
768 
769 	return (0);
770 
771 fail1:
772 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
773 
774 	return (rc);
775 }
776 
777 #endif	/* EFSYS_OPT_DIAG */
778 
779 #if EFSYS_OPT_LOOPBACK
780 
781 extern			void
782 efx_loopback_mask(
783 	__in	efx_loopback_kind_t loopback_kind,
784 	__out	efx_qword_t *maskp)
785 {
786 	efx_qword_t mask;
787 
788 	EFSYS_ASSERT3U(loopback_kind, <, EFX_LOOPBACK_NKINDS);
789 	EFSYS_ASSERT(maskp != NULL);
790 
791 	/* Assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespace agree */
792 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_NONE == EFX_LOOPBACK_OFF);
793 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_DATA == EFX_LOOPBACK_DATA);
794 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMAC == EFX_LOOPBACK_GMAC);
795 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII == EFX_LOOPBACK_XGMII);
796 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGXS == EFX_LOOPBACK_XGXS);
797 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI == EFX_LOOPBACK_XAUI);
798 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII == EFX_LOOPBACK_GMII);
799 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII == EFX_LOOPBACK_SGMII);
800 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGBR == EFX_LOOPBACK_XGBR);
801 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI == EFX_LOOPBACK_XFI);
802 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_FAR == EFX_LOOPBACK_XAUI_FAR);
803 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_FAR == EFX_LOOPBACK_GMII_FAR);
804 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII_FAR == EFX_LOOPBACK_SGMII_FAR);
805 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_FAR == EFX_LOOPBACK_XFI_FAR);
806 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GPHY == EFX_LOOPBACK_GPHY);
807 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS == EFX_LOOPBACK_PHY_XS);
808 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PCS == EFX_LOOPBACK_PCS);
809 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMAPMD == EFX_LOOPBACK_PMA_PMD);
810 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XPORT == EFX_LOOPBACK_XPORT);
811 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII_WS == EFX_LOOPBACK_XGMII_WS);
812 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_WS == EFX_LOOPBACK_XAUI_WS);
813 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_WS_FAR ==
814 	    EFX_LOOPBACK_XAUI_WS_FAR);
815 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_WS_NEAR ==
816 	    EFX_LOOPBACK_XAUI_WS_NEAR);
817 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_WS == EFX_LOOPBACK_GMII_WS);
818 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_WS == EFX_LOOPBACK_XFI_WS);
819 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_WS_FAR ==
820 	    EFX_LOOPBACK_XFI_WS_FAR);
821 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS_WS == EFX_LOOPBACK_PHYXS_WS);
822 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMA_INT == EFX_LOOPBACK_PMA_INT);
823 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_NEAR == EFX_LOOPBACK_SD_NEAR);
824 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FAR == EFX_LOOPBACK_SD_FAR);
825 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMA_INT_WS ==
826 	    EFX_LOOPBACK_PMA_INT_WS);
827 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FEP2_WS ==
828 	    EFX_LOOPBACK_SD_FEP2_WS);
829 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FEP1_5_WS ==
830 	    EFX_LOOPBACK_SD_FEP1_5_WS);
831 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FEP_WS == EFX_LOOPBACK_SD_FEP_WS);
832 	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FES_WS == EFX_LOOPBACK_SD_FES_WS);
833 
834 	/* Build bitmask of possible loopback types */
835 	EFX_ZERO_QWORD(mask);
836 
837 	if ((loopback_kind == EFX_LOOPBACK_KIND_OFF) ||
838 	    (loopback_kind == EFX_LOOPBACK_KIND_ALL)) {
839 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_OFF);
840 	}
841 
842 	if ((loopback_kind == EFX_LOOPBACK_KIND_MAC) ||
843 	    (loopback_kind == EFX_LOOPBACK_KIND_ALL)) {
844 		/*
845 		 * The "MAC" grouping has historically been used by drivers to
846 		 * mean loopbacks supported by on-chip hardware. Keep that
847 		 * meaning here, and include on-chip PHY layer loopbacks.
848 		 */
849 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_DATA);
850 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GMAC);
851 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XGMII);
852 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XGXS);
853 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XAUI);
854 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GMII);
855 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SGMII);
856 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XGBR);
857 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XFI);
858 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XAUI_FAR);
859 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GMII_FAR);
860 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SGMII_FAR);
861 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XFI_FAR);
862 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_PMA_INT);
863 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SD_NEAR);
864 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SD_FAR);
865 	}
866 
867 	if ((loopback_kind == EFX_LOOPBACK_KIND_PHY) ||
868 	    (loopback_kind == EFX_LOOPBACK_KIND_ALL)) {
869 		/*
870 		 * The "PHY" grouping has historically been used by drivers to
871 		 * mean loopbacks supported by off-chip hardware. Keep that
872 		 * meaning here.
873 		 */
874 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GPHY);
875 		EFX_SET_QWORD_BIT(mask,	EFX_LOOPBACK_PHY_XS);
876 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_PCS);
877 		EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_PMA_PMD);
878 	}
879 
880 	*maskp = mask;
881 }
882 
883 	__checkReturn	efx_rc_t
884 efx_mcdi_get_loopback_modes(
885 	__in		efx_nic_t *enp)
886 {
887 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
888 	efx_mcdi_req_t req;
889 	uint8_t payload[MAX(MC_CMD_GET_LOOPBACK_MODES_IN_LEN,
890 			    MC_CMD_GET_LOOPBACK_MODES_OUT_LEN)];
891 	efx_qword_t mask;
892 	efx_qword_t modes;
893 	efx_rc_t rc;
894 
895 	(void) memset(payload, 0, sizeof (payload));
896 	req.emr_cmd = MC_CMD_GET_LOOPBACK_MODES;
897 	req.emr_in_buf = payload;
898 	req.emr_in_length = MC_CMD_GET_LOOPBACK_MODES_IN_LEN;
899 	req.emr_out_buf = payload;
900 	req.emr_out_length = MC_CMD_GET_LOOPBACK_MODES_OUT_LEN;
901 
902 	efx_mcdi_execute(enp, &req);
903 
904 	if (req.emr_rc != 0) {
905 		rc = req.emr_rc;
906 		goto fail1;
907 	}
908 
909 	if (req.emr_out_length_used <
910 	    MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_OFST +
911 	    MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LEN) {
912 		rc = EMSGSIZE;
913 		goto fail2;
914 	}
915 
916 	/*
917 	 * We assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespaces agree
918 	 * in efx_loopback_mask() and in siena_phy.c:siena_phy_get_link().
919 	 */
920 	efx_loopback_mask(EFX_LOOPBACK_KIND_ALL, &mask);
921 
922 	EFX_AND_QWORD(mask,
923 	    *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_SUGGESTED));
924 
925 	modes = *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_100M);
926 	EFX_AND_QWORD(modes, mask);
927 	encp->enc_loopback_types[EFX_LINK_100FDX] = modes;
928 
929 	modes = *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_1G);
930 	EFX_AND_QWORD(modes, mask);
931 	encp->enc_loopback_types[EFX_LINK_1000FDX] = modes;
932 
933 	modes = *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_10G);
934 	EFX_AND_QWORD(modes, mask);
935 	encp->enc_loopback_types[EFX_LINK_10000FDX] = modes;
936 
937 	if (req.emr_out_length_used >=
938 	    MC_CMD_GET_LOOPBACK_MODES_OUT_40G_OFST +
939 	    MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LEN) {
940 		/* Response includes 40G loopback modes */
941 		modes =
942 		    *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_40G);
943 		EFX_AND_QWORD(modes, mask);
944 		encp->enc_loopback_types[EFX_LINK_40000FDX] = modes;
945 	}
946 
947 	EFX_ZERO_QWORD(modes);
948 	EFX_SET_QWORD_BIT(modes, EFX_LOOPBACK_OFF);
949 	EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_100FDX]);
950 	EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_1000FDX]);
951 	EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_10000FDX]);
952 	EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_40000FDX]);
953 	encp->enc_loopback_types[EFX_LINK_UNKNOWN] = modes;
954 
955 	return (0);
956 
957 fail2:
958 	EFSYS_PROBE(fail2);
959 fail1:
960 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
961 
962 	return (rc);
963 }
964 
965 #endif /* EFSYS_OPT_LOOPBACK */
966 
967 	__checkReturn	efx_rc_t
968 efx_nic_calculate_pcie_link_bandwidth(
969 	__in		uint32_t pcie_link_width,
970 	__in		uint32_t pcie_link_gen,
971 	__out		uint32_t *bandwidth_mbpsp)
972 {
973 	uint32_t lane_bandwidth;
974 	uint32_t total_bandwidth;
975 	efx_rc_t rc;
976 
977 	if ((pcie_link_width == 0) || (pcie_link_width > 16) ||
978 	    !ISP2(pcie_link_width)) {
979 		rc = EINVAL;
980 		goto fail1;
981 	}
982 
983 	switch (pcie_link_gen) {
984 	case EFX_PCIE_LINK_SPEED_GEN1:
985 		/* 2.5 Gb/s raw bandwidth with 8b/10b encoding */
986 		lane_bandwidth = 2000;
987 		break;
988 	case EFX_PCIE_LINK_SPEED_GEN2:
989 		/* 5.0 Gb/s raw bandwidth with 8b/10b encoding */
990 		lane_bandwidth = 4000;
991 		break;
992 	case EFX_PCIE_LINK_SPEED_GEN3:
993 		/* 8.0 Gb/s raw bandwidth with 128b/130b encoding */
994 		lane_bandwidth = 7877;
995 		break;
996 	default:
997 		rc = EINVAL;
998 		goto fail2;
999 	}
1000 
1001 	total_bandwidth = lane_bandwidth * pcie_link_width;
1002 	*bandwidth_mbpsp = total_bandwidth;
1003 
1004 	return (0);
1005 
1006 fail2:
1007 	EFSYS_PROBE(fail2);
1008 fail1:
1009 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1010 
1011 	return (rc);
1012 }
1013 
1014 
1015 	__checkReturn	efx_rc_t
1016 efx_nic_check_pcie_link_speed(
1017 	__in		efx_nic_t *enp,
1018 	__in		uint32_t pcie_link_width,
1019 	__in		uint32_t pcie_link_gen,
1020 	__out		efx_pcie_link_performance_t *resultp)
1021 {
1022 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1023 	uint32_t bandwidth;
1024 	efx_pcie_link_performance_t result;
1025 	efx_rc_t rc;
1026 
1027 	if ((encp->enc_required_pcie_bandwidth_mbps == 0) ||
1028 	    (pcie_link_width == 0) || (pcie_link_width == 32) ||
1029 	    (pcie_link_gen == 0)) {
1030 		/*
1031 		 * No usable info on what is required and/or in use. In virtual
1032 		 * machines, sometimes the PCIe link width is reported as 0 or
1033 		 * 32, or the speed as 0.
1034 		 */
1035 		result = EFX_PCIE_LINK_PERFORMANCE_UNKNOWN_BANDWIDTH;
1036 		goto out;
1037 	}
1038 
1039 	/* Calculate the available bandwidth in megabits per second */
1040 	rc = efx_nic_calculate_pcie_link_bandwidth(pcie_link_width,
1041 					    pcie_link_gen, &bandwidth);
1042 	if (rc != 0)
1043 		goto fail1;
1044 
1045 	if (bandwidth < encp->enc_required_pcie_bandwidth_mbps) {
1046 		result = EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_BANDWIDTH;
1047 	} else if (pcie_link_gen < encp->enc_max_pcie_link_gen) {
1048 		/* The link provides enough bandwidth but not optimal latency */
1049 		result = EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_LATENCY;
1050 	} else {
1051 		result = EFX_PCIE_LINK_PERFORMANCE_OPTIMAL;
1052 	}
1053 
1054 out:
1055 	*resultp = result;
1056 
1057 	return (0);
1058 
1059 fail1:
1060 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
1061 
1062 	return (rc);
1063 }
1064