xref: /illumos-gate/usr/src/uts/common/io/sfxge/sfxge_intr.c (revision 49ef7e0638c8b771d8a136eae78b1c0f99acc8e0)
1 /*
2  * Copyright (c) 2008-2016 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  * 1. Redistributions of source code must retain the above copyright notice,
9  *    this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright notice,
11  *    this list of conditions and the following disclaimer in the documentation
12  *    and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are
27  * those of the authors and should not be interpreted as representing official
28  * policies, either expressed or implied, of the FreeBSD Project.
29  */
30 
31 #include <sys/types.h>
32 #include <sys/ddi.h>
33 #include <sys/sunddi.h>
34 #include <sys/atomic.h>
35 #include <sys/modctl.h>
36 #include <sys/conf.h>
37 #include <sys/ethernet.h>
38 #include <sys/pci.h>
39 #include <sys/pcie.h>
40 
41 #include "sfxge.h"
42 
43 #include "efx.h"
44 
45 
46 /* Interrupt table DMA attributes */
47 static ddi_device_acc_attr_t sfxge_intr_devacc = {
48 
49 	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
50 	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
51 	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
52 };
53 
54 static ddi_dma_attr_t sfxge_intr_dma_attr = {
55 	DMA_ATTR_V0,		/* dma_attr_version	*/
56 	0,			/* dma_attr_addr_lo	*/
57 	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
58 	0xffffffffffffffffull,	/* dma_attr_count_max	*/
59 	EFX_INTR_SIZE,		/* dma_attr_align	*/
60 	0xffffffff,		/* dma_attr_burstsizes	*/
61 	1,			/* dma_attr_minxfer	*/
62 	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
63 	0xffffffffffffffffull,	/* dma_attr_seg		*/
64 	1,			/* dma_attr_sgllen	*/
65 	1,			/* dma_attr_granular	*/
66 	0			/* dma_attr_flags	*/
67 };
68 
69 static unsigned int
sfxge_intr_line(caddr_t arg1,caddr_t arg2)70 sfxge_intr_line(caddr_t arg1, caddr_t arg2)
71 {
72 	sfxge_t *sp = (void *)arg1;
73 	efx_nic_t *enp = sp->s_enp;
74 	sfxge_intr_t *sip = &(sp->s_intr);
75 	unsigned int index;
76 	boolean_t fatal;
77 	uint32_t qmask;
78 	int rc;
79 
80 	_NOTE(ARGUNUSED(arg2))
81 
82 	ASSERT3U(sip->si_type, ==, EFX_INTR_LINE);
83 
84 	if (sip->si_state != SFXGE_INTR_STARTED &&
85 	    sip->si_state != SFXGE_INTR_TESTING) {
86 		rc = DDI_INTR_UNCLAIMED;
87 		goto done;
88 	}
89 
90 	if (sip->si_state == SFXGE_INTR_TESTING) {
91 		sip->si_mask |= 1;	/* only one interrupt */
92 		rc = DDI_INTR_CLAIMED;
93 		goto done;
94 	}
95 
96 	efx_intr_status_line(enp, &fatal, &qmask);
97 
98 	if (fatal) {
99 		sfxge_intr_fatal(sp);
100 
101 		rc = DDI_INTR_CLAIMED;
102 		goto done;
103 	}
104 
105 	if (qmask != 0) {
106 		for (index = 0; index < EFX_INTR_NEVQS; index++) {
107 			if (qmask & (1 << index))
108 				(void) sfxge_ev_qpoll(sp, index);
109 		}
110 
111 		sip->si_zero_count = 0;
112 		sfxge_gld_rx_push(sp);
113 		rc = DDI_INTR_CLAIMED;
114 		goto done;
115 	}
116 
117 	/*
118 	 * bug15671/bug17203 workaround. Return CLAIMED for the first ISR=0
119 	 * interrupt, and poll all evqs for work. For subsequent ISR=0
120 	 * interrupts (the line must be shared in this case), just rearm the
121 	 * event queues to ensure we don't miss an interrupt.
122 	 */
123 	if (sip->si_zero_count++ == 0) {
124 		for (index = 0; index < EFX_INTR_NEVQS; index++) {
125 			if (sp->s_sep[index] != NULL)
126 				(void) sfxge_ev_qpoll(sp, index);
127 		}
128 
129 		rc = DDI_INTR_CLAIMED;
130 	} else {
131 		for (index = 0; index < EFX_INTR_NEVQS; index++) {
132 			if (sp->s_sep[index] != NULL)
133 				(void) sfxge_ev_qprime(sp, index);
134 		}
135 
136 		rc = DDI_INTR_UNCLAIMED;
137 	}
138 
139 done:
140 	return (rc);
141 }
142 
143 static unsigned int
sfxge_intr_message(caddr_t arg1,caddr_t arg2)144 sfxge_intr_message(caddr_t arg1, caddr_t arg2)
145 {
146 	sfxge_t *sp = (void *)arg1;
147 	efx_nic_t *enp = sp->s_enp;
148 	sfxge_intr_t *sip = &(sp->s_intr);
149 	unsigned int index = (unsigned int)(uintptr_t)arg2;
150 	boolean_t fatal;
151 	int rc;
152 
153 	ASSERT3U(sip->si_type, ==, EFX_INTR_MESSAGE);
154 
155 	if (sip->si_state != SFXGE_INTR_STARTED &&
156 	    sip->si_state != SFXGE_INTR_TESTING) {
157 		rc = DDI_INTR_UNCLAIMED;
158 		goto done;
159 	}
160 
161 	if (sip->si_state == SFXGE_INTR_TESTING) {
162 		uint64_t mask;
163 
164 		do {
165 			mask = sip->si_mask;
166 		} while (atomic_cas_64(&(sip->si_mask), mask,
167 		    mask | (1 << index)) != mask);
168 
169 		rc = DDI_INTR_CLAIMED;
170 		goto done;
171 	}
172 
173 	efx_intr_status_message(enp, index, &fatal);
174 
175 	if (fatal) {
176 		sfxge_intr_fatal(sp);
177 
178 		rc = DDI_INTR_CLAIMED;
179 		goto done;
180 	}
181 
182 	(void) sfxge_ev_qpoll(sp, index);
183 
184 	sfxge_gld_rx_push(sp);
185 	rc = DDI_INTR_CLAIMED;
186 
187 done:
188 	return (rc);
189 }
190 
191 static int
sfxge_intr_bus_enable(sfxge_t * sp)192 sfxge_intr_bus_enable(sfxge_t *sp)
193 {
194 	sfxge_intr_t *sip = &(sp->s_intr);
195 	ddi_intr_handler_t *handler;
196 	int add_index;
197 	int en_index;
198 	int err;
199 	int rc;
200 
201 	/* Serialise all instances to avoid problems seen in bug31184. */
202 	mutex_enter(&sfxge_global_lock);
203 
204 	switch (sip->si_type) {
205 	case EFX_INTR_MESSAGE:
206 		handler = sfxge_intr_message;
207 		break;
208 
209 	case EFX_INTR_LINE:
210 		handler = sfxge_intr_line;
211 		break;
212 
213 	default:
214 		dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
215 		    "bus_enable: unknown intr type (si_type=%d nalloc=%d)",
216 		    sip->si_type, sip->si_nalloc);
217 		ASSERT(B_FALSE);
218 		rc = EINVAL;
219 		goto fail1;
220 	}
221 
222 	/* Try to add the handlers */
223 	for (add_index = 0; add_index < sip->si_nalloc; add_index++) {
224 		unsigned int pri;
225 
226 		/* This cannot fail unless given invalid inputs. */
227 		err = ddi_intr_get_pri(sip->si_table[add_index], &pri);
228 		ASSERT(err == DDI_SUCCESS);
229 
230 		DTRACE_PROBE2(pri, unsigned int, add_index, unsigned int, pri);
231 
232 		err = ddi_intr_add_handler(sip->si_table[add_index], handler,
233 		    (caddr_t)sp, (caddr_t)(uintptr_t)add_index);
234 		if (err != DDI_SUCCESS) {
235 			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
236 			    "bus_enable: ddi_intr_add_handler failed"
237 			    " err=%d (h=%p idx=%d nalloc=%d)",
238 			    err, (void *)sip->si_table[add_index], add_index,
239 			    sip->si_nalloc);
240 
241 			rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
242 			goto fail2;
243 		}
244 	}
245 
246 	/* Get interrupt capabilities */
247 	err = ddi_intr_get_cap(sip->si_table[0], &(sip->si_cap));
248 	if (err != DDI_SUCCESS) {
249 		dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
250 		    "bus_enable: ddi_intr_get_cap failed"
251 		    " err=%d (h=%p idx=%d nalloc=%d)",
252 		    err, (void *)sip->si_table[0], 0, sip->si_nalloc);
253 
254 		if (err == DDI_EINVAL)
255 			rc = EINVAL;
256 		else if (err == DDI_ENOTSUP)
257 			rc = ENOTSUP;
258 		else
259 			rc = EFAULT;
260 
261 		goto fail3;
262 	}
263 
264 	/* Enable interrupts at the bus  */
265 	if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
266 		en_index = 0; /* Silence gcc */
267 		err = ddi_intr_block_enable(sip->si_table, sip->si_nalloc);
268 		if (err != DDI_SUCCESS) {
269 			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
270 			    "bus_enable: ddi_intr_block_enable failed"
271 			    " err=%d (table=%p nalloc=%d)",
272 			    err, (void *)sip->si_table, sip->si_nalloc);
273 
274 			rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
275 			goto fail4;
276 		}
277 	} else {
278 		for (en_index = 0; en_index < sip->si_nalloc; en_index++) {
279 			err = ddi_intr_enable(sip->si_table[en_index]);
280 			if (err != DDI_SUCCESS) {
281 				dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
282 				    "bus_enable: ddi_intr_enable failed"
283 				    " err=%d (h=%p idx=%d nalloc=%d)",
284 				    err, (void *)sip->si_table[en_index],
285 				    en_index, sip->si_nalloc);
286 
287 				rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
288 				goto fail4;
289 			}
290 		}
291 	}
292 
293 	mutex_exit(&sfxge_global_lock);
294 	return (0);
295 
296 fail4:
297 	DTRACE_PROBE(fail4);
298 
299 	/* Disable the enabled handlers */
300 	if (!(sip->si_cap & DDI_INTR_FLAG_BLOCK)) {
301 		while (--en_index >= 0) {
302 			err = ddi_intr_disable(sip->si_table[en_index]);
303 			if (err != DDI_SUCCESS) {
304 				dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
305 				    "bus_enable: ddi_intr_disable"
306 				    " failed err=%d (h=%p idx=%d nalloc=%d)",
307 				    err, (void *)sip->si_table[en_index],
308 				    en_index, sip->si_nalloc);
309 			}
310 		}
311 	}
312 
313 fail3:
314 	DTRACE_PROBE(fail3);
315 
316 	/* Remove all handlers */
317 	add_index = sip->si_nalloc;
318 
319 fail2:
320 	DTRACE_PROBE(fail2);
321 
322 	/* Remove remaining handlers */
323 	while (--add_index >= 0) {
324 		err = ddi_intr_remove_handler(sip->si_table[add_index]);
325 		if (err != DDI_SUCCESS) {
326 			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
327 			    "bus_enable: ddi_intr_remove_handler"
328 			    " failed err=%d (h=%p idx=%d nalloc=%d)",
329 			    err, (void *)sip->si_table[add_index], add_index,
330 			    sip->si_nalloc);
331 		}
332 	}
333 
334 fail1:
335 	DTRACE_PROBE1(fail1, int, rc);
336 
337 	mutex_exit(&sfxge_global_lock);
338 	return (rc);
339 }
340 
341 static void
sfxge_intr_bus_disable(sfxge_t * sp)342 sfxge_intr_bus_disable(sfxge_t *sp)
343 {
344 	sfxge_intr_t *sip = &(sp->s_intr);
345 	int index;
346 	int err;
347 
348 	/* Serialise all instances to avoid problems seen in bug31184. */
349 	mutex_enter(&sfxge_global_lock);
350 
351 	/* Disable interrupts at the bus */
352 	if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
353 		err = ddi_intr_block_disable(sip->si_table, sip->si_nalloc);
354 		if (err != DDI_SUCCESS) {
355 			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
356 			    "bus_disable: ddi_intr_block_disable"
357 			    " failed err=%d (table=%p nalloc=%d)",
358 			    err, (void *)sip->si_table, sip->si_nalloc);
359 		}
360 	} else {
361 		index = sip->si_nalloc;
362 		while (--index >= 0) {
363 			err = ddi_intr_disable(sip->si_table[index]);
364 			if (err != DDI_SUCCESS) {
365 				dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
366 				    "bus_disable: ddi_intr_disable"
367 				    " failed err=%d (h=%p idx=%d nalloc=%d)",
368 				    err, (void *)sip->si_table[index], index,
369 				    sip->si_nalloc);
370 			}
371 		}
372 	}
373 
374 	sip->si_cap = 0;
375 
376 	/* Remove all handlers */
377 	index = sip->si_nalloc;
378 	while (--index >= 0) {
379 		err = ddi_intr_remove_handler(sip->si_table[index]);
380 		if (err != DDI_SUCCESS) {
381 			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
382 			    "bus_disable: ddi_intr_remove_handler"
383 			    " failed err=%d (h=%p idx=%d nalloc=%d)",
384 			    err, (void *)sip->si_table[index], index,
385 			    sip->si_nalloc);
386 		}
387 	}
388 
389 	mutex_exit(&sfxge_global_lock);
390 }
391 
392 static int
sfxge_intr_nic_enable(sfxge_t * sp)393 sfxge_intr_nic_enable(sfxge_t *sp)
394 {
395 	sfxge_intr_t *sip = &(sp->s_intr);
396 	efsys_mem_t *esmp = &(sip->si_mem);
397 	efx_nic_t *enp = sp->s_enp;
398 	unsigned int index;
399 	uint64_t mask;
400 	unsigned int count;
401 	int rc;
402 
403 	/* Zero the memory */
404 	bzero(esmp->esm_base, EFX_INTR_SIZE);
405 
406 	/* Enable interrupts at the NIC */
407 	if ((rc = efx_intr_init(enp, sip->si_type, esmp)) != 0)
408 		goto fail1;
409 
410 	efx_intr_enable(enp);
411 
412 	/* FIXME FIXME FIXME */
413 	if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
414 		/* Disable interrupt test until supported on Huntington. */
415 		return (0);
416 	}
417 	/* FIXME FIXME FIXME */
418 
419 	/* Test the interrupts */
420 	mask = 0;
421 	for (index = 0; index < sip->si_nalloc; index++) {
422 		mask |= (1 << index);
423 
424 		rc = efx_intr_trigger(enp, index);
425 		ASSERT3U(rc, ==, 0);
426 	}
427 
428 	/* Wait for the tests to complete */
429 	count = 0;
430 	do {
431 		DTRACE_PROBE1(wait, unsigned int, count);
432 
433 		/* Spin for 1 ms */
434 		drv_usecwait(1000);
435 
436 		/*
437 		 * Check to see that all the test interrupts have been
438 		 * processed.
439 		 */
440 		if ((mask & sip->si_mask) == mask)
441 			goto done;
442 
443 	} while (++count < 20);
444 
445 	rc = ETIMEDOUT;
446 	goto fail2;
447 
448 done:
449 	return (0);
450 
451 fail2:
452 	DTRACE_PROBE(fail2);
453 
454 	dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
455 	    "Interrupt test failed (mask=%"PRIx64" got=%"
456 	    PRIx64"). NIC is disabled",
457 	    mask, sip->si_mask);
458 
459 	DTRACE_PROBE2(int_test_fail, uint64_t, mask, uint64_t, sip->si_mask);
460 
461 	sip->si_mask = 0;
462 
463 	/* Disable interrupts at the NIC */
464 	efx_intr_disable(enp);
465 	efx_intr_fini(enp);
466 
467 fail1:
468 	DTRACE_PROBE1(fail1, int, rc);
469 
470 	return (rc);
471 }
472 
473 static void
sfxge_intr_nic_disable(sfxge_t * sp)474 sfxge_intr_nic_disable(sfxge_t *sp)
475 {
476 	sfxge_intr_t *sip = &(sp->s_intr);
477 	efx_nic_t *enp = sp->s_enp;
478 
479 	sip->si_mask = 0;
480 
481 	/* Disable interrupts at the NIC */
482 	efx_intr_disable(enp);
483 	efx_intr_fini(enp);
484 }
485 
486 static inline unsigned
pow2_le(unsigned long n)487 pow2_le(unsigned long n)
488 {
489 	unsigned int order = 1;
490 	ASSERT3U(n, >, 0);
491 	while ((1ul << order) <= n) ++order;
492 	return (1ul << (order - 1));
493 }
494 
495 int
sfxge_intr_init(sfxge_t * sp)496 sfxge_intr_init(sfxge_t *sp)
497 {
498 	dev_info_t *dip = sp->s_dip;
499 	sfxge_intr_t *sip = &(sp->s_intr);
500 	efsys_mem_t *esmp = &(sip->si_mem);
501 	sfxge_dma_buffer_attr_t dma_attr;
502 	int err;
503 	int rc;
504 	int types;
505 	int type;
506 	int index;
507 	unsigned int nalloc;
508 	int navail;
509 
510 	SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
511 
512 	ASSERT3U(sip->si_state, ==, SFXGE_INTR_UNINITIALIZED);
513 
514 #ifdef __sparc
515 	/* PSARC 2007/453 */
516 	(void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
517 	    "#msix-request", NULL, 0);
518 #endif
519 
520 	/* Get the map of supported interrupt types */
521 	err = ddi_intr_get_supported_types(dip, &types);
522 	if (err != DDI_SUCCESS) {
523 		dev_err(dip, CE_WARN, SFXGE_CMN_ERR
524 		    "intr_init: ddi_intr_get_supported_types failed err=%d",
525 		    err);
526 
527 		if (err == DDI_EINVAL)
528 			rc = EINVAL;
529 		else if (err == DDI_INTR_NOTFOUND)
530 			rc = ENOENT;
531 		else
532 			rc = EFAULT;
533 
534 		goto fail1;
535 	}
536 
537 	/* Choose most favourable type */
538 	if (types & DDI_INTR_TYPE_MSIX) {
539 		DTRACE_PROBE(msix);
540 
541 		type = DDI_INTR_TYPE_MSIX;
542 		sip->si_type = EFX_INTR_MESSAGE;
543 	} else {
544 		DTRACE_PROBE(fixed);
545 
546 		ASSERT(types & DDI_INTR_TYPE_FIXED);
547 
548 		type = DDI_INTR_TYPE_FIXED;
549 		sip->si_type = EFX_INTR_LINE;
550 	}
551 
552 	/* Get the number of available interrupts */
553 	navail = 0;
554 	err = ddi_intr_get_navail(dip, type, &navail);
555 	if (err != DDI_SUCCESS) {
556 		dev_err(dip, CE_WARN, SFXGE_CMN_ERR
557 		    "intr_init: ddi_intr_get_navail failed err=%d", err);
558 
559 		if (err == DDI_EINVAL)
560 			rc = EINVAL;
561 		else if (err == DDI_INTR_NOTFOUND)
562 			rc = ENOENT;
563 		else
564 			rc = EFAULT;
565 
566 		goto fail2;
567 	}
568 
569 	/* Double-check */
570 	if (navail == 0) {
571 		rc = ENOENT;
572 		goto fail2;
573 	}
574 
575 	/*
576 	 * Allow greater number of MSI-X interrupts than CPUs.
577 	 * This can be useful to prevent RX no desc drops; See task 32179.
578 	 * Limit non MSI-X interrupts to a single instance.
579 	 */
580 	if (type != DDI_INTR_TYPE_MSIX)
581 		navail = 1;
582 	else
583 		navail = min(navail, sfxge_rx_scale_prop_get(sp));
584 
585 	DTRACE_PROBE1(navail, unsigned int, navail);
586 
587 	/* Allocate a handle table */
588 	sip->si_table_size = navail * sizeof (ddi_intr_handle_t);
589 	sip->si_table = kmem_zalloc(sip->si_table_size, KM_SLEEP);
590 
591 	/*
592 	 * Allocate interrupt handles.
593 	 * Serialise all device instances to avoid problems seen in bug31184.
594 	 */
595 	mutex_enter(&sfxge_global_lock);
596 
597 	err = ddi_intr_alloc(dip, sip->si_table, type, 0,
598 	    navail, &(sip->si_nalloc), DDI_INTR_ALLOC_NORMAL);
599 
600 	mutex_exit(&sfxge_global_lock);
601 
602 	if (err != DDI_SUCCESS) {
603 		dev_err(dip, CE_WARN, SFXGE_CMN_ERR
604 		    "intr_init: ddi_intr_alloc failed err=%d"
605 		    " (navail=%d nalloc=%d)",
606 		    err, navail, sip->si_nalloc);
607 
608 		if (err == DDI_EINVAL)
609 			rc = EINVAL;
610 		else if (err == DDI_EAGAIN)
611 			rc = EAGAIN;
612 		else if (err == DDI_INTR_NOTFOUND)
613 			rc = ENOENT;
614 		else
615 			rc = EFAULT;
616 
617 		goto fail3;
618 	}
619 
620 	/* Double-check */
621 	if (sip->si_nalloc == 0) {
622 		rc = ENOENT;
623 		goto fail3;
624 	}
625 
626 	/* Round down to a power of 2 */
627 	nalloc = pow2_le(sip->si_nalloc);
628 
629 	/* Free off any excess handles */
630 	mutex_enter(&sfxge_global_lock);
631 
632 	index = sip->si_nalloc;
633 	while (--index >= nalloc) {
634 		(void) ddi_intr_free(sip->si_table[index]);
635 		sip->si_table[index] = NULL;
636 	}
637 
638 	mutex_exit(&sfxge_global_lock);
639 
640 	sip->si_nalloc = nalloc;
641 	DTRACE_PROBE1(nalloc, unsigned int, sip->si_nalloc);
642 
643 	dma_attr.sdba_dip	 = sp->s_dip;
644 	dma_attr.sdba_dattrp	 = &sfxge_intr_dma_attr;
645 	dma_attr.sdba_callback	 = DDI_DMA_SLEEP;
646 	dma_attr.sdba_length	 = EFX_INTR_SIZE;
647 	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
648 	dma_attr.sdba_devaccp	 = &sfxge_intr_devacc;
649 	dma_attr.sdba_bindflags	 = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
650 	dma_attr.sdba_maxcookies = 1;
651 	dma_attr.sdba_zeroinit	 = B_TRUE;
652 
653 	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
654 		goto fail4;
655 
656 	/* Store the highest priority for convenience */
657 	sip->si_intr_pri = 0;
658 	for (index = 0; index < sip->si_nalloc; index++) {
659 		uint_t pri;
660 		if ((rc = ddi_intr_get_pri(sip->si_table[index], &pri)) !=  0)
661 			goto fail5;
662 		if (pri > sip->si_intr_pri)
663 			sip->si_intr_pri = pri;
664 	}
665 
666 	sip->si_state = SFXGE_INTR_INITIALIZED;
667 	return (0);
668 
669 fail5:
670 	DTRACE_PROBE(fail5);
671 
672 fail4:
673 	DTRACE_PROBE(fail4);
674 
675 	/* Free interrupt handles */
676 	mutex_exit(&sfxge_global_lock);
677 
678 	index = sip->si_nalloc;
679 	while (--index >= 0) {
680 		err = ddi_intr_free(sip->si_table[index]);
681 		if (err != DDI_SUCCESS) {
682 			dev_err(dip, CE_WARN, SFXGE_CMN_ERR
683 			    "intr_init: ddi_intr_free failed err=%d"
684 			    " (h=%p idx=%d nalloc=%d)",
685 			    err, (void *)sip->si_table[index], index,
686 			    sip->si_nalloc);
687 		}
688 		sip->si_table[index] = NULL;
689 	}
690 	sip->si_nalloc = 0;
691 
692 	mutex_exit(&sfxge_global_lock);
693 
694 fail3:
695 	DTRACE_PROBE(fail3);
696 
697 	/* Free the handle table */
698 	kmem_free(sip->si_table, sip->si_table_size);
699 	sip->si_table = NULL;
700 	sip->si_table_size = 0;
701 
702 fail2:
703 	DTRACE_PROBE(fail2);
704 
705 	/* Clear the interrupt type */
706 	sip->si_type = EFX_INTR_INVALID;
707 
708 fail1:
709 	DTRACE_PROBE1(fail1, int, rc);
710 
711 	SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
712 
713 	return (rc);
714 }
715 
716 int
sfxge_intr_start(sfxge_t * sp)717 sfxge_intr_start(sfxge_t *sp)
718 {
719 	sfxge_intr_t *sip = &(sp->s_intr);
720 	int rc;
721 
722 	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
723 
724 	/* Enable interrupts at the bus */
725 	if ((rc = sfxge_intr_bus_enable(sp)) != 0)
726 		goto fail1;
727 
728 	sip->si_state = SFXGE_INTR_TESTING;
729 
730 	/* Enable interrupts at the NIC */
731 	if ((rc = sfxge_intr_nic_enable(sp)) != 0)
732 		goto fail2;
733 
734 	sip->si_state = SFXGE_INTR_STARTED;
735 
736 	return (0);
737 
738 fail2:
739 	DTRACE_PROBE(fail2);
740 
741 	/* Disable interrupts at the bus */
742 	sfxge_intr_bus_disable(sp);
743 
744 fail1:
745 	DTRACE_PROBE1(fail1, int, rc);
746 
747 	sip->si_state = SFXGE_INTR_INITIALIZED;
748 
749 	return (rc);
750 }
751 
752 void
sfxge_intr_stop(sfxge_t * sp)753 sfxge_intr_stop(sfxge_t *sp)
754 {
755 	sfxge_intr_t *sip = &(sp->s_intr);
756 
757 	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
758 
759 	sip->si_state = SFXGE_INTR_INITIALIZED;
760 
761 	/* Disable interrupts at the NIC */
762 	sfxge_intr_nic_disable(sp);
763 
764 	/* Disable interrupts at the bus */
765 	sfxge_intr_bus_disable(sp);
766 }
767 
768 void
sfxge_intr_fini(sfxge_t * sp)769 sfxge_intr_fini(sfxge_t *sp)
770 {
771 	sfxge_intr_t *sip = &(sp->s_intr);
772 	efsys_mem_t *esmp = &(sip->si_mem);
773 	int index;
774 	int err;
775 
776 	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
777 
778 	sip->si_state = SFXGE_INTR_UNINITIALIZED;
779 
780 	/* Tear down dma setup */
781 	sfxge_dma_buffer_destroy(esmp);
782 
783 
784 	/* Free interrupt handles */
785 	mutex_enter(&sfxge_global_lock);
786 
787 	index = sip->si_nalloc;
788 	while (--index >= 0) {
789 		err = ddi_intr_free(sip->si_table[index]);
790 		if (err != DDI_SUCCESS) {
791 			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
792 			    "intr_fini: ddi_intr_free failed err=%d"
793 			    " (h=%p idx=%d nalloc=%d)",
794 			    err, (void *)sip->si_table[index],
795 			    index, sip->si_nalloc);
796 		}
797 		sip->si_table[index] = NULL;
798 	}
799 	sip->si_nalloc = 0;
800 
801 	mutex_exit(&sfxge_global_lock);
802 
803 	/* Free the handle table */
804 	kmem_free(sip->si_table, sip->si_table_size);
805 	sip->si_table = NULL;
806 	sip->si_table_size = 0;
807 
808 	/* Clear the interrupt type */
809 	sip->si_type = EFX_INTR_INVALID;
810 
811 	SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
812 }
813