xref: /illumos-gate/usr/src/uts/common/io/avintr.c (revision 0dee7919e2f2a6479d16b370af93747b9416b242)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Autovectored Interrupt Configuration and Deconfiguration
31  */
32 
33 #include <sys/param.h>
34 #include <sys/cmn_err.h>
35 #include <sys/trap.h>
36 #include <sys/t_lock.h>
37 #include <sys/avintr.h>
38 #include <sys/kmem.h>
39 #include <sys/machlock.h>
40 #include <sys/systm.h>
41 #include <sys/machsystm.h>
42 #include <sys/sunddi.h>
43 #include <sys/x_call.h>
44 #include <sys/cpuvar.h>
45 #include <sys/atomic.h>
46 #include <sys/smp_impldefs.h>
47 #include <sys/sdt.h>
48 #include <sys/stack.h>
49 #include <sys/ddi_impldefs.h>
50 
51 static int insert_av(void *intr_id, struct av_head *vectp, avfunc f,
52 	caddr_t arg1, caddr_t arg2, int pri_level, dev_info_t *dip);
53 static void remove_av(void *intr_id, struct av_head *vectp, avfunc f,
54 	int pri_level, int vect);
55 
56 /*
57  * Arrange for a driver to be called when a particular
58  * auto-vectored interrupt occurs.
59  * NOTE: if a device can generate interrupts on more than
60  * one level, or if a driver services devices that interrupt
61  * on more than one level, then the driver should install
62  * itself on each of those levels.
63  */
64 static char badsoft[] =
65 	"add_avintr: bad soft interrupt level %d for driver '%s'\n";
66 static char multilevel[] =
67 	"!IRQ%d is being shared by drivers with different interrupt levels.\n"
68 	"This may result in reduced system performance.";
69 static char multilevel2[] =
70 	"Cannot register interrupt for '%s' device at IPL %d because it\n"
71 	"conflicts with another device using the same vector %d with an IPL\n"
72 	"of %d. Reconfigure the conflicting devices to use different vectors.";
73 
74 #define	MAX_VECT	256
75 struct autovec *nmivect = NULL;
76 struct av_head autovect[MAX_VECT];
77 struct av_head softvect[LOCK_LEVEL + 1];
78 kmutex_t av_lock;
79 ddi_softint_hdl_impl_t softlevel1_hdl =
80 	{0, NULL, NULL, 0, 0, NULL, NULL, NULL};
81 
82 void
83 set_pending(int pri)
84 {
85 	atomic_or_32((uint32_t *)&CPU->cpu_softinfo.st_pending, 1 << pri);
86 }
87 
88 /*
89  * register nmi interrupt routine. The first arg is used only to order
90  * various nmi interrupt service routines in the chain. Higher lvls will
91  * be called first
92  */
93 int
94 add_nmintr(int lvl, avfunc nmintr, char *name, caddr_t arg)
95 {
96 	struct autovec  *mem;
97 	struct autovec *p, *prev = NULL;
98 
99 	if (nmintr == NULL) {
100 		printf("Attempt to add null vect for %s on nmi\n", name);
101 		return (0);
102 
103 	}
104 
105 	mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
106 	mem->av_vector = nmintr;
107 	mem->av_intarg1 = arg;
108 	mem->av_intarg2 = NULL;
109 	mem->av_intr_id = NULL;
110 	mem->av_prilevel = lvl;
111 	mem->av_dip = NULL;
112 	mem->av_link = NULL;
113 
114 	mutex_enter(&av_lock);
115 
116 	if (!nmivect) {
117 		nmivect = mem;
118 		mutex_exit(&av_lock);
119 		return (1);
120 	}
121 	/* find where it goes in list */
122 	for (p = nmivect; p != NULL; p = p->av_link) {
123 		if (p->av_vector == nmintr && p->av_intarg1 == arg) {
124 			/*
125 			 * already in list
126 			 * So? Somebody added the same interrupt twice.
127 			 */
128 			cmn_err(CE_WARN, "Driver already registered '%s'",
129 			    name);
130 			kmem_free(mem, sizeof (struct autovec));
131 			mutex_exit(&av_lock);
132 			return (0);
133 		}
134 		if (p->av_prilevel < lvl) {
135 			if (p == nmivect) {   /* it's at head of list */
136 				mem->av_link = p;
137 				nmivect = mem;
138 			} else {
139 				mem->av_link = p;
140 				prev->av_link = mem;
141 			}
142 			mutex_exit(&av_lock);
143 			return (1);
144 		}
145 		prev = p;
146 
147 	}
148 	/* didn't find it, add it to the end */
149 	prev->av_link = mem;
150 	mutex_exit(&av_lock);
151 	return (1);
152 
153 }
154 
155 /*
156  * register a hardware interrupt handler.
157  */
158 int
159 add_avintr(void *intr_id, int lvl, avfunc xxintr, char *name, int vect,
160     caddr_t arg1, caddr_t arg2, dev_info_t *dip)
161 {
162 	struct av_head *vecp = (struct av_head *)0;
163 	avfunc f;
164 	int s, vectindex;			/* save old spl value */
165 	ushort_t hi_pri;
166 
167 	if ((f = xxintr) == NULL) {
168 		printf("Attempt to add null vect for %s on vector %d\n",
169 			name, vect);
170 		return (0);
171 
172 	}
173 	vectindex = vect % MAX_VECT;
174 
175 	vecp = &autovect[vectindex];
176 
177 	/*
178 	 * "hi_pri == 0" implies all entries on list are "unused",
179 	 * which means that it's OK to just insert this one.
180 	 */
181 	hi_pri = vecp->avh_hi_pri;
182 	if (vecp->avh_link && (hi_pri != 0)) {
183 		if (((hi_pri > LOCK_LEVEL) && (lvl < LOCK_LEVEL)) ||
184 		    ((hi_pri < LOCK_LEVEL) && (lvl > LOCK_LEVEL))) {
185 			cmn_err(CE_WARN, multilevel2, name, lvl, vect,
186 				hi_pri);
187 			return (0);
188 		}
189 		if ((vecp->avh_lo_pri != lvl) || (hi_pri != lvl))
190 			cmn_err(CE_NOTE, multilevel, vect);
191 	}
192 
193 	if (!insert_av(intr_id, vecp, f, arg1, arg2, lvl, dip))
194 		return (0);
195 	s = splhi();
196 	/*
197 	 * do what ever machine specific things are necessary
198 	 * to set priority level (e.g. set picmasks)
199 	 */
200 	mutex_enter(&av_lock);
201 	(*addspl)(vect, lvl, vecp->avh_lo_pri, vecp->avh_hi_pri);
202 	mutex_exit(&av_lock);
203 	splx(s);
204 	return (1);
205 
206 }
207 
208 void
209 update_avsoftintr_args(void *intr_id, int lvl, caddr_t arg2)
210 {
211 	struct autovec *p;
212 	struct autovec *target = NULL;
213 	struct av_head *vectp = (struct av_head *)&softvect[lvl];
214 
215 	for (p = vectp->avh_link; p && p->av_vector; p = p->av_link) {
216 		if (p->av_intr_id == intr_id) {
217 			target = p;
218 			break;
219 		}
220 	}
221 
222 	if (target == NULL)
223 		return;
224 	target->av_intarg2 = arg2;
225 }
226 
227 /*
228  * Register a software interrupt handler
229  */
230 int
231 add_avsoftintr(void *intr_id, int lvl, avfunc xxintr, char *name,
232     caddr_t arg1, caddr_t arg2)
233 {
234 	int slvl;
235 
236 	if ((slvl = slvltovect(lvl)) != -1)
237 		return (add_avintr(intr_id, lvl, xxintr,
238 		    name, slvl, arg1, arg2, NULL));
239 
240 	if (intr_id == NULL) {
241 		printf("Attempt to add null intr_id for %s on level %d\n",
242 		    name, lvl);
243 		return (0);
244 	}
245 
246 	if (xxintr == NULL) {
247 		printf("Attempt to add null handler for %s on level %d\n",
248 		    name, lvl);
249 		return (0);
250 	}
251 
252 	if (lvl <= 0 || lvl > LOCK_LEVEL) {
253 		printf(badsoft, lvl, name);
254 		return (0);
255 	}
256 	if (!insert_av(intr_id, &softvect[lvl], xxintr, arg1, arg2,
257 	    lvl, NULL)) {
258 		return (0);
259 	}
260 	return (1);
261 }
262 
263 /* insert an interrupt vector into chain */
264 static int
265 insert_av(void *intr_id, struct av_head *vectp, avfunc f, caddr_t arg1,
266     caddr_t arg2, int pri_level, dev_info_t *dip)
267 {
268 	/*
269 	 * Protect rewrites of the list
270 	 */
271 	struct autovec *p, *mem;
272 
273 	mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
274 	mem->av_vector = f;
275 	mem->av_intarg1 = arg1;
276 	mem->av_intarg2 = arg2;
277 	mem->av_intr_id = intr_id;
278 	mem->av_prilevel = pri_level;
279 	mem->av_dip = dip;
280 	mem->av_link = NULL;
281 
282 	mutex_enter(&av_lock);
283 
284 	if (vectp->avh_link == NULL) {	/* Nothing on list - put it at head */
285 		vectp->avh_link = mem;
286 		vectp->avh_hi_pri = vectp->avh_lo_pri = (ushort_t)pri_level;
287 
288 		mutex_exit(&av_lock);
289 		return (1);
290 	}
291 
292 	/* find where it goes in list */
293 	for (p = vectp->avh_link; p != NULL; p = p->av_link) {
294 		if (p->av_vector == NULL) {	/* freed struct available */
295 			kmem_free(mem, sizeof (struct autovec));
296 			p->av_intarg1 = arg1;
297 			p->av_intarg2 = arg2;
298 			p->av_intr_id = intr_id;
299 			p->av_prilevel = pri_level;
300 			p->av_dip = dip;
301 			if (pri_level > (int)vectp->avh_hi_pri) {
302 				vectp->avh_hi_pri = (ushort_t)pri_level;
303 			}
304 			if (pri_level < (int)vectp->avh_lo_pri) {
305 				vectp->avh_lo_pri = (ushort_t)pri_level;
306 			}
307 			p->av_vector = f;
308 			mutex_exit(&av_lock);
309 			return (1);
310 		}
311 	}
312 	/* insert new intpt at beginning of chain */
313 	mem->av_link = vectp->avh_link;
314 	vectp->avh_link = mem;
315 	if (pri_level > (int)vectp->avh_hi_pri) {
316 		vectp->avh_hi_pri = (ushort_t)pri_level;
317 	}
318 	if (pri_level < (int)vectp->avh_lo_pri) {
319 		vectp->avh_lo_pri = (ushort_t)pri_level;
320 	}
321 	mutex_exit(&av_lock);
322 
323 	return (1);
324 }
325 
326 /*
327  * Remove a driver from the autovector list.
328  */
329 int
330 rem_avsoftintr(void *intr_id, int lvl, avfunc xxintr)
331 {
332 	struct av_head *vecp = (struct av_head *)0;
333 	int slvl;
334 
335 	if (xxintr == NULL)
336 		return (0);
337 
338 	if ((slvl = slvltovect(lvl)) != -1) {
339 		rem_avintr(intr_id, lvl, xxintr, slvl);
340 		return (1);
341 	}
342 
343 	if (lvl <= 0 && lvl >= LOCK_LEVEL) {
344 		return (0);
345 	}
346 	vecp = &softvect[lvl];
347 	remove_av(intr_id, vecp, xxintr, lvl, 0);
348 
349 	return (1);
350 }
351 
352 void
353 rem_avintr(void *intr_id, int lvl, avfunc xxintr, int vect)
354 {
355 	struct av_head *vecp = (struct av_head *)0;
356 	avfunc f;
357 	int s, vectindex;			/* save old spl value */
358 
359 	if ((f = xxintr) == NULL)
360 		return;
361 
362 	vectindex = vect % MAX_VECT;
363 	vecp = &autovect[vectindex];
364 	remove_av(intr_id, vecp, f, lvl, vect);
365 	s = splhi();
366 	mutex_enter(&av_lock);
367 	(*delspl)(vect, lvl, vecp->avh_lo_pri, vecp->avh_hi_pri);
368 	mutex_exit(&av_lock);
369 	splx(s);
370 }
371 
372 
373 /*
374  * After having made a change to an autovector list, wait until we have
375  * seen each cpu not executing an interrupt at that level--so we know our
376  * change has taken effect completely (no old state in registers, etc).
377  */
378 void
379 wait_till_seen(int ipl)
380 {
381 	int cpu_in_chain, cix;
382 	struct cpu *cpup;
383 	cpuset_t cpus_to_check;
384 
385 	CPUSET_ALL(cpus_to_check);
386 	do {
387 		cpu_in_chain = 0;
388 		for (cix = 0; cix < NCPU; cix++) {
389 			cpup = cpu[cix];
390 			if (cpup != NULL && CPU_IN_SET(cpus_to_check, cix)) {
391 				if (intr_active(cpup, ipl)) {
392 					cpu_in_chain = 1;
393 				} else {
394 					CPUSET_DEL(cpus_to_check, cix);
395 				}
396 			}
397 		}
398 	} while (cpu_in_chain);
399 }
400 
401 /* remove an interrupt vector from the chain */
402 static void
403 remove_av(void *intr_id, struct av_head *vectp, avfunc f, int pri_level,
404 	int vect)
405 {
406 	struct autovec *endp, *p, *target;
407 	int	lo_pri, hi_pri;
408 	int	ipl;
409 	/*
410 	 * Protect rewrites of the list
411 	 */
412 	target = NULL;
413 
414 	mutex_enter(&av_lock);
415 	ipl = pri_level;
416 	lo_pri = MAXIPL;
417 	hi_pri = 0;
418 	for (endp = p = vectp->avh_link; p && p->av_vector; p = p->av_link) {
419 		endp = p;
420 		if ((p->av_vector == f) && (p->av_intr_id == intr_id)) {
421 			/* found the handler */
422 			target = p;
423 			continue;
424 		}
425 		if (p->av_prilevel > hi_pri)
426 			hi_pri = p->av_prilevel;
427 		if (p->av_prilevel < lo_pri)
428 			lo_pri = p->av_prilevel;
429 	}
430 	if (ipl < hi_pri)
431 		ipl = hi_pri;
432 	if (target == NULL) {	/* not found */
433 		printf("Couldn't remove function %p at %d, %d\n",
434 			(void *)f, vect, pri_level);
435 		mutex_exit(&av_lock);
436 		return;
437 	}
438 
439 	target->av_vector = NULL;
440 	wait_till_seen(ipl);
441 	if (endp != target) {	/* vector to be removed is not last in chain */
442 		target->av_vector = endp->av_vector;
443 		target->av_intarg1 = endp->av_intarg1;
444 		target->av_intarg2 = endp->av_intarg2;
445 		target->av_intr_id = endp->av_intr_id;
446 		target->av_prilevel = endp->av_prilevel;
447 		target->av_dip = endp->av_dip;
448 		/*
449 		 * We have a hole here where the routine corresponding to
450 		 * endp may not get called. Do a wait_till_seen to take care
451 		 * of this.
452 		 */
453 		wait_till_seen(ipl);
454 		endp->av_vector = NULL;
455 	}
456 
457 	if (lo_pri > hi_pri) {	/* the chain is now empty */
458 		/* Leave the unused entries here for probable future use */
459 		vectp->avh_lo_pri = MAXIPL;
460 		vectp->avh_hi_pri = 0;
461 	} else {
462 		if ((int)vectp->avh_lo_pri < lo_pri)
463 			vectp->avh_lo_pri = (ushort_t)lo_pri;
464 		if ((int)vectp->avh_hi_pri > hi_pri)
465 			vectp->avh_hi_pri = (ushort_t)hi_pri;
466 	}
467 	mutex_exit(&av_lock);
468 	wait_till_seen(ipl);
469 }
470 
471 /*
472  * Trigger a soft interrupt.
473  */
474 void
475 siron(void)
476 {
477 	softlevel1_hdl.ih_pending = 1;
478 	(*setsoftint)(1);
479 }
480 
481 /*
482  * Walk the autovector table for this vector, invoking each
483  * interrupt handler as we go.
484  */
485 void
486 av_dispatch_autovect(uint_t vec)
487 {
488 	struct autovec *av;
489 
490 	ASSERT_STACK_ALIGNED();
491 
492 	while ((av = autovect[vec].avh_link) != NULL) {
493 		uint_t numcalled = 0;
494 		uint_t claimed = 0;
495 
496 		for (; av; av = av->av_link) {
497 			uint_t r;
498 			uint_t (*intr)() = av->av_vector;
499 			caddr_t arg1 = av->av_intarg1;
500 			caddr_t arg2 = av->av_intarg2;
501 			dev_info_t *dip = av->av_dip;
502 
503 			numcalled++;
504 			if (intr == NULL)
505 				break;
506 
507 			DTRACE_PROBE4(interrupt__start, dev_info_t *, dip,
508 			    void *, intr, caddr_t, arg1, caddr_t, arg2);
509 			r = (*intr)(arg1, arg2);
510 			DTRACE_PROBE4(interrupt__complete, dev_info_t *, dip,
511 			    void *, intr, caddr_t, arg1, uint_t, r);
512 			claimed |= r;
513 		}
514 
515 		/*
516 		 * If there's only one interrupt handler in the chain,
517 		 * or if no-one claimed the interrupt at all give up now.
518 		 */
519 		if (numcalled == 1 || claimed == 0)
520 			break;
521 	}
522 }
523 
524 /*
525  * Call every soft interrupt handler we can find at this level once.
526  */
527 void
528 av_dispatch_softvect(uint_t pil)
529 {
530 	struct autovec *av;
531 	ddi_softint_hdl_impl_t	*hdlp;
532 	uint_t (*intr)();
533 	caddr_t arg1;
534 	caddr_t arg2;
535 
536 	ASSERT_STACK_ALIGNED();
537 	ASSERT(pil >= 0 && pil <= PIL_MAX);
538 
539 	for (av = softvect[pil].avh_link; av; av = av->av_link) {
540 		if ((intr = av->av_vector) == NULL)
541 			break;
542 		arg1 = av->av_intarg1;
543 		arg2 = av->av_intarg2;
544 
545 		hdlp = (ddi_softint_hdl_impl_t *)av->av_intr_id;
546 		ASSERT(hdlp);
547 
548 		if (hdlp->ih_pending) {
549 			hdlp->ih_pending = 0;
550 			(void) (*intr)(arg1, arg2);
551 		}
552 	}
553 }
554 
555 struct regs;
556 
557 /*
558  * Call every NMI handler we know of once.
559  */
560 void
561 av_dispatch_nmivect(struct regs *rp)
562 {
563 	struct autovec *av;
564 
565 	ASSERT_STACK_ALIGNED();
566 
567 	for (av = nmivect; av; av = av->av_link)
568 		(void) (av->av_vector)(av->av_intarg1, rp);
569 }
570