68  * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes.
206  * current CPU contains a number of kernel threads (kthread_t) that can be used
216  * interrupts, but the notification vector is different. Each CPU has a bitmask
217  * of pending software interrupts. We can notify a CPU to process software
326  * now have per-cpu vectors.
504  * This variable tracks the last place events were disabled on each cpu
517  * Set cpu's base SPL level to the highest active interrupt level
522 	struct cpu *cpu = CPU;  in set_base_spl()  local
523 	uint16_t active = (uint16_t)cpu->cpu_intr_actv;  in set_base_spl()
525 	cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active);  in set_base_spl()
529  * Do all the work necessary to set up the cpu and thread structures
539 hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)  in hilevel_intr_prolog()  argument
541 	struct machcpu *mcpu = &cpu->cpu_m;  in hilevel_intr_prolog()
549 		cpu->cpu_profile_pil = oldpil;  in hilevel_intr_prolog()
551 			cpu->cpu_profile_pc = 0;  in hilevel_intr_prolog()
552 			cpu->cpu_profile_upc = rp->r_pc;  in hilevel_intr_prolog()
553 			cpu->cpu_cpcprofile_pc = 0;  in hilevel_intr_prolog()
554 			cpu->cpu_cpcprofile_upc = rp->r_pc;  in hilevel_intr_prolog()
556 			cpu->cpu_profile_pc = rp->r_pc;  in hilevel_intr_prolog()
557 			cpu->cpu_profile_upc = 0;  in hilevel_intr_prolog()
558 			cpu->cpu_cpcprofile_pc = rp->r_pc;  in hilevel_intr_prolog()
559 			cpu->cpu_cpcprofile_upc = 0;  in hilevel_intr_prolog()
563 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;  in hilevel_intr_prolog()
577 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;  in hilevel_intr_prolog()
585 		kthread_t *t = cpu->cpu_thread;  in hilevel_intr_prolog()
595 			cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;  in hilevel_intr_prolog()
603 	 * Store starting timestamp in CPU structure for this PIL.  in hilevel_intr_prolog()
607 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);  in hilevel_intr_prolog()
616 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;  in hilevel_intr_prolog()
620 	mask = cpu->cpu_intr_actv;  in hilevel_intr_prolog()
622 	cpu->cpu_intr_actv |= (1 << pil);  in hilevel_intr_prolog()
637 hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)  in hilevel_intr_epilog()  argument
639 	struct machcpu *mcpu = &cpu->cpu_m;  in hilevel_intr_epilog()
646 	cpu->cpu_stats.sys.intr[pil - 1]++;  in hilevel_intr_epilog()
648 	ASSERT(cpu->cpu_intr_actv & (1 << pil));  in hilevel_intr_epilog()
657 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;  in hilevel_intr_epilog()
662 			cpu->cpu_intr_actv &= ~(1 << pil);  in hilevel_intr_epilog()
664 		cpu->cpu_intr_actv &= ~(1 << pil);  in hilevel_intr_epilog()
671 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;  in hilevel_intr_epilog()
678 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;  in hilevel_intr_epilog()
700 		kthread_t *t = cpu->cpu_thread;  in hilevel_intr_epilog()
711 	return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);  in hilevel_intr_epilog()
715  * Set up the cpu, thread and interrupt thread structures for
720 intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)  in intr_thread_prolog()  argument
722 	struct machcpu *mcpu = &cpu->cpu_m;  in intr_thread_prolog()
727 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);  in intr_thread_prolog()
728 	cpu->cpu_intr_actv |= (1 << pil);  in intr_thread_prolog()
733 	 * allocate one for each level on each CPU.  in intr_thread_prolog()
737 	t = cpu->cpu_thread;  in intr_thread_prolog()
741 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;  in intr_thread_prolog()
750 	 * unlink the interrupt thread off the cpu  in intr_thread_prolog()
757 	it = cpu->cpu_intr_thread;  in intr_thread_prolog()
758 	cpu->cpu_intr_thread = it->t_link;  in intr_thread_prolog()
769 	cpu->cpu_thread = it;		/* new curthread on this cpu */  in intr_thread_prolog()
788 intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)  in intr_thread_epilog()  argument
790 	struct machcpu *mcpu = &cpu->cpu_m;  in intr_thread_epilog()
792 	kthread_t *it = cpu->cpu_thread;	/* curthread */  in intr_thread_epilog()
798 	cpu->cpu_stats.sys.intr[pil - 1]++;  in intr_thread_epilog()
803 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;  in intr_thread_epilog()
805 	ASSERT(cpu->cpu_intr_actv & (1 << pil));  in intr_thread_epilog()
806 	cpu->cpu_intr_actv &= ~(1 << pil);  in intr_thread_epilog()
822 		 * this interrupt thread back on the CPU's free list and  in intr_thread_epilog()
829 		cpu->cpu_stats.sys.intrblk++;  in intr_thread_epilog()
831 		 * Set CPU's base SPL based on active interrupts bitmask  in intr_thread_epilog()
834 		basespl = cpu->cpu_base_spl;  in intr_thread_epilog()
843 		it->t_link = cpu->cpu_intr_thread;  in intr_thread_epilog()
844 		cpu->cpu_intr_thread = it;  in intr_thread_epilog()
853 	it->t_link = cpu->cpu_intr_thread;  in intr_thread_epilog()
854 	cpu->cpu_intr_thread = it;  in intr_thread_epilog()
857 	basespl = cpu->cpu_base_spl;  in intr_thread_epilog()
863 	cpu->cpu_thread = t;  in intr_thread_epilog()
888  * handling all interrupts at the specified pil on this CPU. It is
915  * Whenever interrupts arrive on a CPU which is handling a lower pil
926 	struct cpu *cpu;  in intr_get_time()  local
933 	cpu = CPU;  in intr_get_time()
934 	mcpu = &cpu->cpu_m;  in intr_get_time()
935 	t = cpu->cpu_thread;  in intr_get_time()
937 	ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);  in intr_get_time()
950 	cpu->cpu_intracct[cpu->cpu_mstate] += delta;  in intr_get_time()
958 	struct cpu *cpu,  in dosoftint_prolog()  argument
964 	struct machcpu *mcpu = &cpu->cpu_m;  in dosoftint_prolog()
972 	if (pil <= oldpil || pil <= cpu->cpu_base_spl)  in dosoftint_prolog()
982 	 * CPU might get preempted just after the address computation,  in dosoftint_prolog()
983 	 * but just before the atomic transaction, so another CPU would  in dosoftint_prolog()
984 	 * actually set the original CPU's st_pending bit.  However,  in dosoftint_prolog()
992 	 * disabled on -this- CPU.  This code would probably be cheaper:  in dosoftint_prolog()
1015 	 * allocate one for each level on the CPU.  in dosoftint_prolog()
1017 	it = cpu->cpu_intr_thread;  in dosoftint_prolog()
1018 	cpu->cpu_intr_thread = it->t_link;  in dosoftint_prolog()
1021 	t = cpu->cpu_thread;  in dosoftint_prolog()
1025 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;  in dosoftint_prolog()
1049 	cpu->cpu_thread = it;  in dosoftint_prolog()
1053 	 * Set bit for this pil in CPU's interrupt active bitmask.  in dosoftint_prolog()
1055 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);  in dosoftint_prolog()
1056 	cpu->cpu_intr_actv |= (1 << pil);  in dosoftint_prolog()
1069 dosoftint_epilog(struct cpu *cpu, uint_t oldpil)  in dosoftint_epilog()  argument
1071 	struct machcpu *mcpu = &cpu->cpu_m;  in dosoftint_epilog()
1077 	it = cpu->cpu_thread;  in dosoftint_epilog()
1080 	cpu->cpu_stats.sys.intr[pil - 1]++;  in dosoftint_epilog()
1082 	ASSERT(cpu->cpu_intr_actv & (1 << pil));  in dosoftint_epilog()
1083 	cpu->cpu_intr_actv &= ~(1 << pil);  in dosoftint_epilog()
1086 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;  in dosoftint_epilog()
1096 		 * This was an interrupt thread, so set CPU's base SPL.  in dosoftint_epilog()
1100 		it->t_link = cpu->cpu_intr_thread;  in dosoftint_epilog()
1101 		cpu->cpu_intr_thread = it;  in dosoftint_epilog()
1108 	it->t_link = cpu->cpu_intr_thread;  in dosoftint_epilog()
1109 	cpu->cpu_intr_thread = it;  in dosoftint_epilog()
1112 	cpu->cpu_thread = t;  in dosoftint_epilog()
1116 	basespl = cpu->cpu_base_spl;  in dosoftint_epilog()
1146  * Create interrupt kstats for this CPU.
1164 	intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",  in cpu_create_intrstat()
1188  * Delete interrupt kstats for this CPU.
1193 	kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);  in cpu_delete_intrstat()
1197  * Convert interrupt statistics from CPU ticks to nanoseconds and
1230 	cpu_t		*cpu;  in cpu_intr_swtch_enter()  local
1253 		cpu = CPU;  in cpu_intr_swtch_enter()
1254 		cpu->cpu_m.intrstat[t->t_pil][0] += interval;  in cpu_intr_swtch_enter()
1256 		atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],  in cpu_intr_swtch_enter()
1298 	struct cpu *cpu = CPU;  in dispatch_softint()  local
1301 	av_dispatch_softvect((int)cpu->cpu_thread->t_pil);  in dispatch_softint()
1308 	dosoftint_epilog(cpu, oldpil);  in dispatch_softint()
1317 	struct cpu *cpu = CPU;  in dispatch_hardint()  local
1327 	intr_thread_epilog(cpu, vector, oldipl);  in dispatch_hardint()
1337 	struct cpu *cpu = CPU;  in dosoftint()  local
1341 	while (cpu->cpu_softinfo.st_pending) {  in dosoftint()
1342 		oldipl = cpu->cpu_pri;  in dosoftint()
1343 		newsp = dosoftint_prolog(cpu, (caddr_t)regs,  in dosoftint()
1344 		    cpu->cpu_softinfo.st_pending, oldipl);  in dosoftint()
1363 	struct cpu *cpu = CPU;  in do_interrupt()  local
1364 	int newipl, oldipl = cpu->cpu_pri;  in do_interrupt()
1372 	ttp->ttr_spl = cpu->cpu_base_spl;  in do_interrupt()
1378 	++*(uint16_t *)&cpu->cpu_m.mcpu_istamp;  in do_interrupt()
1402 	cpu->cpu_pri = newipl;  in do_interrupt()
1409 		 * High priority interrupts run on this cpu's interrupt stack.  in do_interrupt()
1411 		if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) {  in do_interrupt()
1412 			newsp = cpu->cpu_intr_stack;  in do_interrupt()
1417 		(void) hilevel_intr_epilog(cpu, newipl, oldipl, vector);  in do_interrupt()
1422 		newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl);  in do_interrupt()
1430 	if (cpu->cpu_softinfo.st_pending)  in do_interrupt()
1454 	tp = CPU->cpu_thread;  in sys_rtt_common()
1467 			trap(rp, (caddr_t)0, CPU->cpu_id);  in sys_rtt_common()
1540 	if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {  in sys_rtt_common()
1588 	    bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri)))
1593  * We are careful not to set priority lower than CPU->cpu_base_pri,
1596  * and look at CPU->cpu_base_pri
1602 	cpu_t	*cpu;  in do_splx()  local
1606 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */  in do_splx()
1607 	curpri = cpu->cpu_m.mcpu_pri;  in do_splx()
1608 	basepri = cpu->cpu_base_spl;  in do_splx()
1611 	cpu->cpu_m.mcpu_pri = newpri;  in do_splx()
1632 	cpu_t	*cpu;  in splr()  local
1636 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */  in splr()
1637 	curpri = cpu->cpu_m.mcpu_pri;  in splr()
1642 		basepri = cpu->cpu_base_spl;  in splr()
1645 		cpu->cpu_m.mcpu_pri = newpri;  in splr()
1660 	return (CPU->cpu_m.mcpu_pri);  in getpil()