1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #pragma ident "%Z%%M% %I% %E% SMI"
27
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/sysmacros.h>
31 #include <sys/systm.h>
32 #include <sys/cmn_err.h>
33 #include <sys/debug.h>
34 #include <sys/inline.h>
35 #include <sys/disp.h>
36 #include <sys/kmem.h>
37 #include <sys/cpuvar.h>
38 #include <sys/vtrace.h>
39 #include <sys/lockstat.h>
40 #include <sys/spl.h>
41 #include <sys/atomic.h>
42 #include <sys/cpu.h>
43
44 /*
45 * We check CPU_ON_INTR(CPU) when exiting a disp lock, rather than when
46 * entering it, for a purely pragmatic reason: when exiting a disp lock
47 * we know that we must be at PIL 10, and thus not preemptible; therefore
48 * we can safely load the CPU pointer without worrying about it changing.
49 */
50 static void
disp_onintr_panic(void)51 disp_onintr_panic(void)
52 {
53 panic("dispatcher invoked from high-level interrupt handler");
54 }
55
56 /* ARGSUSED */
57 void
disp_lock_init(disp_lock_t * lp,char * name)58 disp_lock_init(disp_lock_t *lp, char *name)
59 {
60 DISP_LOCK_INIT(lp);
61 }
62
63 /* ARGSUSED */
64 void
disp_lock_destroy(disp_lock_t * lp)65 disp_lock_destroy(disp_lock_t *lp)
66 {
67 DISP_LOCK_DESTROY(lp);
68 }
69
70 void
disp_lock_enter_high(disp_lock_t * lp)71 disp_lock_enter_high(disp_lock_t *lp)
72 {
73 lock_set(lp);
74 }
75
76 void
disp_lock_exit_high(disp_lock_t * lp)77 disp_lock_exit_high(disp_lock_t *lp)
78 {
79 if (CPU_ON_INTR(CPU) != 0)
80 disp_onintr_panic();
81 ASSERT(DISP_LOCK_HELD(lp));
82 lock_clear(lp);
83 }
84
85 void
disp_lock_enter(disp_lock_t * lp)86 disp_lock_enter(disp_lock_t *lp)
87 {
88 lock_set_spl(lp, ipltospl(DISP_LEVEL), &curthread->t_oldspl);
89 }
90
91 void
disp_lock_exit(disp_lock_t * lp)92 disp_lock_exit(disp_lock_t *lp)
93 {
94 if (CPU_ON_INTR(CPU) != 0)
95 disp_onintr_panic();
96 ASSERT(DISP_LOCK_HELD(lp));
97 if (CPU->cpu_kprunrun) {
98 lock_clear_splx(lp, curthread->t_oldspl);
99 kpreempt(KPREEMPT_SYNC);
100 } else {
101 lock_clear_splx(lp, curthread->t_oldspl);
102 }
103 }
104
105 void
disp_lock_exit_nopreempt(disp_lock_t * lp)106 disp_lock_exit_nopreempt(disp_lock_t *lp)
107 {
108 if (CPU_ON_INTR(CPU) != 0)
109 disp_onintr_panic();
110 ASSERT(DISP_LOCK_HELD(lp));
111 lock_clear_splx(lp, curthread->t_oldspl);
112 }
113
114 /*
115 * Thread_lock() - get the correct dispatcher lock for the thread.
116 */
117 void
thread_lock(kthread_id_t t)118 thread_lock(kthread_id_t t)
119 {
120 int s = splhigh();
121
122 if (CPU_ON_INTR(CPU) != 0)
123 disp_onintr_panic();
124
125 for (;;) {
126 lock_t *volatile *tlpp = &t->t_lockp;
127 lock_t *lp = *tlpp;
128 if (lock_try(lp)) {
129 if (lp == *tlpp) {
130 curthread->t_oldspl = (ushort_t)s;
131 return;
132 }
133 lock_clear(lp);
134 } else {
135 hrtime_t spin_time =
136 LOCKSTAT_START_TIME(LS_THREAD_LOCK_SPIN);
137 /*
138 * Lower spl and spin on lock with non-atomic load
139 * to avoid cache activity. Spin until the lock
140 * becomes available or spontaneously changes.
141 */
142 splx(s);
143 while (lp == *tlpp && LOCK_HELD(lp)) {
144 if (panicstr) {
145 curthread->t_oldspl = splhigh();
146 return;
147 }
148 SMT_PAUSE();
149 }
150
151 LOCKSTAT_RECORD_TIME(LS_THREAD_LOCK_SPIN,
152 lp, spin_time);
153 s = splhigh();
154 }
155 }
156 }
157
158 /*
159 * Thread_lock_high() - get the correct dispatcher lock for the thread.
160 * This version is called when already at high spl.
161 */
162 void
thread_lock_high(kthread_id_t t)163 thread_lock_high(kthread_id_t t)
164 {
165 if (CPU_ON_INTR(CPU) != 0)
166 disp_onintr_panic();
167
168 for (;;) {
169 lock_t *volatile *tlpp = &t->t_lockp;
170 lock_t *lp = *tlpp;
171 if (lock_try(lp)) {
172 if (lp == *tlpp)
173 return;
174 lock_clear(lp);
175 } else {
176 hrtime_t spin_time =
177 LOCKSTAT_START_TIME(LS_THREAD_LOCK_HIGH_SPIN);
178 while (lp == *tlpp && LOCK_HELD(lp)) {
179 if (panicstr)
180 return;
181 SMT_PAUSE();
182 }
183 LOCKSTAT_RECORD_TIME(LS_THREAD_LOCK_HIGH_SPIN,
184 lp, spin_time);
185 }
186 }
187 }
188
189 /*
190 * Called by THREAD_TRANSITION macro to change the thread state to
191 * the intermediate state-in-transititon state.
192 */
193 void
thread_transition(kthread_id_t t)194 thread_transition(kthread_id_t t)
195 {
196 disp_lock_t *lp;
197
198 ASSERT(THREAD_LOCK_HELD(t));
199 ASSERT(t->t_lockp != &transition_lock);
200
201 lp = t->t_lockp;
202 t->t_lockp = &transition_lock;
203 disp_lock_exit_high(lp);
204 }
205
206 /*
207 * Put thread in stop state, and set the lock pointer to the stop_lock.
208 * This effectively drops the lock on the thread, since the stop_lock
209 * isn't held.
210 * Eventually, stop_lock could be hashed if there is too much contention.
211 */
212 void
thread_stop(kthread_id_t t)213 thread_stop(kthread_id_t t)
214 {
215 disp_lock_t *lp;
216
217 ASSERT(THREAD_LOCK_HELD(t));
218 ASSERT(t->t_lockp != &stop_lock);
219
220 lp = t->t_lockp;
221 t->t_state = TS_STOPPED;
222 /*
223 * Ensure that t_state reaches global visibility before t_lockp
224 */
225 membar_producer();
226 t->t_lockp = &stop_lock;
227 disp_lock_exit(lp);
228 }
229