xref: /titanic_44/usr/src/uts/i86pc/io/cbe.c (revision a38ddfee9c8c6b6c5a2947ff52fd2338362a4444)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/systm.h>
28 #include <sys/cyclic.h>
29 #include <sys/cyclic_impl.h>
30 #include <sys/spl.h>
31 #include <sys/x_call.h>
32 #include <sys/kmem.h>
33 #include <sys/machsystm.h>
34 #include <sys/smp_impldefs.h>
35 #include <sys/psm_types.h>
36 #include <sys/psm.h>
37 #include <sys/atomic.h>
38 #include <sys/clock.h>
39 #include <sys/x86_archext.h>
40 #include <sys/ddi_impldefs.h>
41 #include <sys/ddi_intr.h>
42 #include <sys/avintr.h>
43 
44 static int cbe_vector;
45 static int cbe_ticks = 0;
46 
47 /*
48  * cbe_xcall_lock is used to protect the xcall globals since the cyclic
49  * reprogramming API does not use cpu_lock.
50  */
51 static kmutex_t cbe_xcall_lock;
52 static cyc_func_t volatile cbe_xcall_func;
53 static cpu_t *volatile cbe_xcall_cpu;
54 static void *cbe_xcall_farg;
55 static cpuset_t cbe_enabled;
56 
57 static ddi_softint_hdl_impl_t cbe_low_hdl =
58 	{0, NULL, NULL, NULL, 0, NULL, NULL, NULL};
59 static ddi_softint_hdl_impl_t cbe_clock_hdl =
60 	{0, NULL, NULL, NULL, 0, NULL, NULL, NULL};
61 
62 cyclic_id_t cbe_hres_cyclic;
63 int cbe_psm_timer_mode = TIMER_ONESHOT;
64 static hrtime_t cbe_timer_resolution;
65 
66 extern int tsc_gethrtime_enable;
67 
68 void cbe_hres_tick(void);
69 
70 int
71 cbe_softclock(void)
72 {
73 	cyclic_softint(CPU, CY_LOCK_LEVEL);
74 	return (1);
75 }
76 
77 int
78 cbe_low_level(void)
79 {
80 	cpu_t *cpu = CPU;
81 
82 	cyclic_softint(cpu, CY_LOW_LEVEL);
83 	return (1);
84 }
85 
86 /*
87  * We can be in cbe_fire() either due to a cyclic-induced cross call, or due
88  * to the timer firing at level-14.  Because cyclic_fire() can tolerate
89  * spurious calls, it would not matter if we called cyclic_fire() in both
90  * cases.
91  */
92 int
93 cbe_fire(void)
94 {
95 	cpu_t *cpu = CPU;
96 	processorid_t me = cpu->cpu_id, i;
97 	int cross_call = (cbe_xcall_func != NULL && cbe_xcall_cpu == cpu);
98 
99 	cyclic_fire(cpu);
100 
101 	if (cbe_psm_timer_mode != TIMER_ONESHOT && me == 0 && !cross_call) {
102 		for (i = 1; i < NCPU; i++) {
103 			if (CPU_IN_SET(cbe_enabled, i)) {
104 				XC_TRACE(TT_XC_CBE_FIRE, -1, i);
105 				send_dirint(i, CBE_HIGH_PIL);
106 			}
107 		}
108 	}
109 
110 	if (cross_call) {
111 		ASSERT(cbe_xcall_func != NULL && cbe_xcall_cpu == cpu);
112 		(*cbe_xcall_func)(cbe_xcall_farg);
113 		cbe_xcall_func = NULL;
114 		cbe_xcall_cpu = NULL;
115 	}
116 
117 	return (1);
118 }
119 
120 /*ARGSUSED*/
121 void
122 cbe_softint(void *arg, cyc_level_t level)
123 {
124 	switch (level) {
125 	case CY_LOW_LEVEL:
126 		(*setsoftint)(CBE_LOW_PIL, cbe_low_hdl.ih_pending);
127 		break;
128 	case CY_LOCK_LEVEL:
129 		(*setsoftint)(CBE_LOCK_PIL, cbe_clock_hdl.ih_pending);
130 		break;
131 	default:
132 		panic("cbe_softint: unexpected soft level %d", level);
133 	}
134 }
135 
136 /*ARGSUSED*/
137 void
138 cbe_reprogram(void *arg, hrtime_t time)
139 {
140 	if (cbe_psm_timer_mode == TIMER_ONESHOT)
141 		(*psm_timer_reprogram)(time);
142 }
143 
144 /*ARGSUSED*/
145 cyc_cookie_t
146 cbe_set_level(void *arg, cyc_level_t level)
147 {
148 	int ipl;
149 
150 	switch (level) {
151 	case CY_LOW_LEVEL:
152 		ipl = CBE_LOW_PIL;
153 		break;
154 	case CY_LOCK_LEVEL:
155 		ipl = CBE_LOCK_PIL;
156 		break;
157 	case CY_HIGH_LEVEL:
158 		ipl = CBE_HIGH_PIL;
159 		break;
160 	default:
161 		panic("cbe_set_level: unexpected level %d", level);
162 	}
163 
164 	return (splr(ipltospl(ipl)));
165 }
166 
167 /*ARGSUSED*/
168 void
169 cbe_restore_level(void *arg, cyc_cookie_t cookie)
170 {
171 	splx(cookie);
172 }
173 
174 /*ARGSUSED*/
175 void
176 cbe_xcall(void *arg, cpu_t *dest, cyc_func_t func, void *farg)
177 {
178 	kpreempt_disable();
179 
180 	if (dest == CPU) {
181 		(*func)(farg);
182 		kpreempt_enable();
183 		return;
184 	}
185 
186 	mutex_enter(&cbe_xcall_lock);
187 
188 	ASSERT(cbe_xcall_func == NULL);
189 
190 	cbe_xcall_farg = farg;
191 	membar_producer();
192 	cbe_xcall_cpu = dest;
193 	cbe_xcall_func = func;
194 
195 	XC_TRACE(TT_XC_CBE_XCALL, -1, dest->cpu_id);
196 	send_dirint(dest->cpu_id, CBE_HIGH_PIL);
197 
198 	while (cbe_xcall_func != NULL || cbe_xcall_cpu != NULL)
199 		continue;
200 
201 	mutex_exit(&cbe_xcall_lock);
202 
203 	kpreempt_enable();
204 
205 	ASSERT(cbe_xcall_func == NULL && cbe_xcall_cpu == NULL);
206 }
207 
208 void *
209 cbe_configure(cpu_t *cpu)
210 {
211 	return (cpu);
212 }
213 
214 #ifndef __xpv
215 /*
216  * declarations needed for time adjustment
217  */
218 extern void	tsc_suspend(void);
219 extern void	tsc_resume(void);
220 /*
221  * Call the resume function in the cyclic, instead of inline in the
222  * resume path.
223  */
224 extern int	tsc_resume_in_cyclic;
225 #endif
226 
227 /*ARGSUSED*/
228 static void
229 cbe_suspend(cyb_arg_t arg)
230 {
231 #ifndef __xpv
232 	/*
233 	 * This is an x86 backend, so let the tsc_suspend
234 	 * that is specific to x86 platforms do the work.
235 	 */
236 	tsc_suspend();
237 #endif
238 }
239 
240 /*ARGSUSED*/
241 static void
242 cbe_resume(cyb_arg_t arg)
243 {
244 #ifndef __xpv
245 	if (tsc_resume_in_cyclic) {
246 		tsc_resume();
247 	}
248 #endif
249 }
250 
251 void
252 cbe_enable(void *arg)
253 {
254 	processorid_t me = ((cpu_t *)arg)->cpu_id;
255 
256 	/* neither enable nor disable cpu0 if TIMER_PERIODIC is set */
257 	if ((cbe_psm_timer_mode != TIMER_ONESHOT) && (me == 0))
258 		return;
259 
260 	/*
261 	 * Added (me == 0) to the ASSERT because the timer isn't
262 	 * disabled on CPU 0, and cbe_enable is called when we resume.
263 	 */
264 	ASSERT((me == 0) || !CPU_IN_SET(cbe_enabled, me));
265 	CPUSET_ADD(cbe_enabled, me);
266 	if (cbe_psm_timer_mode == TIMER_ONESHOT)
267 		(*psm_timer_enable)();
268 }
269 
270 void
271 cbe_disable(void *arg)
272 {
273 	processorid_t me = ((cpu_t *)arg)->cpu_id;
274 
275 	/* neither enable nor disable cpu0 if TIMER_PERIODIC is set */
276 	if ((cbe_psm_timer_mode != TIMER_ONESHOT) && (me == 0))
277 		return;
278 
279 	ASSERT(CPU_IN_SET(cbe_enabled, me));
280 	CPUSET_DEL(cbe_enabled, me);
281 	if (cbe_psm_timer_mode == TIMER_ONESHOT)
282 		(*psm_timer_disable)();
283 }
284 
285 /*
286  * Unbound cyclic, called once per tick (every nsec_per_tick ns).
287  */
288 void
289 cbe_hres_tick(void)
290 {
291 	int s;
292 
293 	dtrace_hres_tick();
294 
295 	/*
296 	 * Because hres_tick effectively locks hres_lock, we must be at the
297 	 * same PIL as that used for CLOCK_LOCK.
298 	 */
299 	s = splr(ipltospl(XC_HI_PIL));
300 	hres_tick();
301 	splx(s);
302 
303 	if ((cbe_ticks % hz) == 0)
304 		(*hrtime_tick)();
305 
306 	cbe_ticks++;
307 
308 }
309 
310 void
311 cbe_init_pre(void)
312 {
313 	cbe_vector = (*psm_get_clockirq)(CBE_HIGH_PIL);
314 
315 	CPUSET_ZERO(cbe_enabled);
316 
317 	cbe_timer_resolution = (*clkinitf)(TIMER_ONESHOT, &cbe_psm_timer_mode);
318 }
319 
320 void
321 cbe_init(void)
322 {
323 	cyc_backend_t cbe = {
324 		cbe_configure,		/* cyb_configure */
325 		NULL,			/* cyb_unconfigure */
326 		cbe_enable,		/* cyb_enable */
327 		cbe_disable,		/* cyb_disable */
328 		cbe_reprogram,		/* cyb_reprogram */
329 		cbe_softint,		/* cyb_softint */
330 		cbe_set_level,		/* cyb_set_level */
331 		cbe_restore_level,	/* cyb_restore_level */
332 		cbe_xcall,		/* cyb_xcall */
333 		cbe_suspend,		/* cyb_suspend */
334 		cbe_resume		/* cyb_resume */
335 	};
336 	cyc_handler_t hdlr;
337 	cyc_time_t when;
338 
339 	mutex_init(&cbe_xcall_lock, NULL, MUTEX_DEFAULT, NULL);
340 
341 	mutex_enter(&cpu_lock);
342 	cyclic_init(&cbe, cbe_timer_resolution);
343 	mutex_exit(&cpu_lock);
344 
345 	(void) add_avintr(NULL, CBE_HIGH_PIL, (avfunc)cbe_fire,
346 	    "cbe_fire_master", cbe_vector, 0, NULL, NULL, NULL);
347 
348 	if (psm_get_ipivect != NULL) {
349 		(void) add_avintr(NULL, CBE_HIGH_PIL, (avfunc)cbe_fire,
350 		    "cbe_fire_slave",
351 		    (*psm_get_ipivect)(CBE_HIGH_PIL, PSM_INTR_IPI_HI),
352 		    0, NULL, NULL, NULL);
353 	}
354 
355 	(void) add_avsoftintr((void *)&cbe_clock_hdl, CBE_LOCK_PIL,
356 	    (avfunc)cbe_softclock, "softclock", NULL, NULL);
357 
358 	(void) add_avsoftintr((void *)&cbe_low_hdl, CBE_LOW_PIL,
359 	    (avfunc)cbe_low_level, "low level", NULL, NULL);
360 
361 	mutex_enter(&cpu_lock);
362 
363 	hdlr.cyh_level = CY_HIGH_LEVEL;
364 	hdlr.cyh_func = (cyc_func_t)cbe_hres_tick;
365 	hdlr.cyh_arg = NULL;
366 
367 	when.cyt_when = 0;
368 	when.cyt_interval = nsec_per_tick;
369 
370 	cbe_hres_cyclic = cyclic_add(&hdlr, &when);
371 
372 	if (psm_post_cyclic_setup != NULL)
373 		(*psm_post_cyclic_setup)(NULL);
374 
375 	mutex_exit(&cpu_lock);
376 }
377