1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright (c) 2010, Intel Corporation.
26 * All rights reserved.
27 */
28 /*
29 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
30 */
31
32 #include <sys/time.h>
33 #include <sys/psm.h>
34 #include <sys/psm_common.h>
35 #include <sys/apic.h>
36 #include <sys/pit.h>
37 #include <sys/x86_archext.h>
38 #include <sys/archsystm.h>
39 #include <sys/machsystm.h>
40 #include <sys/cpuvar.h>
41 #include <sys/clock.h>
42 #include <sys/apic_timer.h>
43
44 /*
45 * preferred apic timer mode, allow tuning from the /etc/system file.
46 */
47 int apic_timer_preferred_mode = APIC_TIMER_MODE_DEADLINE;
48
49 int apic_oneshot = 0;
50 uint_t apic_hertz_count;
51 uint_t apic_nsec_per_intr = 0;
52 uint64_t apic_ticks_per_SFnsecs; /* # of ticks in SF nsecs */
53
54 static int apic_min_timer_ticks = 1; /* minimum timer tick */
55 static hrtime_t apic_nsec_max;
56
57 static void periodic_timer_enable(void);
58 static void periodic_timer_disable(void);
59 static void periodic_timer_reprogram(hrtime_t);
60 static void oneshot_timer_enable(void);
61 static void oneshot_timer_disable(void);
62 static void oneshot_timer_reprogram(hrtime_t);
63 static void deadline_timer_enable(void);
64 static void deadline_timer_disable(void);
65 static void deadline_timer_reprogram(hrtime_t);
66
67 extern int apic_clkvect;
68 extern uint32_t apic_divide_reg_init;
69
70 /*
71 * apic timer data structure
72 */
73 typedef struct apic_timer {
74 int mode;
75 void (*apic_timer_enable_ops)(void);
76 void (*apic_timer_disable_ops)(void);
77 void (*apic_timer_reprogram_ops)(hrtime_t);
78 } apic_timer_t;
79
80 static apic_timer_t apic_timer;
81
82 /*
83 * apic timer initialization
84 *
85 * For the one-shot mode request case, the function returns the
86 * resolution (in nanoseconds) for the hardware timer interrupt.
87 * If one-shot mode capability is not available, the return value
88 * will be 0.
89 */
90 int
apic_timer_init(int hertz)91 apic_timer_init(int hertz)
92 {
93 uint_t apic_ticks = 0;
94 uint_t pit_ticks;
95 int ret, timer_mode;
96 uint16_t pit_ticks_adj;
97 static int firsttime = 1;
98
99 if (firsttime) {
100 /* first time calibrate on CPU0 only */
101
102 apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
103 apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
104 apic_ticks = apic_calibrate(apicadr, &pit_ticks_adj);
105
106 /* total number of PIT ticks corresponding to apic_ticks */
107 pit_ticks = APIC_TIME_COUNT + pit_ticks_adj;
108
109 /*
110 * Determine the number of nanoseconds per APIC clock tick
111 * and then determine how many APIC ticks to interrupt at the
112 * desired frequency
113 * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s
114 * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s
115 * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9)
116 * pic_ticks_per_SFns =
117 * (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9)
118 */
119 apic_ticks_per_SFnsecs = ((SF * apic_ticks * PIT_HZ) /
120 ((uint64_t)pit_ticks * NANOSEC));
121
122 /* the interval timer initial count is 32 bit max */
123 apic_nsec_max = APIC_TICKS_TO_NSECS(APIC_MAXVAL);
124 firsttime = 0;
125 }
126
127 if (hertz == 0) {
128 /* requested one_shot */
129
130 /*
131 * return 0 if TSC is not supported.
132 */
133 if (!tsc_gethrtime_enable)
134 return (0);
135 /*
136 * return 0 if one_shot is not preferred.
137 * here, APIC_TIMER_DEADLINE is also an one_shot mode.
138 */
139 if ((apic_timer_preferred_mode != APIC_TIMER_MODE_ONESHOT) &&
140 (apic_timer_preferred_mode != APIC_TIMER_MODE_DEADLINE))
141 return (0);
142
143 apic_oneshot = 1;
144 ret = (int)APIC_TICKS_TO_NSECS(1);
145 if ((apic_timer_preferred_mode == APIC_TIMER_MODE_DEADLINE) &&
146 cpuid_deadline_tsc_supported()) {
147 timer_mode = APIC_TIMER_MODE_DEADLINE;
148 } else {
149 timer_mode = APIC_TIMER_MODE_ONESHOT;
150 }
151 } else {
152 /* periodic */
153 apic_nsec_per_intr = NANOSEC / hertz;
154 apic_hertz_count = APIC_NSECS_TO_TICKS(apic_nsec_per_intr);
155
156 /* program the local APIC to interrupt at the given frequency */
157 apic_reg_ops->apic_write(APIC_INIT_COUNT, apic_hertz_count);
158 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
159 (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC);
160 apic_oneshot = 0;
161 timer_mode = APIC_TIMER_MODE_PERIODIC;
162 ret = NANOSEC / hertz;
163 }
164
165 /*
166 * initialize apic_timer data structure, install the timer ops
167 */
168 apic_timer.mode = timer_mode;
169 switch (timer_mode) {
170 default:
171 /* FALLTHROUGH */
172 case APIC_TIMER_MODE_ONESHOT:
173 apic_timer.apic_timer_enable_ops = oneshot_timer_enable;
174 apic_timer.apic_timer_disable_ops = oneshot_timer_disable;
175 apic_timer.apic_timer_reprogram_ops = oneshot_timer_reprogram;
176 break;
177
178 case APIC_TIMER_MODE_PERIODIC:
179 apic_timer.apic_timer_enable_ops = periodic_timer_enable;
180 apic_timer.apic_timer_disable_ops = periodic_timer_disable;
181 apic_timer.apic_timer_reprogram_ops = periodic_timer_reprogram;
182 break;
183
184 case APIC_TIMER_MODE_DEADLINE:
185 apic_timer.apic_timer_enable_ops = deadline_timer_enable;
186 apic_timer.apic_timer_disable_ops = deadline_timer_disable;
187 apic_timer.apic_timer_reprogram_ops = deadline_timer_reprogram;
188 break;
189 }
190
191 return (ret);
192 }
193
194 /*
195 * periodic timer mode ops
196 */
197 /* periodic timer enable */
198 static void
periodic_timer_enable(void)199 periodic_timer_enable(void)
200 {
201 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
202 (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC);
203 }
204
205 /* periodic timer disable */
206 static void
periodic_timer_disable(void)207 periodic_timer_disable(void)
208 {
209 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
210 (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
211 }
212
213 /* periodic timer reprogram */
214 static void
periodic_timer_reprogram(hrtime_t time)215 periodic_timer_reprogram(hrtime_t time)
216 {
217 uint_t ticks;
218 /* time is the interval for periodic mode */
219 ticks = APIC_NSECS_TO_TICKS(time);
220
221 if (ticks < apic_min_timer_ticks)
222 ticks = apic_min_timer_ticks;
223
224 apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks);
225 }
226
227 /*
228 * oneshot timer mode ops
229 */
230 /* oneshot timer enable */
231 static void
oneshot_timer_enable(void)232 oneshot_timer_enable(void)
233 {
234 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
235 (apic_clkvect + APIC_BASE_VECT));
236 }
237
238 /* oneshot timer disable */
239 static void
oneshot_timer_disable(void)240 oneshot_timer_disable(void)
241 {
242 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
243 (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
244 }
245
246 /* oneshot timer reprogram */
247 static void
oneshot_timer_reprogram(hrtime_t time)248 oneshot_timer_reprogram(hrtime_t time)
249 {
250 hrtime_t now;
251 int64_t delta;
252 uint_t ticks;
253
254 now = gethrtime();
255 delta = time - now;
256
257 if (delta <= 0) {
258 /*
259 * requested to generate an interrupt in the past
260 * generate an interrupt as soon as possible
261 */
262 ticks = apic_min_timer_ticks;
263 } else if (delta > apic_nsec_max) {
264 /*
265 * requested to generate an interrupt at a time
266 * further than what we are capable of. Set to max
267 * the hardware can handle
268 */
269 ticks = APIC_MAXVAL;
270 #ifdef DEBUG
271 cmn_err(CE_CONT, "apic_timer_reprogram, request at"
272 " %lld too far in future, current time"
273 " %lld \n", time, now);
274 #endif
275 } else {
276 ticks = APIC_NSECS_TO_TICKS(delta);
277 }
278
279 if (ticks < apic_min_timer_ticks)
280 ticks = apic_min_timer_ticks;
281
282 apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks);
283 }
284
285 /*
286 * deadline timer mode ops
287 */
288 /* deadline timer enable */
289 static void
deadline_timer_enable(void)290 deadline_timer_enable(void)
291 {
292 uint64_t ticks;
293
294 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
295 (apic_clkvect + APIC_BASE_VECT) | AV_DEADLINE);
296 /*
297 * Now we have to serialize this per the SDM. That is to
298 * say, the above enabling can race in the pipeline with
299 * changes to the MSR. We need to make sure the above
300 * operation is complete before we proceed to reprogram
301 * the deadline value in reprogram(). The algorithm
302 * recommended by the Intel SDM 3A in 10.5.1.4 is:
303 *
304 * a) write a big value to the deadline register
305 * b) read the register back
306 * c) if it reads zero, go back to a and try again
307 */
308
309 do {
310 /* write a really big value */
311 wrmsr(IA32_DEADLINE_TSC_MSR, 1ULL << 63);
312 ticks = rdmsr(IA32_DEADLINE_TSC_MSR);
313 } while (ticks == 0);
314 }
315
316 /* deadline timer disable */
317 static void
deadline_timer_disable(void)318 deadline_timer_disable(void)
319 {
320 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
321 (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
322 }
323
324 /* deadline timer reprogram */
325 static void
deadline_timer_reprogram(hrtime_t time)326 deadline_timer_reprogram(hrtime_t time)
327 {
328 int64_t delta;
329 uint64_t ticks;
330
331 /*
332 * Note that this entire routine is called with
333 * CBE_HIGH_PIL, so we needn't worry about preemption.
334 */
335 delta = time - gethrtime();
336
337 /* The unscalehrtime wants unsigned values. */
338 delta = max(delta, 0);
339
340 /* Now we shouldn't be interrupted, we can set the deadline */
341 ticks = (uint64_t)tsc_read() + unscalehrtime(delta);
342 wrmsr(IA32_DEADLINE_TSC_MSR, ticks);
343 }
344
345 /*
346 * This function will reprogram the timer.
347 *
348 * When in oneshot mode the argument is the absolute time in future to
349 * generate the interrupt at.
350 *
351 * When in periodic mode, the argument is the interval at which the
352 * interrupts should be generated. There is no need to support the periodic
353 * mode timer change at this time.
354 */
355 void
apic_timer_reprogram(hrtime_t time)356 apic_timer_reprogram(hrtime_t time)
357 {
358 /*
359 * we should be Called from high PIL context (CBE_HIGH_PIL),
360 * so kpreempt is disabled.
361 */
362 apic_timer.apic_timer_reprogram_ops(time);
363 }
364
365 /*
366 * This function will enable timer interrupts.
367 */
368 void
apic_timer_enable(void)369 apic_timer_enable(void)
370 {
371 /*
372 * we should be Called from high PIL context (CBE_HIGH_PIL),
373 * so kpreempt is disabled.
374 */
375 apic_timer.apic_timer_enable_ops();
376 }
377
378 /*
379 * This function will disable timer interrupts.
380 */
381 void
apic_timer_disable(void)382 apic_timer_disable(void)
383 {
384 /*
385 * we should be Called from high PIL context (CBE_HIGH_PIL),
386 * so kpreempt is disabled.
387 */
388 apic_timer.apic_timer_disable_ops();
389 }
390
391 /*
392 * Set timer far into the future and return timer
393 * current count in nanoseconds.
394 */
395 hrtime_t
apic_timer_stop_count(void)396 apic_timer_stop_count(void)
397 {
398 hrtime_t ns_val;
399 int enable_val, count_val;
400
401 /*
402 * Should be called with interrupts disabled.
403 */
404 ASSERT(!interrupts_enabled());
405
406 enable_val = apic_reg_ops->apic_read(APIC_LOCAL_TIMER);
407 if ((enable_val & AV_MASK) == AV_MASK)
408 return ((hrtime_t)-1); /* timer is disabled */
409
410 count_val = apic_reg_ops->apic_read(APIC_CURR_COUNT);
411 ns_val = APIC_TICKS_TO_NSECS(count_val);
412
413 apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
414
415 return (ns_val);
416 }
417
418 /*
419 * Reprogram timer after Deep C-State.
420 */
421 void
apic_timer_restart(hrtime_t time)422 apic_timer_restart(hrtime_t time)
423 {
424 apic_timer_reprogram(time);
425 }
426