1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2017 by Delphix. All rights reserved.
24 */
25 /*
26 * Copyright (c) 2010, Intel Corporation.
27 * All rights reserved.
28 */
29 /*
30 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
31 */
32
33 #include <sys/time.h>
34 #include <sys/psm.h>
35 #include <sys/psm_common.h>
36 #include <sys/apic.h>
37 #include <sys/pit.h>
38 #include <sys/x86_archext.h>
39 #include <sys/archsystm.h>
40 #include <sys/machsystm.h>
41 #include <sys/cpuvar.h>
42 #include <sys/clock.h>
43 #include <sys/apic_timer.h>
44
45 /*
46 * preferred apic timer mode, allow tuning from the /etc/system file.
47 */
48 int apic_timer_preferred_mode = APIC_TIMER_MODE_DEADLINE;
49
50 int apic_oneshot = 0;
51 uint_t apic_hertz_count;
52 uint_t apic_nsec_per_intr = 0;
53 uint64_t apic_ticks_per_SFnsecs; /* # of ticks in SF nsecs */
54
55 static int apic_min_timer_ticks = 1; /* minimum timer tick */
56 static hrtime_t apic_nsec_max;
57
58 static void periodic_timer_enable(void);
59 static void periodic_timer_disable(void);
60 static void periodic_timer_reprogram(hrtime_t);
61 static void oneshot_timer_enable(void);
62 static void oneshot_timer_disable(void);
63 static void oneshot_timer_reprogram(hrtime_t);
64 static void deadline_timer_enable(void);
65 static void deadline_timer_disable(void);
66 static void deadline_timer_reprogram(hrtime_t);
67
68 extern int apic_clkvect;
69 extern uint32_t apic_divide_reg_init;
70
71 /*
72 * apic timer data structure
73 */
74 typedef struct apic_timer {
75 int mode;
76 void (*apic_timer_enable_ops)(void);
77 void (*apic_timer_disable_ops)(void);
78 void (*apic_timer_reprogram_ops)(hrtime_t);
79 } apic_timer_t;
80
81 static apic_timer_t apic_timer;
82
83 /*
84 * apic timer initialization
85 *
86 * For the one-shot mode request case, the function returns the
87 * resolution (in nanoseconds) for the hardware timer interrupt.
88 * If one-shot mode capability is not available, the return value
89 * will be 0.
90 */
91 int
apic_timer_init(int hertz)92 apic_timer_init(int hertz)
93 {
94 int ret, timer_mode;
95 static int firsttime = 1;
96
97 if (firsttime) {
98 /* first time calibrate on CPU0 only */
99 apic_ticks_per_SFnsecs = apic_calibrate();
100
101 /* the interval timer initial count is 32 bit max */
102 apic_nsec_max = APIC_TICKS_TO_NSECS(APIC_MAXVAL);
103 firsttime = 0;
104 }
105
106 if (hertz == 0) {
107 /* requested one_shot */
108
109 /*
110 * return 0 if TSC is not supported.
111 */
112 if (!tsc_gethrtime_enable)
113 return (0);
114 /*
115 * return 0 if one_shot is not preferred.
116 * here, APIC_TIMER_DEADLINE is also an one_shot mode.
117 */
118 if ((apic_timer_preferred_mode != APIC_TIMER_MODE_ONESHOT) &&
119 (apic_timer_preferred_mode != APIC_TIMER_MODE_DEADLINE))
120 return (0);
121
122 apic_oneshot = 1;
123 ret = (int)APIC_TICKS_TO_NSECS(1);
124 if ((apic_timer_preferred_mode == APIC_TIMER_MODE_DEADLINE) &&
125 cpuid_deadline_tsc_supported()) {
126 timer_mode = APIC_TIMER_MODE_DEADLINE;
127 } else {
128 timer_mode = APIC_TIMER_MODE_ONESHOT;
129 }
130 } else {
131 /* periodic */
132 apic_nsec_per_intr = NANOSEC / hertz;
133 apic_hertz_count = APIC_NSECS_TO_TICKS(apic_nsec_per_intr);
134
135 /* program the local APIC to interrupt at the given frequency */
136 apic_reg_ops->apic_write(APIC_INIT_COUNT, apic_hertz_count);
137 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
138 (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC);
139 apic_oneshot = 0;
140 timer_mode = APIC_TIMER_MODE_PERIODIC;
141 ret = NANOSEC / hertz;
142 }
143
144 /*
145 * initialize apic_timer data structure, install the timer ops
146 */
147 apic_timer.mode = timer_mode;
148 switch (timer_mode) {
149 default:
150 /* FALLTHROUGH */
151 case APIC_TIMER_MODE_ONESHOT:
152 apic_timer.apic_timer_enable_ops = oneshot_timer_enable;
153 apic_timer.apic_timer_disable_ops = oneshot_timer_disable;
154 apic_timer.apic_timer_reprogram_ops = oneshot_timer_reprogram;
155 break;
156
157 case APIC_TIMER_MODE_PERIODIC:
158 apic_timer.apic_timer_enable_ops = periodic_timer_enable;
159 apic_timer.apic_timer_disable_ops = periodic_timer_disable;
160 apic_timer.apic_timer_reprogram_ops = periodic_timer_reprogram;
161 break;
162
163 case APIC_TIMER_MODE_DEADLINE:
164 apic_timer.apic_timer_enable_ops = deadline_timer_enable;
165 apic_timer.apic_timer_disable_ops = deadline_timer_disable;
166 apic_timer.apic_timer_reprogram_ops = deadline_timer_reprogram;
167 break;
168 }
169
170 return (ret);
171 }
172
173 /*
174 * periodic timer mode ops
175 */
176 /* periodic timer enable */
177 static void
periodic_timer_enable(void)178 periodic_timer_enable(void)
179 {
180 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
181 (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC);
182 }
183
184 /* periodic timer disable */
185 static void
periodic_timer_disable(void)186 periodic_timer_disable(void)
187 {
188 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
189 (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
190 }
191
192 /* periodic timer reprogram */
193 static void
periodic_timer_reprogram(hrtime_t time)194 periodic_timer_reprogram(hrtime_t time)
195 {
196 uint_t ticks;
197 /* time is the interval for periodic mode */
198 ticks = APIC_NSECS_TO_TICKS(time);
199
200 if (ticks < apic_min_timer_ticks)
201 ticks = apic_min_timer_ticks;
202
203 apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks);
204 }
205
206 /*
207 * oneshot timer mode ops
208 */
209 /* oneshot timer enable */
210 static void
oneshot_timer_enable(void)211 oneshot_timer_enable(void)
212 {
213 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
214 (apic_clkvect + APIC_BASE_VECT));
215 }
216
217 /* oneshot timer disable */
218 static void
oneshot_timer_disable(void)219 oneshot_timer_disable(void)
220 {
221 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
222 (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
223 }
224
225 /* oneshot timer reprogram */
226 static void
oneshot_timer_reprogram(hrtime_t time)227 oneshot_timer_reprogram(hrtime_t time)
228 {
229 hrtime_t now;
230 int64_t delta;
231 uint_t ticks;
232
233 now = gethrtime();
234 delta = time - now;
235
236 if (delta <= 0) {
237 /*
238 * requested to generate an interrupt in the past
239 * generate an interrupt as soon as possible
240 */
241 ticks = apic_min_timer_ticks;
242 } else if (delta > apic_nsec_max) {
243 /*
244 * requested to generate an interrupt at a time
245 * further than what we are capable of. Set to max
246 * the hardware can handle
247 */
248 ticks = APIC_MAXVAL;
249 #ifdef DEBUG
250 cmn_err(CE_CONT, "apic_timer_reprogram, request at"
251 " %lld too far in future, current time"
252 " %lld \n", time, now);
253 #endif
254 } else {
255 ticks = APIC_NSECS_TO_TICKS(delta);
256 }
257
258 if (ticks < apic_min_timer_ticks)
259 ticks = apic_min_timer_ticks;
260
261 apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks);
262 }
263
264 /*
265 * deadline timer mode ops
266 */
267 /* deadline timer enable */
268 static void
deadline_timer_enable(void)269 deadline_timer_enable(void)
270 {
271 uint64_t ticks;
272
273 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
274 (apic_clkvect + APIC_BASE_VECT) | AV_DEADLINE);
275 /*
276 * Now we have to serialize this per the SDM. That is to
277 * say, the above enabling can race in the pipeline with
278 * changes to the MSR. We need to make sure the above
279 * operation is complete before we proceed to reprogram
280 * the deadline value in reprogram(). The algorithm
281 * recommended by the Intel SDM 3A in 10.5.1.4 is:
282 *
283 * a) write a big value to the deadline register
284 * b) read the register back
285 * c) if it reads zero, go back to a and try again
286 */
287
288 do {
289 /* write a really big value */
290 wrmsr(IA32_DEADLINE_TSC_MSR, 1ULL << 63);
291 ticks = rdmsr(IA32_DEADLINE_TSC_MSR);
292 } while (ticks == 0);
293 }
294
295 /* deadline timer disable */
296 static void
deadline_timer_disable(void)297 deadline_timer_disable(void)
298 {
299 apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
300 (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
301 }
302
303 /* deadline timer reprogram */
304 static void
deadline_timer_reprogram(hrtime_t time)305 deadline_timer_reprogram(hrtime_t time)
306 {
307 int64_t delta;
308 uint64_t ticks;
309
310 /*
311 * Note that this entire routine is called with
312 * CBE_HIGH_PIL, so we needn't worry about preemption.
313 */
314 delta = time - gethrtime();
315
316 /* The unscalehrtime wants unsigned values. */
317 delta = max(delta, 0);
318
319 /* Now we shouldn't be interrupted, we can set the deadline */
320 ticks = (uint64_t)tsc_read() + unscalehrtime(delta);
321 wrmsr(IA32_DEADLINE_TSC_MSR, ticks);
322 }
323
324 /*
325 * This function will reprogram the timer.
326 *
327 * When in oneshot mode the argument is the absolute time in future to
328 * generate the interrupt at.
329 *
330 * When in periodic mode, the argument is the interval at which the
331 * interrupts should be generated. There is no need to support the periodic
332 * mode timer change at this time.
333 */
334 void
apic_timer_reprogram(hrtime_t time)335 apic_timer_reprogram(hrtime_t time)
336 {
337 /*
338 * we should be Called from high PIL context (CBE_HIGH_PIL),
339 * so kpreempt is disabled.
340 */
341 apic_timer.apic_timer_reprogram_ops(time);
342 }
343
344 /*
345 * This function will enable timer interrupts.
346 */
347 void
apic_timer_enable(void)348 apic_timer_enable(void)
349 {
350 /*
351 * we should be Called from high PIL context (CBE_HIGH_PIL),
352 * so kpreempt is disabled.
353 */
354 apic_timer.apic_timer_enable_ops();
355 }
356
357 /*
358 * This function will disable timer interrupts.
359 */
360 void
apic_timer_disable(void)361 apic_timer_disable(void)
362 {
363 /*
364 * we should be Called from high PIL context (CBE_HIGH_PIL),
365 * so kpreempt is disabled.
366 */
367 apic_timer.apic_timer_disable_ops();
368 }
369
370 /*
371 * Set timer far into the future and return timer
372 * current count in nanoseconds.
373 */
374 hrtime_t
apic_timer_stop_count(void)375 apic_timer_stop_count(void)
376 {
377 hrtime_t ns_val;
378 int enable_val, count_val;
379
380 /*
381 * Should be called with interrupts disabled.
382 */
383 ASSERT(!interrupts_enabled());
384
385 enable_val = apic_reg_ops->apic_read(APIC_LOCAL_TIMER);
386 if ((enable_val & AV_MASK) == AV_MASK)
387 return ((hrtime_t)-1); /* timer is disabled */
388
389 count_val = apic_reg_ops->apic_read(APIC_CURR_COUNT);
390 ns_val = APIC_TICKS_TO_NSECS(count_val);
391
392 apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
393
394 return (ns_val);
395 }
396
397 /*
398 * Reprogram timer after Deep C-State.
399 */
400 void
apic_timer_restart(hrtime_t time)401 apic_timer_restart(hrtime_t time)
402 {
403 apic_timer_reprogram(time);
404 }
405