1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2017 by Delphix. All rights reserved. 24 */ 25 /* 26 * Copyright (c) 2010, Intel Corporation. 27 * All rights reserved. 28 */ 29 /* 30 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 31 */ 32 33 #include <sys/time.h> 34 #include <sys/psm.h> 35 #include <sys/psm_common.h> 36 #include <sys/apic.h> 37 #include <sys/pit.h> 38 #include <sys/x86_archext.h> 39 #include <sys/archsystm.h> 40 #include <sys/machsystm.h> 41 #include <sys/cpuvar.h> 42 #include <sys/clock.h> 43 #include <sys/apic_timer.h> 44 45 /* 46 * preferred apic timer mode, allow tuning from the /etc/system file. 47 */ 48 int apic_timer_preferred_mode = APIC_TIMER_MODE_DEADLINE; 49 50 int apic_oneshot = 0; 51 uint_t apic_hertz_count; 52 uint_t apic_nsec_per_intr = 0; 53 uint64_t apic_ticks_per_SFnsecs; /* # of ticks in SF nsecs */ 54 55 static int apic_min_timer_ticks = 1; /* minimum timer tick */ 56 static hrtime_t apic_nsec_max; 57 58 static void periodic_timer_enable(void); 59 static void periodic_timer_disable(void); 60 static void periodic_timer_reprogram(hrtime_t); 61 static void oneshot_timer_enable(void); 62 static void oneshot_timer_disable(void); 63 static void oneshot_timer_reprogram(hrtime_t); 64 static void deadline_timer_enable(void); 65 static void deadline_timer_disable(void); 66 static void deadline_timer_reprogram(hrtime_t); 67 68 extern int apic_clkvect; 69 extern uint32_t apic_divide_reg_init; 70 71 /* 72 * apic timer data structure 73 */ 74 typedef struct apic_timer { 75 int mode; 76 void (*apic_timer_enable_ops)(void); 77 void (*apic_timer_disable_ops)(void); 78 void (*apic_timer_reprogram_ops)(hrtime_t); 79 } apic_timer_t; 80 81 static apic_timer_t apic_timer; 82 83 /* 84 * apic timer initialization 85 * 86 * For the one-shot mode request case, the function returns the 87 * resolution (in nanoseconds) for the hardware timer interrupt. 88 * If one-shot mode capability is not available, the return value 89 * will be 0. 90 */ 91 int 92 apic_timer_init(int hertz) 93 { 94 int ret, timer_mode; 95 static int firsttime = 1; 96 97 if (firsttime) { 98 /* first time calibrate on CPU0 only */ 99 apic_ticks_per_SFnsecs = apic_calibrate(); 100 101 /* the interval timer initial count is 32 bit max */ 102 apic_nsec_max = APIC_TICKS_TO_NSECS(APIC_MAXVAL); 103 firsttime = 0; 104 } 105 106 if (hertz == 0) { 107 /* requested one_shot */ 108 109 /* 110 * return 0 if TSC is not supported. 111 */ 112 if (!tsc_gethrtime_enable) 113 return (0); 114 /* 115 * return 0 if one_shot is not preferred. 116 * here, APIC_TIMER_DEADLINE is also an one_shot mode. 117 */ 118 if ((apic_timer_preferred_mode != APIC_TIMER_MODE_ONESHOT) && 119 (apic_timer_preferred_mode != APIC_TIMER_MODE_DEADLINE)) 120 return (0); 121 122 apic_oneshot = 1; 123 ret = (int)APIC_TICKS_TO_NSECS(1); 124 if ((apic_timer_preferred_mode == APIC_TIMER_MODE_DEADLINE) && 125 cpuid_deadline_tsc_supported()) { 126 timer_mode = APIC_TIMER_MODE_DEADLINE; 127 } else { 128 timer_mode = APIC_TIMER_MODE_ONESHOT; 129 } 130 } else { 131 /* periodic */ 132 apic_nsec_per_intr = NANOSEC / hertz; 133 apic_hertz_count = APIC_NSECS_TO_TICKS(apic_nsec_per_intr); 134 135 /* program the local APIC to interrupt at the given frequency */ 136 apic_reg_ops->apic_write(APIC_INIT_COUNT, apic_hertz_count); 137 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 138 (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC); 139 apic_oneshot = 0; 140 timer_mode = APIC_TIMER_MODE_PERIODIC; 141 ret = NANOSEC / hertz; 142 } 143 144 /* 145 * initialize apic_timer data structure, install the timer ops 146 */ 147 apic_timer.mode = timer_mode; 148 switch (timer_mode) { 149 default: 150 /* FALLTHROUGH */ 151 case APIC_TIMER_MODE_ONESHOT: 152 apic_timer.apic_timer_enable_ops = oneshot_timer_enable; 153 apic_timer.apic_timer_disable_ops = oneshot_timer_disable; 154 apic_timer.apic_timer_reprogram_ops = oneshot_timer_reprogram; 155 break; 156 157 case APIC_TIMER_MODE_PERIODIC: 158 apic_timer.apic_timer_enable_ops = periodic_timer_enable; 159 apic_timer.apic_timer_disable_ops = periodic_timer_disable; 160 apic_timer.apic_timer_reprogram_ops = periodic_timer_reprogram; 161 break; 162 163 case APIC_TIMER_MODE_DEADLINE: 164 apic_timer.apic_timer_enable_ops = deadline_timer_enable; 165 apic_timer.apic_timer_disable_ops = deadline_timer_disable; 166 apic_timer.apic_timer_reprogram_ops = deadline_timer_reprogram; 167 break; 168 } 169 170 return (ret); 171 } 172 173 /* 174 * periodic timer mode ops 175 */ 176 /* periodic timer enable */ 177 static void 178 periodic_timer_enable(void) 179 { 180 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 181 (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC); 182 } 183 184 /* periodic timer disable */ 185 static void 186 periodic_timer_disable(void) 187 { 188 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 189 (apic_clkvect + APIC_BASE_VECT) | AV_MASK); 190 } 191 192 /* periodic timer reprogram */ 193 static void 194 periodic_timer_reprogram(hrtime_t time) 195 { 196 uint_t ticks; 197 /* time is the interval for periodic mode */ 198 ticks = APIC_NSECS_TO_TICKS(time); 199 200 if (ticks < apic_min_timer_ticks) 201 ticks = apic_min_timer_ticks; 202 203 apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks); 204 } 205 206 /* 207 * oneshot timer mode ops 208 */ 209 /* oneshot timer enable */ 210 static void 211 oneshot_timer_enable(void) 212 { 213 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 214 (apic_clkvect + APIC_BASE_VECT)); 215 } 216 217 /* oneshot timer disable */ 218 static void 219 oneshot_timer_disable(void) 220 { 221 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 222 (apic_clkvect + APIC_BASE_VECT) | AV_MASK); 223 } 224 225 /* oneshot timer reprogram */ 226 static void 227 oneshot_timer_reprogram(hrtime_t time) 228 { 229 hrtime_t now; 230 int64_t delta; 231 uint_t ticks; 232 233 now = gethrtime(); 234 delta = time - now; 235 236 if (delta <= 0) { 237 /* 238 * requested to generate an interrupt in the past 239 * generate an interrupt as soon as possible 240 */ 241 ticks = apic_min_timer_ticks; 242 } else if (delta > apic_nsec_max) { 243 /* 244 * requested to generate an interrupt at a time 245 * further than what we are capable of. Set to max 246 * the hardware can handle 247 */ 248 ticks = APIC_MAXVAL; 249 #ifdef DEBUG 250 cmn_err(CE_CONT, "apic_timer_reprogram, request at" 251 " %lld too far in future, current time" 252 " %lld \n", time, now); 253 #endif 254 } else { 255 ticks = APIC_NSECS_TO_TICKS(delta); 256 } 257 258 if (ticks < apic_min_timer_ticks) 259 ticks = apic_min_timer_ticks; 260 261 apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks); 262 } 263 264 /* 265 * deadline timer mode ops 266 */ 267 /* deadline timer enable */ 268 static void 269 deadline_timer_enable(void) 270 { 271 uint64_t ticks; 272 273 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 274 (apic_clkvect + APIC_BASE_VECT) | AV_DEADLINE); 275 /* 276 * Now we have to serialize this per the SDM. That is to 277 * say, the above enabling can race in the pipeline with 278 * changes to the MSR. We need to make sure the above 279 * operation is complete before we proceed to reprogram 280 * the deadline value in reprogram(). The algorithm 281 * recommended by the Intel SDM 3A in 10.5.1.4 is: 282 * 283 * a) write a big value to the deadline register 284 * b) read the register back 285 * c) if it reads zero, go back to a and try again 286 */ 287 288 do { 289 /* write a really big value */ 290 wrmsr(IA32_DEADLINE_TSC_MSR, 1ULL << 63); 291 ticks = rdmsr(IA32_DEADLINE_TSC_MSR); 292 } while (ticks == 0); 293 } 294 295 /* deadline timer disable */ 296 static void 297 deadline_timer_disable(void) 298 { 299 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 300 (apic_clkvect + APIC_BASE_VECT) | AV_MASK); 301 } 302 303 /* deadline timer reprogram */ 304 static void 305 deadline_timer_reprogram(hrtime_t time) 306 { 307 int64_t delta; 308 uint64_t ticks; 309 310 /* 311 * Note that this entire routine is called with 312 * CBE_HIGH_PIL, so we needn't worry about preemption. 313 */ 314 delta = time - gethrtime(); 315 316 /* The unscalehrtime wants unsigned values. */ 317 delta = max(delta, 0); 318 319 /* Now we shouldn't be interrupted, we can set the deadline */ 320 ticks = (uint64_t)tsc_read() + unscalehrtime(delta); 321 wrmsr(IA32_DEADLINE_TSC_MSR, ticks); 322 } 323 324 /* 325 * This function will reprogram the timer. 326 * 327 * When in oneshot mode the argument is the absolute time in future to 328 * generate the interrupt at. 329 * 330 * When in periodic mode, the argument is the interval at which the 331 * interrupts should be generated. There is no need to support the periodic 332 * mode timer change at this time. 333 */ 334 void 335 apic_timer_reprogram(hrtime_t time) 336 { 337 /* 338 * we should be Called from high PIL context (CBE_HIGH_PIL), 339 * so kpreempt is disabled. 340 */ 341 apic_timer.apic_timer_reprogram_ops(time); 342 } 343 344 /* 345 * This function will enable timer interrupts. 346 */ 347 void 348 apic_timer_enable(void) 349 { 350 /* 351 * we should be Called from high PIL context (CBE_HIGH_PIL), 352 * so kpreempt is disabled. 353 */ 354 apic_timer.apic_timer_enable_ops(); 355 } 356 357 /* 358 * This function will disable timer interrupts. 359 */ 360 void 361 apic_timer_disable(void) 362 { 363 /* 364 * we should be Called from high PIL context (CBE_HIGH_PIL), 365 * so kpreempt is disabled. 366 */ 367 apic_timer.apic_timer_disable_ops(); 368 } 369 370 /* 371 * Set timer far into the future and return timer 372 * current count in nanoseconds. 373 */ 374 hrtime_t 375 apic_timer_stop_count(void) 376 { 377 hrtime_t ns_val; 378 int enable_val, count_val; 379 380 /* 381 * Should be called with interrupts disabled. 382 */ 383 ASSERT(!interrupts_enabled()); 384 385 enable_val = apic_reg_ops->apic_read(APIC_LOCAL_TIMER); 386 if ((enable_val & AV_MASK) == AV_MASK) 387 return ((hrtime_t)-1); /* timer is disabled */ 388 389 count_val = apic_reg_ops->apic_read(APIC_CURR_COUNT); 390 ns_val = APIC_TICKS_TO_NSECS(count_val); 391 392 apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL); 393 394 return (ns_val); 395 } 396 397 /* 398 * Reprogram timer after Deep C-State. 399 */ 400 void 401 apic_timer_restart(hrtime_t time) 402 { 403 apic_timer_reprogram(time); 404 } 405