1 /*- 2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 /* 31 * Common routines to manage event timers hardware. 32 */ 33 34 /* XEN has own timer routines now. */ 35 #ifndef XEN 36 37 #include "opt_kdtrace.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/bus.h> 42 #include <sys/lock.h> 43 #include <sys/kdb.h> 44 #include <sys/mutex.h> 45 #include <sys/proc.h> 46 #include <sys/kernel.h> 47 #include <sys/sched.h> 48 #include <sys/smp.h> 49 #include <sys/sysctl.h> 50 #include <sys/timeet.h> 51 52 #include <machine/atomic.h> 53 #include <machine/clock.h> 54 #include <machine/cpu.h> 55 #include <machine/smp.h> 56 57 #ifdef KDTRACE_HOOKS 58 #include <sys/dtrace_bsd.h> 59 cyclic_clock_func_t cyclic_clock_func[MAXCPU]; 60 #endif 61 62 static void cpu_restartclocks(void); 63 static void timercheck(void); 64 inline static int doconfigtimer(int i); 65 static void configtimer(int i); 66 67 static struct eventtimer *timer[2] = { NULL, NULL }; 68 static int timertest = 0; 69 static int timerticks[2] = { 0, 0 }; 70 static int profiling_on = 0; 71 static struct bintime timerperiod[2]; 72 73 static char timername[2][32]; 74 TUNABLE_STR("kern.eventtimer.timer1", timername[0], sizeof(*timername)); 75 TUNABLE_STR("kern.eventtimer.timer2", timername[1], sizeof(*timername)); 76 77 static u_int singlemul = 0; 78 TUNABLE_INT("kern.eventtimer.singlemul", &singlemul); 79 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul, 80 0, "Multiplier, used in single timer mode"); 81 82 typedef u_int tc[2]; 83 static DPCPU_DEFINE(tc, configtimer); 84 85 #define FREQ2BT(freq, bt) \ 86 { \ 87 (bt)->sec = 0; \ 88 (bt)->frac = ((uint64_t)0x8000000000000000 / (freq)) << 1; \ 89 } 90 #define BT2FREQ(bt) \ 91 (((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) / \ 92 ((bt)->frac >> 1)) 93 94 /* Per-CPU timer1 handler. */ 95 static int 96 hardclockhandler(struct trapframe *frame) 97 { 98 99 #ifdef KDTRACE_HOOKS 100 /* 101 * If the DTrace hooks are configured and a callback function 102 * has been registered, then call it to process the high speed 103 * timers. 104 */ 105 int cpu = curcpu; 106 if (cyclic_clock_func[cpu] != NULL) 107 (*cyclic_clock_func[cpu])(frame); 108 #endif 109 110 timer1clock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); 111 return (FILTER_HANDLED); 112 } 113 114 /* Per-CPU timer2 handler. */ 115 static int 116 statclockhandler(struct trapframe *frame) 117 { 118 119 timer2clock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); 120 return (FILTER_HANDLED); 121 } 122 123 /* timer1 broadcast IPI handler. */ 124 int 125 hardclockintr(struct trapframe *frame) 126 { 127 128 if (doconfigtimer(0)) 129 return (FILTER_HANDLED); 130 return (hardclockhandler(frame)); 131 } 132 133 /* timer2 broadcast IPI handler. */ 134 int 135 statclockintr(struct trapframe *frame) 136 { 137 138 if (doconfigtimer(1)) 139 return (FILTER_HANDLED); 140 return (statclockhandler(frame)); 141 } 142 143 /* timer1 callback. */ 144 static void 145 timer1cb(struct eventtimer *et, void *arg) 146 { 147 148 #ifdef SMP 149 /* Broadcast interrupt to other CPUs for non-per-CPU timers */ 150 if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0) 151 ipi_all_but_self(IPI_HARDCLOCK); 152 #endif 153 if (timertest) { 154 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) { 155 timerticks[0]++; 156 if (timerticks[0] >= timer1hz) { 157 ET_LOCK(); 158 timercheck(); 159 ET_UNLOCK(); 160 } 161 } 162 } 163 hardclockhandler(curthread->td_intr_frame); 164 } 165 166 /* timer2 callback. */ 167 static void 168 timer2cb(struct eventtimer *et, void *arg) 169 { 170 171 #ifdef SMP 172 /* Broadcast interrupt to other CPUs for non-per-CPU timers */ 173 if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0) 174 ipi_all_but_self(IPI_STATCLOCK); 175 #endif 176 if (timertest) { 177 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) { 178 timerticks[1]++; 179 if (timerticks[1] >= timer2hz * 2) { 180 ET_LOCK(); 181 timercheck(); 182 ET_UNLOCK(); 183 } 184 } 185 } 186 statclockhandler(curthread->td_intr_frame); 187 } 188 189 /* 190 * Check that both timers are running with at least 1/4 of configured rate. 191 * If not - replace the broken one. 192 */ 193 static void 194 timercheck(void) 195 { 196 197 if (!timertest) 198 return; 199 timertest = 0; 200 if (timerticks[0] * 4 < timer1hz) { 201 printf("Event timer \"%s\" is dead.\n", timer[0]->et_name); 202 timer1hz = 0; 203 configtimer(0); 204 et_ban(timer[0]); 205 et_free(timer[0]); 206 timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 207 if (timer[0] == NULL) { 208 timer2hz = 0; 209 configtimer(1); 210 et_free(timer[1]); 211 timer[1] = NULL; 212 timer[0] = timer[1]; 213 } 214 et_init(timer[0], timer1cb, NULL, NULL); 215 cpu_restartclocks(); 216 return; 217 } 218 if (timerticks[1] * 4 < timer2hz) { 219 printf("Event timer \"%s\" is dead.\n", timer[1]->et_name); 220 timer2hz = 0; 221 configtimer(1); 222 et_ban(timer[1]); 223 et_free(timer[1]); 224 timer[1] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 225 if (timer[1] != NULL) 226 et_init(timer[1], timer2cb, NULL, NULL); 227 cpu_restartclocks(); 228 return; 229 } 230 } 231 232 /* 233 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler. 234 */ 235 inline static int 236 doconfigtimer(int i) 237 { 238 tc *conf; 239 240 conf = DPCPU_PTR(configtimer); 241 if (atomic_load_acq_int(*conf + i)) { 242 if (i == 0 ? timer1hz : timer2hz) 243 et_start(timer[i], NULL, &timerperiod[i]); 244 else 245 et_stop(timer[i]); 246 atomic_store_rel_int(*conf + i, 0); 247 return (1); 248 } 249 return (0); 250 } 251 252 /* 253 * Reconfigure specified timer. 254 * For per-CPU timers use IPI to make other CPUs to reconfigure. 255 */ 256 static void 257 configtimer(int i) 258 { 259 #ifdef SMP 260 tc *conf; 261 int cpu; 262 263 critical_enter(); 264 #endif 265 /* Start/stop global timer or per-CPU timer of this CPU. */ 266 if (i == 0 ? timer1hz : timer2hz) 267 et_start(timer[i], NULL, &timerperiod[i]); 268 else 269 et_stop(timer[i]); 270 #ifdef SMP 271 if ((timer[i]->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) { 272 critical_exit(); 273 return; 274 } 275 /* Set reconfigure flags for other CPUs. */ 276 CPU_FOREACH(cpu) { 277 conf = DPCPU_ID_PTR(cpu, configtimer); 278 atomic_store_rel_int(*conf + i, (cpu == curcpu) ? 0 : 1); 279 } 280 /* Send reconfigure IPI. */ 281 ipi_all_but_self(i == 0 ? IPI_HARDCLOCK : IPI_STATCLOCK); 282 /* Wait for reconfiguration completed. */ 283 restart: 284 cpu_spinwait(); 285 CPU_FOREACH(cpu) { 286 if (cpu == curcpu) 287 continue; 288 conf = DPCPU_ID_PTR(cpu, configtimer); 289 if (atomic_load_acq_int(*conf + i)) 290 goto restart; 291 } 292 critical_exit(); 293 #endif 294 } 295 296 static int 297 round_freq(struct eventtimer *et, int freq) 298 { 299 uint64_t div; 300 301 if (et->et_frequency != 0) { 302 div = lmax((et->et_frequency + freq / 2) / freq, 1); 303 if (et->et_flags & ET_FLAGS_POW2DIV) 304 div = 1 << (flsl(div + div / 2) - 1); 305 freq = (et->et_frequency + div / 2) / div; 306 } 307 if (et->et_min_period.sec > 0) 308 freq = 0; 309 else if (et->et_min_period.frac != 0) 310 freq = min(freq, BT2FREQ(&et->et_min_period)); 311 if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0) 312 freq = max(freq, BT2FREQ(&et->et_max_period)); 313 return (freq); 314 } 315 316 /* 317 * Configure and start event timers. 318 */ 319 void 320 cpu_initclocks_bsp(void) 321 { 322 int base, div; 323 324 timer[0] = et_find(timername[0], ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 325 if (timer[0] == NULL) 326 timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 327 if (timer[0] == NULL) 328 panic("No usable event timer found!"); 329 et_init(timer[0], timer1cb, NULL, NULL); 330 timer[1] = et_find(timername[1][0] ? timername[1] : NULL, 331 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 332 if (timer[1]) 333 et_init(timer[1], timer2cb, NULL, NULL); 334 /* 335 * We honor the requested 'hz' value. 336 * We want to run stathz in the neighborhood of 128hz. 337 * We would like profhz to run as often as possible. 338 */ 339 if (singlemul == 0) { 340 if (hz >= 1500 || (hz % 128) == 0) 341 singlemul = 1; 342 else if (hz >= 750) 343 singlemul = 2; 344 else 345 singlemul = 4; 346 } 347 if (timer[1] == NULL) { 348 base = round_freq(timer[0], hz * singlemul); 349 singlemul = max((base + hz / 2) / hz, 1); 350 hz = (base + singlemul / 2) / singlemul; 351 if (base <= 128) 352 stathz = base; 353 else { 354 div = base / 128; 355 if (div >= singlemul && (div % singlemul) == 0) 356 div++; 357 stathz = base / div; 358 } 359 profhz = stathz; 360 while ((profhz + stathz) <= 128 * 64) 361 profhz += stathz; 362 profhz = round_freq(timer[0], profhz); 363 } else { 364 hz = round_freq(timer[0], hz); 365 stathz = round_freq(timer[1], 127); 366 profhz = round_freq(timer[1], stathz * 64); 367 } 368 tick = 1000000 / hz; 369 ET_LOCK(); 370 cpu_restartclocks(); 371 ET_UNLOCK(); 372 } 373 374 /* Start per-CPU event timers on APs. */ 375 void 376 cpu_initclocks_ap(void) 377 { 378 379 ET_LOCK(); 380 if (timer[0]->et_flags & ET_FLAGS_PERCPU) 381 et_start(timer[0], NULL, &timerperiod[0]); 382 if (timer[1] && timer[1]->et_flags & ET_FLAGS_PERCPU) 383 et_start(timer[1], NULL, &timerperiod[1]); 384 ET_UNLOCK(); 385 } 386 387 /* Reconfigure and restart event timers after configuration changes. */ 388 static void 389 cpu_restartclocks(void) 390 { 391 392 /* Stop all event timers. */ 393 timertest = 0; 394 if (timer1hz) { 395 timer1hz = 0; 396 configtimer(0); 397 } 398 if (timer[1] && timer2hz) { 399 timer2hz = 0; 400 configtimer(1); 401 } 402 /* Calculate new event timers parameters. */ 403 if (timer[1] == NULL) { 404 timer1hz = hz * singlemul; 405 while (timer1hz < (profiling_on ? profhz : stathz)) 406 timer1hz += hz; 407 timer2hz = 0; 408 } else { 409 timer1hz = hz; 410 timer2hz = profiling_on ? profhz : stathz; 411 timer2hz = round_freq(timer[1], timer2hz); 412 } 413 timer1hz = round_freq(timer[0], timer1hz); 414 printf("Starting kernel event timers: %s @ %dHz, %s @ %dHz\n", 415 timer[0]->et_name, timer1hz, 416 timer[1] ? timer[1]->et_name : "NONE", timer2hz); 417 /* Restart event timers. */ 418 FREQ2BT(timer1hz, &timerperiod[0]); 419 configtimer(0); 420 if (timer[1]) { 421 timerticks[0] = 0; 422 timerticks[1] = 0; 423 FREQ2BT(timer2hz, &timerperiod[1]); 424 configtimer(1); 425 timertest = 1; 426 } 427 } 428 429 /* Switch to profiling clock rates. */ 430 void 431 cpu_startprofclock(void) 432 { 433 434 ET_LOCK(); 435 profiling_on = 1; 436 cpu_restartclocks(); 437 ET_UNLOCK(); 438 } 439 440 /* Switch to regular clock rates. */ 441 void 442 cpu_stopprofclock(void) 443 { 444 445 ET_LOCK(); 446 profiling_on = 0; 447 cpu_restartclocks(); 448 ET_UNLOCK(); 449 } 450 451 /* Report or change the active event timers hardware. */ 452 static int 453 sysctl_kern_eventtimer_timer1(SYSCTL_HANDLER_ARGS) 454 { 455 char buf[32]; 456 struct eventtimer *et; 457 int error; 458 459 ET_LOCK(); 460 et = timer[0]; 461 snprintf(buf, sizeof(buf), "%s", et->et_name); 462 ET_UNLOCK(); 463 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 464 ET_LOCK(); 465 et = timer[0]; 466 if (error != 0 || req->newptr == NULL || 467 strcmp(buf, et->et_name) == 0) { 468 ET_UNLOCK(); 469 return (error); 470 } 471 et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 472 if (et == NULL) { 473 ET_UNLOCK(); 474 return (ENOENT); 475 } 476 timer1hz = 0; 477 configtimer(0); 478 et_free(timer[0]); 479 timer[0] = et; 480 et_init(timer[0], timer1cb, NULL, NULL); 481 cpu_restartclocks(); 482 ET_UNLOCK(); 483 return (error); 484 } 485 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer1, 486 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 487 0, 0, sysctl_kern_eventtimer_timer1, "A", "Primary event timer"); 488 489 static int 490 sysctl_kern_eventtimer_timer2(SYSCTL_HANDLER_ARGS) 491 { 492 char buf[32]; 493 struct eventtimer *et; 494 int error; 495 496 ET_LOCK(); 497 et = timer[1]; 498 if (et == NULL) 499 snprintf(buf, sizeof(buf), "NONE"); 500 else 501 snprintf(buf, sizeof(buf), "%s", et->et_name); 502 ET_UNLOCK(); 503 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 504 ET_LOCK(); 505 et = timer[1]; 506 if (error != 0 || req->newptr == NULL || 507 strcmp(buf, et ? et->et_name : "NONE") == 0) { 508 ET_UNLOCK(); 509 return (error); 510 } 511 et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 512 if (et == NULL && strcasecmp(buf, "NONE") != 0) { 513 ET_UNLOCK(); 514 return (ENOENT); 515 } 516 if (timer[1] != NULL) { 517 timer2hz = 0; 518 configtimer(1); 519 et_free(timer[1]); 520 } 521 timer[1] = et; 522 if (timer[1] != NULL) 523 et_init(timer[1], timer2cb, NULL, NULL); 524 cpu_restartclocks(); 525 ET_UNLOCK(); 526 return (error); 527 } 528 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer2, 529 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 530 0, 0, sysctl_kern_eventtimer_timer2, "A", "Secondary event timer"); 531 532 #endif 533 534