1 /*- 2 * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 /* 31 * Common routines to manage event timers hardware. 32 */ 33 34 #include "opt_device_polling.h" 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/bus.h> 39 #include <sys/limits.h> 40 #include <sys/lock.h> 41 #include <sys/kdb.h> 42 #include <sys/ktr.h> 43 #include <sys/mutex.h> 44 #include <sys/proc.h> 45 #include <sys/kernel.h> 46 #include <sys/sched.h> 47 #include <sys/smp.h> 48 #include <sys/sysctl.h> 49 #include <sys/timeet.h> 50 #include <sys/timetc.h> 51 52 #include <machine/atomic.h> 53 #include <machine/clock.h> 54 #include <machine/cpu.h> 55 #include <machine/smp.h> 56 57 int cpu_deepest_sleep = 0; /* Deepest Cx state available. */ 58 int cpu_disable_c2_sleep = 0; /* Timer dies in C2. */ 59 int cpu_disable_c3_sleep = 0; /* Timer dies in C3. */ 60 61 static void setuptimer(void); 62 static void loadtimer(sbintime_t now, int first); 63 static int doconfigtimer(void); 64 static void configtimer(int start); 65 static int round_freq(struct eventtimer *et, int freq); 66 67 static sbintime_t getnextcpuevent(int idle); 68 static sbintime_t getnextevent(void); 69 static int handleevents(sbintime_t now, int fake); 70 71 static struct mtx et_hw_mtx; 72 73 #define ET_HW_LOCK(state) \ 74 { \ 75 if (timer->et_flags & ET_FLAGS_PERCPU) \ 76 mtx_lock_spin(&(state)->et_hw_mtx); \ 77 else \ 78 mtx_lock_spin(&et_hw_mtx); \ 79 } 80 81 #define ET_HW_UNLOCK(state) \ 82 { \ 83 if (timer->et_flags & ET_FLAGS_PERCPU) \ 84 mtx_unlock_spin(&(state)->et_hw_mtx); \ 85 else \ 86 mtx_unlock_spin(&et_hw_mtx); \ 87 } 88 89 static struct eventtimer *timer = NULL; 90 static sbintime_t timerperiod; /* Timer period for periodic mode. */ 91 static sbintime_t statperiod; /* statclock() events period. */ 92 static sbintime_t profperiod; /* profclock() events period. */ 93 static sbintime_t nexttick; /* Next global timer tick time. */ 94 static u_int busy = 1; /* Reconfiguration is in progress. */ 95 static int profiling; /* Profiling events enabled. */ 96 97 static char timername[32]; /* Wanted timer. */ 98 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername)); 99 100 static int singlemul; /* Multiplier for periodic mode. */ 101 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RWTUN, &singlemul, 102 0, "Multiplier for periodic mode"); 103 104 static u_int idletick; /* Run periodic events when idle. */ 105 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RWTUN, &idletick, 106 0, "Run periodic events when idle"); 107 108 static int periodic; /* Periodic or one-shot mode. */ 109 static int want_periodic; /* What mode to prefer. */ 110 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic); 111 112 struct pcpu_state { 113 struct mtx et_hw_mtx; /* Per-CPU timer mutex. */ 114 u_int action; /* Reconfiguration requests. */ 115 u_int handle; /* Immediate handle resuests. */ 116 sbintime_t now; /* Last tick time. */ 117 sbintime_t nextevent; /* Next scheduled event on this CPU. */ 118 sbintime_t nexttick; /* Next timer tick time. */ 119 sbintime_t nexthard; /* Next hardclock() event. */ 120 sbintime_t nextstat; /* Next statclock() event. */ 121 sbintime_t nextprof; /* Next profclock() event. */ 122 sbintime_t nextcall; /* Next callout event. */ 123 sbintime_t nextcallopt; /* Next optional callout event. */ 124 int ipi; /* This CPU needs IPI. */ 125 int idle; /* This CPU is in idle mode. */ 126 }; 127 128 static DPCPU_DEFINE(struct pcpu_state, timerstate); 129 DPCPU_DEFINE(sbintime_t, hardclocktime); 130 131 /* 132 * Timer broadcast IPI handler. 133 */ 134 int 135 hardclockintr(void) 136 { 137 sbintime_t now; 138 struct pcpu_state *state; 139 int done; 140 141 if (doconfigtimer() || busy) 142 return (FILTER_HANDLED); 143 state = DPCPU_PTR(timerstate); 144 now = state->now; 145 CTR3(KTR_SPARE2, "ipi at %d: now %d.%08x", 146 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 147 done = handleevents(now, 0); 148 return (done ? FILTER_HANDLED : FILTER_STRAY); 149 } 150 151 /* 152 * Handle all events for specified time on this CPU 153 */ 154 static int 155 handleevents(sbintime_t now, int fake) 156 { 157 sbintime_t t, *hct; 158 struct trapframe *frame; 159 struct pcpu_state *state; 160 int usermode; 161 int done, runs; 162 163 CTR3(KTR_SPARE2, "handle at %d: now %d.%08x", 164 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 165 done = 0; 166 if (fake) { 167 frame = NULL; 168 usermode = 0; 169 } else { 170 frame = curthread->td_intr_frame; 171 usermode = TRAPF_USERMODE(frame); 172 } 173 174 state = DPCPU_PTR(timerstate); 175 176 runs = 0; 177 while (now >= state->nexthard) { 178 state->nexthard += tick_sbt; 179 runs++; 180 } 181 if (runs) { 182 hct = DPCPU_PTR(hardclocktime); 183 *hct = state->nexthard - tick_sbt; 184 if (fake < 2) { 185 hardclock_cnt(runs, usermode); 186 done = 1; 187 } 188 } 189 runs = 0; 190 while (now >= state->nextstat) { 191 state->nextstat += statperiod; 192 runs++; 193 } 194 if (runs && fake < 2) { 195 statclock_cnt(runs, usermode); 196 done = 1; 197 } 198 if (profiling) { 199 runs = 0; 200 while (now >= state->nextprof) { 201 state->nextprof += profperiod; 202 runs++; 203 } 204 if (runs && !fake) { 205 profclock_cnt(runs, usermode, TRAPF_PC(frame)); 206 done = 1; 207 } 208 } else 209 state->nextprof = state->nextstat; 210 if (now >= state->nextcallopt) { 211 state->nextcall = state->nextcallopt = SBT_MAX; 212 callout_process(now); 213 } 214 215 t = getnextcpuevent(0); 216 ET_HW_LOCK(state); 217 if (!busy) { 218 state->idle = 0; 219 state->nextevent = t; 220 loadtimer(now, (fake == 2) && 221 (timer->et_flags & ET_FLAGS_PERCPU)); 222 } 223 ET_HW_UNLOCK(state); 224 return (done); 225 } 226 227 /* 228 * Schedule binuptime of the next event on current CPU. 229 */ 230 static sbintime_t 231 getnextcpuevent(int idle) 232 { 233 sbintime_t event; 234 struct pcpu_state *state; 235 u_int hardfreq; 236 237 state = DPCPU_PTR(timerstate); 238 /* Handle hardclock() events, skipping some if CPU is idle. */ 239 event = state->nexthard; 240 if (idle) { 241 hardfreq = (u_int)hz / 2; 242 if (tc_min_ticktock_freq > 2 243 #ifdef SMP 244 && curcpu == CPU_FIRST() 245 #endif 246 ) 247 hardfreq = hz / tc_min_ticktock_freq; 248 if (hardfreq > 1) 249 event += tick_sbt * (hardfreq - 1); 250 } 251 /* Handle callout events. */ 252 if (event > state->nextcall) 253 event = state->nextcall; 254 if (!idle) { /* If CPU is active - handle other types of events. */ 255 if (event > state->nextstat) 256 event = state->nextstat; 257 if (profiling && event > state->nextprof) 258 event = state->nextprof; 259 } 260 return (event); 261 } 262 263 /* 264 * Schedule binuptime of the next event on all CPUs. 265 */ 266 static sbintime_t 267 getnextevent(void) 268 { 269 struct pcpu_state *state; 270 sbintime_t event; 271 #ifdef SMP 272 int cpu; 273 #endif 274 int c; 275 276 state = DPCPU_PTR(timerstate); 277 event = state->nextevent; 278 c = -1; 279 #ifdef SMP 280 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) { 281 CPU_FOREACH(cpu) { 282 state = DPCPU_ID_PTR(cpu, timerstate); 283 if (event > state->nextevent) { 284 event = state->nextevent; 285 c = cpu; 286 } 287 } 288 } 289 #endif 290 CTR4(KTR_SPARE2, "next at %d: next %d.%08x by %d", 291 curcpu, (int)(event >> 32), (u_int)(event & 0xffffffff), c); 292 return (event); 293 } 294 295 /* Hardware timer callback function. */ 296 static void 297 timercb(struct eventtimer *et, void *arg) 298 { 299 sbintime_t now; 300 sbintime_t *next; 301 struct pcpu_state *state; 302 #ifdef SMP 303 int cpu, bcast; 304 #endif 305 306 /* Do not touch anything if somebody reconfiguring timers. */ 307 if (busy) 308 return; 309 /* Update present and next tick times. */ 310 state = DPCPU_PTR(timerstate); 311 if (et->et_flags & ET_FLAGS_PERCPU) { 312 next = &state->nexttick; 313 } else 314 next = &nexttick; 315 now = sbinuptime(); 316 if (periodic) 317 *next = now + timerperiod; 318 else 319 *next = -1; /* Next tick is not scheduled yet. */ 320 state->now = now; 321 CTR3(KTR_SPARE2, "intr at %d: now %d.%08x", 322 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 323 324 #ifdef SMP 325 #ifdef EARLY_AP_STARTUP 326 MPASS(mp_ncpus == 1 || smp_started); 327 #endif 328 /* Prepare broadcasting to other CPUs for non-per-CPU timers. */ 329 bcast = 0; 330 #ifdef EARLY_AP_STARTUP 331 if ((et->et_flags & ET_FLAGS_PERCPU) == 0) { 332 #else 333 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) { 334 #endif 335 CPU_FOREACH(cpu) { 336 state = DPCPU_ID_PTR(cpu, timerstate); 337 ET_HW_LOCK(state); 338 state->now = now; 339 if (now >= state->nextevent) { 340 state->nextevent += SBT_1S; 341 if (curcpu != cpu) { 342 state->ipi = 1; 343 bcast = 1; 344 } 345 } 346 ET_HW_UNLOCK(state); 347 } 348 } 349 #endif 350 351 /* Handle events for this time on this CPU. */ 352 handleevents(now, 0); 353 354 #ifdef SMP 355 /* Broadcast interrupt to other CPUs for non-per-CPU timers. */ 356 if (bcast) { 357 CPU_FOREACH(cpu) { 358 if (curcpu == cpu) 359 continue; 360 state = DPCPU_ID_PTR(cpu, timerstate); 361 if (state->ipi) { 362 state->ipi = 0; 363 ipi_cpu(cpu, IPI_HARDCLOCK); 364 } 365 } 366 } 367 #endif 368 } 369 370 /* 371 * Load new value into hardware timer. 372 */ 373 static void 374 loadtimer(sbintime_t now, int start) 375 { 376 struct pcpu_state *state; 377 sbintime_t new; 378 sbintime_t *next; 379 uint64_t tmp; 380 int eq; 381 382 if (timer->et_flags & ET_FLAGS_PERCPU) { 383 state = DPCPU_PTR(timerstate); 384 next = &state->nexttick; 385 } else 386 next = &nexttick; 387 if (periodic) { 388 if (start) { 389 /* 390 * Try to start all periodic timers aligned 391 * to period to make events synchronous. 392 */ 393 tmp = now % timerperiod; 394 new = timerperiod - tmp; 395 if (new < tmp) /* Left less then passed. */ 396 new += timerperiod; 397 CTR5(KTR_SPARE2, "load p at %d: now %d.%08x first in %d.%08x", 398 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff), 399 (int)(new >> 32), (u_int)(new & 0xffffffff)); 400 *next = new + now; 401 et_start(timer, new, timerperiod); 402 } 403 } else { 404 new = getnextevent(); 405 eq = (new == *next); 406 CTR4(KTR_SPARE2, "load at %d: next %d.%08x eq %d", 407 curcpu, (int)(new >> 32), (u_int)(new & 0xffffffff), eq); 408 if (!eq) { 409 *next = new; 410 et_start(timer, new - now, 0); 411 } 412 } 413 } 414 415 /* 416 * Prepare event timer parameters after configuration changes. 417 */ 418 static void 419 setuptimer(void) 420 { 421 int freq; 422 423 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0) 424 periodic = 0; 425 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0) 426 periodic = 1; 427 singlemul = MIN(MAX(singlemul, 1), 20); 428 freq = hz * singlemul; 429 while (freq < (profiling ? profhz : stathz)) 430 freq += hz; 431 freq = round_freq(timer, freq); 432 timerperiod = SBT_1S / freq; 433 } 434 435 /* 436 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler. 437 */ 438 static int 439 doconfigtimer(void) 440 { 441 sbintime_t now; 442 struct pcpu_state *state; 443 444 state = DPCPU_PTR(timerstate); 445 switch (atomic_load_acq_int(&state->action)) { 446 case 1: 447 now = sbinuptime(); 448 ET_HW_LOCK(state); 449 loadtimer(now, 1); 450 ET_HW_UNLOCK(state); 451 state->handle = 0; 452 atomic_store_rel_int(&state->action, 0); 453 return (1); 454 case 2: 455 ET_HW_LOCK(state); 456 et_stop(timer); 457 ET_HW_UNLOCK(state); 458 state->handle = 0; 459 atomic_store_rel_int(&state->action, 0); 460 return (1); 461 } 462 if (atomic_readandclear_int(&state->handle) && !busy) { 463 now = sbinuptime(); 464 handleevents(now, 0); 465 return (1); 466 } 467 return (0); 468 } 469 470 /* 471 * Reconfigure specified timer. 472 * For per-CPU timers use IPI to make other CPUs to reconfigure. 473 */ 474 static void 475 configtimer(int start) 476 { 477 sbintime_t now, next; 478 struct pcpu_state *state; 479 int cpu; 480 481 if (start) { 482 setuptimer(); 483 now = sbinuptime(); 484 } else 485 now = 0; 486 critical_enter(); 487 ET_HW_LOCK(DPCPU_PTR(timerstate)); 488 if (start) { 489 /* Initialize time machine parameters. */ 490 next = now + timerperiod; 491 if (periodic) 492 nexttick = next; 493 else 494 nexttick = -1; 495 #ifdef EARLY_AP_STARTUP 496 MPASS(mp_ncpus == 1 || smp_started); 497 #endif 498 CPU_FOREACH(cpu) { 499 state = DPCPU_ID_PTR(cpu, timerstate); 500 state->now = now; 501 #ifndef EARLY_AP_STARTUP 502 if (!smp_started && cpu != CPU_FIRST()) 503 state->nextevent = SBT_MAX; 504 else 505 #endif 506 state->nextevent = next; 507 if (periodic) 508 state->nexttick = next; 509 else 510 state->nexttick = -1; 511 state->nexthard = next; 512 state->nextstat = next; 513 state->nextprof = next; 514 state->nextcall = next; 515 state->nextcallopt = next; 516 hardclock_sync(cpu); 517 } 518 busy = 0; 519 /* Start global timer or per-CPU timer of this CPU. */ 520 loadtimer(now, 1); 521 } else { 522 busy = 1; 523 /* Stop global timer or per-CPU timer of this CPU. */ 524 et_stop(timer); 525 } 526 ET_HW_UNLOCK(DPCPU_PTR(timerstate)); 527 #ifdef SMP 528 #ifdef EARLY_AP_STARTUP 529 /* If timer is global we are done. */ 530 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) { 531 #else 532 /* If timer is global or there is no other CPUs yet - we are done. */ 533 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) { 534 #endif 535 critical_exit(); 536 return; 537 } 538 /* Set reconfigure flags for other CPUs. */ 539 CPU_FOREACH(cpu) { 540 state = DPCPU_ID_PTR(cpu, timerstate); 541 atomic_store_rel_int(&state->action, 542 (cpu == curcpu) ? 0 : ( start ? 1 : 2)); 543 } 544 /* Broadcast reconfigure IPI. */ 545 ipi_all_but_self(IPI_HARDCLOCK); 546 /* Wait for reconfiguration completed. */ 547 restart: 548 cpu_spinwait(); 549 CPU_FOREACH(cpu) { 550 if (cpu == curcpu) 551 continue; 552 state = DPCPU_ID_PTR(cpu, timerstate); 553 if (atomic_load_acq_int(&state->action)) 554 goto restart; 555 } 556 #endif 557 critical_exit(); 558 } 559 560 /* 561 * Calculate nearest frequency supported by hardware timer. 562 */ 563 static int 564 round_freq(struct eventtimer *et, int freq) 565 { 566 uint64_t div; 567 568 if (et->et_frequency != 0) { 569 div = lmax((et->et_frequency + freq / 2) / freq, 1); 570 if (et->et_flags & ET_FLAGS_POW2DIV) 571 div = 1 << (flsl(div + div / 2) - 1); 572 freq = (et->et_frequency + div / 2) / div; 573 } 574 if (et->et_min_period > SBT_1S) 575 panic("Event timer \"%s\" doesn't support sub-second periods!", 576 et->et_name); 577 else if (et->et_min_period != 0) 578 freq = min(freq, SBT2FREQ(et->et_min_period)); 579 if (et->et_max_period < SBT_1S && et->et_max_period != 0) 580 freq = max(freq, SBT2FREQ(et->et_max_period)); 581 return (freq); 582 } 583 584 /* 585 * Configure and start event timers (BSP part). 586 */ 587 void 588 cpu_initclocks_bsp(void) 589 { 590 struct pcpu_state *state; 591 int base, div, cpu; 592 593 mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN); 594 CPU_FOREACH(cpu) { 595 state = DPCPU_ID_PTR(cpu, timerstate); 596 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN); 597 state->nextcall = SBT_MAX; 598 state->nextcallopt = SBT_MAX; 599 } 600 periodic = want_periodic; 601 /* Grab requested timer or the best of present. */ 602 if (timername[0]) 603 timer = et_find(timername, 0, 0); 604 if (timer == NULL && periodic) { 605 timer = et_find(NULL, 606 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 607 } 608 if (timer == NULL) { 609 timer = et_find(NULL, 610 ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT); 611 } 612 if (timer == NULL && !periodic) { 613 timer = et_find(NULL, 614 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 615 } 616 if (timer == NULL) 617 panic("No usable event timer found!"); 618 et_init(timer, timercb, NULL, NULL); 619 620 /* Adapt to timer capabilities. */ 621 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0) 622 periodic = 0; 623 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0) 624 periodic = 1; 625 if (timer->et_flags & ET_FLAGS_C3STOP) 626 cpu_disable_c3_sleep++; 627 628 /* 629 * We honor the requested 'hz' value. 630 * We want to run stathz in the neighborhood of 128hz. 631 * We would like profhz to run as often as possible. 632 */ 633 if (singlemul <= 0 || singlemul > 20) { 634 if (hz >= 1500 || (hz % 128) == 0) 635 singlemul = 1; 636 else if (hz >= 750) 637 singlemul = 2; 638 else 639 singlemul = 4; 640 } 641 if (periodic) { 642 base = round_freq(timer, hz * singlemul); 643 singlemul = max((base + hz / 2) / hz, 1); 644 hz = (base + singlemul / 2) / singlemul; 645 if (base <= 128) 646 stathz = base; 647 else { 648 div = base / 128; 649 if (div >= singlemul && (div % singlemul) == 0) 650 div++; 651 stathz = base / div; 652 } 653 profhz = stathz; 654 while ((profhz + stathz) <= 128 * 64) 655 profhz += stathz; 656 profhz = round_freq(timer, profhz); 657 } else { 658 hz = round_freq(timer, hz); 659 stathz = round_freq(timer, 127); 660 profhz = round_freq(timer, stathz * 64); 661 } 662 tick = 1000000 / hz; 663 tick_sbt = SBT_1S / hz; 664 tick_bt = sbttobt(tick_sbt); 665 statperiod = SBT_1S / stathz; 666 profperiod = SBT_1S / profhz; 667 ET_LOCK(); 668 configtimer(1); 669 ET_UNLOCK(); 670 } 671 672 /* 673 * Start per-CPU event timers on APs. 674 */ 675 void 676 cpu_initclocks_ap(void) 677 { 678 sbintime_t now; 679 struct pcpu_state *state; 680 struct thread *td; 681 682 state = DPCPU_PTR(timerstate); 683 now = sbinuptime(); 684 ET_HW_LOCK(state); 685 state->now = now; 686 hardclock_sync(curcpu); 687 spinlock_enter(); 688 ET_HW_UNLOCK(state); 689 td = curthread; 690 td->td_intr_nesting_level++; 691 handleevents(state->now, 2); 692 td->td_intr_nesting_level--; 693 spinlock_exit(); 694 } 695 696 /* 697 * Switch to profiling clock rates. 698 */ 699 void 700 cpu_startprofclock(void) 701 { 702 703 ET_LOCK(); 704 if (profiling == 0) { 705 if (periodic) { 706 configtimer(0); 707 profiling = 1; 708 configtimer(1); 709 } else 710 profiling = 1; 711 } else 712 profiling++; 713 ET_UNLOCK(); 714 } 715 716 /* 717 * Switch to regular clock rates. 718 */ 719 void 720 cpu_stopprofclock(void) 721 { 722 723 ET_LOCK(); 724 if (profiling == 1) { 725 if (periodic) { 726 configtimer(0); 727 profiling = 0; 728 configtimer(1); 729 } else 730 profiling = 0; 731 } else 732 profiling--; 733 ET_UNLOCK(); 734 } 735 736 /* 737 * Switch to idle mode (all ticks handled). 738 */ 739 sbintime_t 740 cpu_idleclock(void) 741 { 742 sbintime_t now, t; 743 struct pcpu_state *state; 744 745 if (idletick || busy || 746 (periodic && (timer->et_flags & ET_FLAGS_PERCPU)) 747 #ifdef DEVICE_POLLING 748 || curcpu == CPU_FIRST() 749 #endif 750 ) 751 return (-1); 752 state = DPCPU_PTR(timerstate); 753 if (periodic) 754 now = state->now; 755 else 756 now = sbinuptime(); 757 CTR3(KTR_SPARE2, "idle at %d: now %d.%08x", 758 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 759 t = getnextcpuevent(1); 760 ET_HW_LOCK(state); 761 state->idle = 1; 762 state->nextevent = t; 763 if (!periodic) 764 loadtimer(now, 0); 765 ET_HW_UNLOCK(state); 766 return (MAX(t - now, 0)); 767 } 768 769 /* 770 * Switch to active mode (skip empty ticks). 771 */ 772 void 773 cpu_activeclock(void) 774 { 775 sbintime_t now; 776 struct pcpu_state *state; 777 struct thread *td; 778 779 state = DPCPU_PTR(timerstate); 780 if (state->idle == 0 || busy) 781 return; 782 if (periodic) 783 now = state->now; 784 else 785 now = sbinuptime(); 786 CTR3(KTR_SPARE2, "active at %d: now %d.%08x", 787 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 788 spinlock_enter(); 789 td = curthread; 790 td->td_intr_nesting_level++; 791 handleevents(now, 1); 792 td->td_intr_nesting_level--; 793 spinlock_exit(); 794 } 795 796 /* 797 * Change the frequency of the given timer. This changes et->et_frequency and 798 * if et is the active timer it reconfigures the timer on all CPUs. This is 799 * intended to be a private interface for the use of et_change_frequency() only. 800 */ 801 void 802 cpu_et_frequency(struct eventtimer *et, uint64_t newfreq) 803 { 804 805 ET_LOCK(); 806 if (et == timer) { 807 configtimer(0); 808 et->et_frequency = newfreq; 809 configtimer(1); 810 } else 811 et->et_frequency = newfreq; 812 ET_UNLOCK(); 813 } 814 815 void 816 cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt) 817 { 818 struct pcpu_state *state; 819 820 /* Do not touch anything if somebody reconfiguring timers. */ 821 if (busy) 822 return; 823 CTR6(KTR_SPARE2, "new co at %d: on %d at %d.%08x - %d.%08x", 824 curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff), 825 (int)(bt >> 32), (u_int)(bt & 0xffffffff)); 826 state = DPCPU_ID_PTR(cpu, timerstate); 827 ET_HW_LOCK(state); 828 829 /* 830 * If there is callout time already set earlier -- do nothing. 831 * This check may appear redundant because we check already in 832 * callout_process() but this double check guarantees we're safe 833 * with respect to race conditions between interrupts execution 834 * and scheduling. 835 */ 836 state->nextcallopt = bt_opt; 837 if (bt >= state->nextcall) 838 goto done; 839 state->nextcall = bt; 840 /* If there is some other event set earlier -- do nothing. */ 841 if (bt >= state->nextevent) 842 goto done; 843 state->nextevent = bt; 844 /* If timer is periodic -- there is nothing to reprogram. */ 845 if (periodic) 846 goto done; 847 /* If timer is global or of the current CPU -- reprogram it. */ 848 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) { 849 loadtimer(sbinuptime(), 0); 850 done: 851 ET_HW_UNLOCK(state); 852 return; 853 } 854 /* Otherwise make other CPU to reprogram it. */ 855 state->handle = 1; 856 ET_HW_UNLOCK(state); 857 #ifdef SMP 858 ipi_cpu(cpu, IPI_HARDCLOCK); 859 #endif 860 } 861 862 /* 863 * Report or change the active event timers hardware. 864 */ 865 static int 866 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS) 867 { 868 char buf[32]; 869 struct eventtimer *et; 870 int error; 871 872 ET_LOCK(); 873 et = timer; 874 snprintf(buf, sizeof(buf), "%s", et->et_name); 875 ET_UNLOCK(); 876 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 877 ET_LOCK(); 878 et = timer; 879 if (error != 0 || req->newptr == NULL || 880 strcasecmp(buf, et->et_name) == 0) { 881 ET_UNLOCK(); 882 return (error); 883 } 884 et = et_find(buf, 0, 0); 885 if (et == NULL) { 886 ET_UNLOCK(); 887 return (ENOENT); 888 } 889 configtimer(0); 890 et_free(timer); 891 if (et->et_flags & ET_FLAGS_C3STOP) 892 cpu_disable_c3_sleep++; 893 if (timer->et_flags & ET_FLAGS_C3STOP) 894 cpu_disable_c3_sleep--; 895 periodic = want_periodic; 896 timer = et; 897 et_init(timer, timercb, NULL, NULL); 898 configtimer(1); 899 ET_UNLOCK(); 900 return (error); 901 } 902 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer, 903 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 904 0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer"); 905 906 /* 907 * Report or change the active event timer periodicity. 908 */ 909 static int 910 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS) 911 { 912 int error, val; 913 914 val = periodic; 915 error = sysctl_handle_int(oidp, &val, 0, req); 916 if (error != 0 || req->newptr == NULL) 917 return (error); 918 ET_LOCK(); 919 configtimer(0); 920 periodic = want_periodic = val; 921 configtimer(1); 922 ET_UNLOCK(); 923 return (error); 924 } 925 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic, 926 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 927 0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode"); 928 929 #include "opt_ddb.h" 930 931 #ifdef DDB 932 #include <ddb/ddb.h> 933 934 DB_SHOW_COMMAND(clocksource, db_show_clocksource) 935 { 936 struct pcpu_state *st; 937 int c; 938 939 CPU_FOREACH(c) { 940 st = DPCPU_ID_PTR(c, timerstate); 941 db_printf( 942 "CPU %2d: action %d handle %d ipi %d idle %d\n" 943 " now %#jx nevent %#jx (%jd)\n" 944 " ntick %#jx (%jd) nhard %#jx (%jd)\n" 945 " nstat %#jx (%jd) nprof %#jx (%jd)\n" 946 " ncall %#jx (%jd) ncallopt %#jx (%jd)\n", 947 c, st->action, st->handle, st->ipi, st->idle, 948 (uintmax_t)st->now, 949 (uintmax_t)st->nextevent, 950 (uintmax_t)(st->nextevent - st->now) / tick_sbt, 951 (uintmax_t)st->nexttick, 952 (uintmax_t)(st->nexttick - st->now) / tick_sbt, 953 (uintmax_t)st->nexthard, 954 (uintmax_t)(st->nexthard - st->now) / tick_sbt, 955 (uintmax_t)st->nextstat, 956 (uintmax_t)(st->nextstat - st->now) / tick_sbt, 957 (uintmax_t)st->nextprof, 958 (uintmax_t)(st->nextprof - st->now) / tick_sbt, 959 (uintmax_t)st->nextcall, 960 (uintmax_t)(st->nextcall - st->now) / tick_sbt, 961 (uintmax_t)st->nextcallopt, 962 (uintmax_t)(st->nextcallopt - st->now) / tick_sbt); 963 } 964 } 965 966 #endif 967