1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer, 12 * without modification, immediately at the beginning of the file. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 /* 33 * Common routines to manage event timers hardware. 34 */ 35 36 #include "opt_device_polling.h" 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/limits.h> 42 #include <sys/lock.h> 43 #include <sys/kdb.h> 44 #include <sys/ktr.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/kernel.h> 48 #include <sys/sched.h> 49 #include <sys/smp.h> 50 #include <sys/sysctl.h> 51 #include <sys/timeet.h> 52 #include <sys/timetc.h> 53 54 #include <machine/atomic.h> 55 #include <machine/clock.h> 56 #include <machine/cpu.h> 57 #include <machine/smp.h> 58 59 int cpu_disable_c2_sleep = 0; /* Timer dies in C2. */ 60 int cpu_disable_c3_sleep = 0; /* Timer dies in C3. */ 61 62 static void setuptimer(void); 63 static void loadtimer(sbintime_t now, int first); 64 static int doconfigtimer(void); 65 static void configtimer(int start); 66 static int round_freq(struct eventtimer *et, int freq); 67 68 static sbintime_t getnextcpuevent(int idle); 69 static sbintime_t getnextevent(void); 70 static int handleevents(sbintime_t now, int fake); 71 72 static struct mtx et_hw_mtx; 73 74 #define ET_HW_LOCK(state) \ 75 { \ 76 if (timer->et_flags & ET_FLAGS_PERCPU) \ 77 mtx_lock_spin(&(state)->et_hw_mtx); \ 78 else \ 79 mtx_lock_spin(&et_hw_mtx); \ 80 } 81 82 #define ET_HW_UNLOCK(state) \ 83 { \ 84 if (timer->et_flags & ET_FLAGS_PERCPU) \ 85 mtx_unlock_spin(&(state)->et_hw_mtx); \ 86 else \ 87 mtx_unlock_spin(&et_hw_mtx); \ 88 } 89 90 static struct eventtimer *timer = NULL; 91 static sbintime_t timerperiod; /* Timer period for periodic mode. */ 92 static sbintime_t statperiod; /* statclock() events period. */ 93 static sbintime_t profperiod; /* profclock() events period. */ 94 static sbintime_t nexttick; /* Next global timer tick time. */ 95 static u_int busy = 1; /* Reconfiguration is in progress. */ 96 static int profiling; /* Profiling events enabled. */ 97 98 static char timername[32]; /* Wanted timer. */ 99 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername)); 100 101 static int singlemul; /* Multiplier for periodic mode. */ 102 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RWTUN, &singlemul, 103 0, "Multiplier for periodic mode"); 104 105 static u_int idletick; /* Run periodic events when idle. */ 106 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RWTUN, &idletick, 107 0, "Run periodic events when idle"); 108 109 static int periodic; /* Periodic or one-shot mode. */ 110 static int want_periodic; /* What mode to prefer. */ 111 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic); 112 113 struct pcpu_state { 114 struct mtx et_hw_mtx; /* Per-CPU timer mutex. */ 115 u_int action; /* Reconfiguration requests. */ 116 u_int handle; /* Immediate handle resuests. */ 117 sbintime_t now; /* Last tick time. */ 118 sbintime_t nextevent; /* Next scheduled event on this CPU. */ 119 sbintime_t nexttick; /* Next timer tick time. */ 120 sbintime_t nexthard; /* Next hardclock() event. */ 121 sbintime_t nextstat; /* Next statclock() event. */ 122 sbintime_t nextprof; /* Next profclock() event. */ 123 sbintime_t nextcall; /* Next callout event. */ 124 sbintime_t nextcallopt; /* Next optional callout event. */ 125 int ipi; /* This CPU needs IPI. */ 126 int idle; /* This CPU is in idle mode. */ 127 }; 128 129 DPCPU_DEFINE_STATIC(struct pcpu_state, timerstate); 130 DPCPU_DEFINE(sbintime_t, hardclocktime); 131 132 /* 133 * Timer broadcast IPI handler. 134 */ 135 int 136 hardclockintr(void) 137 { 138 sbintime_t now; 139 struct pcpu_state *state; 140 int done; 141 142 if (doconfigtimer() || busy) 143 return (FILTER_HANDLED); 144 state = DPCPU_PTR(timerstate); 145 now = state->now; 146 CTR3(KTR_SPARE2, "ipi at %d: now %d.%08x", 147 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 148 done = handleevents(now, 0); 149 return (done ? FILTER_HANDLED : FILTER_STRAY); 150 } 151 152 /* 153 * Handle all events for specified time on this CPU 154 */ 155 static int 156 handleevents(sbintime_t now, int fake) 157 { 158 sbintime_t t, *hct; 159 struct trapframe *frame; 160 struct pcpu_state *state; 161 int usermode; 162 int done, runs; 163 164 CTR3(KTR_SPARE2, "handle at %d: now %d.%08x", 165 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 166 done = 0; 167 if (fake) { 168 frame = NULL; 169 usermode = 0; 170 } else { 171 frame = curthread->td_intr_frame; 172 usermode = TRAPF_USERMODE(frame); 173 } 174 175 state = DPCPU_PTR(timerstate); 176 177 runs = 0; 178 while (now >= state->nexthard) { 179 state->nexthard += tick_sbt; 180 runs++; 181 } 182 if (runs) { 183 hct = DPCPU_PTR(hardclocktime); 184 *hct = state->nexthard - tick_sbt; 185 if (fake < 2) { 186 hardclock(runs, usermode); 187 done = 1; 188 } 189 } 190 runs = 0; 191 while (now >= state->nextstat) { 192 state->nextstat += statperiod; 193 runs++; 194 } 195 if (runs && fake < 2) { 196 statclock(runs, usermode); 197 done = 1; 198 } 199 if (profiling) { 200 runs = 0; 201 while (now >= state->nextprof) { 202 state->nextprof += profperiod; 203 runs++; 204 } 205 if (runs && !fake) { 206 profclock(runs, usermode, TRAPF_PC(frame)); 207 done = 1; 208 } 209 } else 210 state->nextprof = state->nextstat; 211 if (now >= state->nextcallopt || now >= state->nextcall) { 212 state->nextcall = state->nextcallopt = SBT_MAX; 213 callout_process(now); 214 } 215 216 t = getnextcpuevent(0); 217 ET_HW_LOCK(state); 218 if (!busy) { 219 state->idle = 0; 220 state->nextevent = t; 221 loadtimer(now, (fake == 2) && 222 (timer->et_flags & ET_FLAGS_PERCPU)); 223 } 224 ET_HW_UNLOCK(state); 225 return (done); 226 } 227 228 /* 229 * Schedule binuptime of the next event on current CPU. 230 */ 231 static sbintime_t 232 getnextcpuevent(int idle) 233 { 234 sbintime_t event; 235 struct pcpu_state *state; 236 u_int hardfreq; 237 238 state = DPCPU_PTR(timerstate); 239 /* Handle hardclock() events, skipping some if CPU is idle. */ 240 event = state->nexthard; 241 if (idle) { 242 if (tc_min_ticktock_freq > 1 243 #ifdef SMP 244 && curcpu == CPU_FIRST() 245 #endif 246 ) 247 hardfreq = hz / tc_min_ticktock_freq; 248 else 249 hardfreq = hz; 250 if (hardfreq > 1) 251 event += tick_sbt * (hardfreq - 1); 252 } 253 /* Handle callout events. */ 254 if (event > state->nextcall) 255 event = state->nextcall; 256 if (!idle) { /* If CPU is active - handle other types of events. */ 257 if (event > state->nextstat) 258 event = state->nextstat; 259 if (profiling && event > state->nextprof) 260 event = state->nextprof; 261 } 262 return (event); 263 } 264 265 /* 266 * Schedule binuptime of the next event on all CPUs. 267 */ 268 static sbintime_t 269 getnextevent(void) 270 { 271 struct pcpu_state *state; 272 sbintime_t event; 273 #ifdef SMP 274 int cpu; 275 #endif 276 #ifdef KTR 277 int c; 278 279 c = -1; 280 #endif 281 state = DPCPU_PTR(timerstate); 282 event = state->nextevent; 283 #ifdef SMP 284 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) { 285 CPU_FOREACH(cpu) { 286 state = DPCPU_ID_PTR(cpu, timerstate); 287 if (event > state->nextevent) { 288 event = state->nextevent; 289 #ifdef KTR 290 c = cpu; 291 #endif 292 } 293 } 294 } 295 #endif 296 CTR4(KTR_SPARE2, "next at %d: next %d.%08x by %d", 297 curcpu, (int)(event >> 32), (u_int)(event & 0xffffffff), c); 298 return (event); 299 } 300 301 /* Hardware timer callback function. */ 302 static void 303 timercb(struct eventtimer *et, void *arg) 304 { 305 sbintime_t now; 306 sbintime_t *next; 307 struct pcpu_state *state; 308 #ifdef SMP 309 int cpu, bcast; 310 #endif 311 312 /* Do not touch anything if somebody reconfiguring timers. */ 313 if (busy) 314 return; 315 /* Update present and next tick times. */ 316 state = DPCPU_PTR(timerstate); 317 if (et->et_flags & ET_FLAGS_PERCPU) { 318 next = &state->nexttick; 319 } else 320 next = &nexttick; 321 now = sbinuptime(); 322 if (periodic) 323 *next = now + timerperiod; 324 else 325 *next = -1; /* Next tick is not scheduled yet. */ 326 state->now = now; 327 CTR3(KTR_SPARE2, "intr at %d: now %d.%08x", 328 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 329 330 #ifdef SMP 331 #ifdef EARLY_AP_STARTUP 332 MPASS(mp_ncpus == 1 || smp_started); 333 #endif 334 /* Prepare broadcasting to other CPUs for non-per-CPU timers. */ 335 bcast = 0; 336 #ifdef EARLY_AP_STARTUP 337 if ((et->et_flags & ET_FLAGS_PERCPU) == 0) { 338 #else 339 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) { 340 #endif 341 CPU_FOREACH(cpu) { 342 state = DPCPU_ID_PTR(cpu, timerstate); 343 ET_HW_LOCK(state); 344 state->now = now; 345 if (now >= state->nextevent) { 346 state->nextevent += SBT_1S; 347 if (curcpu != cpu) { 348 state->ipi = 1; 349 bcast = 1; 350 } 351 } 352 ET_HW_UNLOCK(state); 353 } 354 } 355 #endif 356 357 /* Handle events for this time on this CPU. */ 358 handleevents(now, 0); 359 360 #ifdef SMP 361 /* Broadcast interrupt to other CPUs for non-per-CPU timers. */ 362 if (bcast) { 363 CPU_FOREACH(cpu) { 364 if (curcpu == cpu) 365 continue; 366 state = DPCPU_ID_PTR(cpu, timerstate); 367 if (state->ipi) { 368 state->ipi = 0; 369 ipi_cpu(cpu, IPI_HARDCLOCK); 370 } 371 } 372 } 373 #endif 374 } 375 376 /* 377 * Load new value into hardware timer. 378 */ 379 static void 380 loadtimer(sbintime_t now, int start) 381 { 382 struct pcpu_state *state; 383 sbintime_t new; 384 sbintime_t *next; 385 uint64_t tmp; 386 int eq; 387 388 if (timer->et_flags & ET_FLAGS_PERCPU) { 389 state = DPCPU_PTR(timerstate); 390 next = &state->nexttick; 391 } else 392 next = &nexttick; 393 if (periodic) { 394 if (start) { 395 /* 396 * Try to start all periodic timers aligned 397 * to period to make events synchronous. 398 */ 399 tmp = now % timerperiod; 400 new = timerperiod - tmp; 401 if (new < tmp) /* Left less then passed. */ 402 new += timerperiod; 403 CTR5(KTR_SPARE2, "load p at %d: now %d.%08x first in %d.%08x", 404 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff), 405 (int)(new >> 32), (u_int)(new & 0xffffffff)); 406 *next = new + now; 407 et_start(timer, new, timerperiod); 408 } 409 } else { 410 new = getnextevent(); 411 eq = (new == *next); 412 CTR4(KTR_SPARE2, "load at %d: next %d.%08x eq %d", 413 curcpu, (int)(new >> 32), (u_int)(new & 0xffffffff), eq); 414 if (!eq) { 415 *next = new; 416 et_start(timer, new - now, 0); 417 } 418 } 419 } 420 421 /* 422 * Prepare event timer parameters after configuration changes. 423 */ 424 static void 425 setuptimer(void) 426 { 427 int freq; 428 429 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0) 430 periodic = 0; 431 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0) 432 periodic = 1; 433 singlemul = MIN(MAX(singlemul, 1), 20); 434 freq = hz * singlemul; 435 while (freq < (profiling ? profhz : stathz)) 436 freq += hz; 437 freq = round_freq(timer, freq); 438 timerperiod = SBT_1S / freq; 439 } 440 441 /* 442 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler. 443 */ 444 static int 445 doconfigtimer(void) 446 { 447 sbintime_t now; 448 struct pcpu_state *state; 449 450 state = DPCPU_PTR(timerstate); 451 switch (atomic_load_acq_int(&state->action)) { 452 case 1: 453 now = sbinuptime(); 454 ET_HW_LOCK(state); 455 loadtimer(now, 1); 456 ET_HW_UNLOCK(state); 457 state->handle = 0; 458 atomic_store_rel_int(&state->action, 0); 459 return (1); 460 case 2: 461 ET_HW_LOCK(state); 462 et_stop(timer); 463 ET_HW_UNLOCK(state); 464 state->handle = 0; 465 atomic_store_rel_int(&state->action, 0); 466 return (1); 467 } 468 if (atomic_readandclear_int(&state->handle) && !busy) { 469 now = sbinuptime(); 470 handleevents(now, 0); 471 return (1); 472 } 473 return (0); 474 } 475 476 /* 477 * Reconfigure specified timer. 478 * For per-CPU timers use IPI to make other CPUs to reconfigure. 479 */ 480 static void 481 configtimer(int start) 482 { 483 sbintime_t now, next; 484 struct pcpu_state *state; 485 int cpu; 486 487 if (start) { 488 setuptimer(); 489 now = sbinuptime(); 490 } else 491 now = 0; 492 critical_enter(); 493 ET_HW_LOCK(DPCPU_PTR(timerstate)); 494 if (start) { 495 /* Initialize time machine parameters. */ 496 next = now + timerperiod; 497 if (periodic) 498 nexttick = next; 499 else 500 nexttick = -1; 501 #ifdef EARLY_AP_STARTUP 502 MPASS(mp_ncpus == 1 || smp_started); 503 #endif 504 CPU_FOREACH(cpu) { 505 state = DPCPU_ID_PTR(cpu, timerstate); 506 state->now = now; 507 #ifndef EARLY_AP_STARTUP 508 if (!smp_started && cpu != CPU_FIRST()) 509 state->nextevent = SBT_MAX; 510 else 511 #endif 512 state->nextevent = next; 513 if (periodic) 514 state->nexttick = next; 515 else 516 state->nexttick = -1; 517 state->nexthard = next; 518 state->nextstat = next; 519 state->nextprof = next; 520 state->nextcall = next; 521 state->nextcallopt = next; 522 hardclock_sync(cpu); 523 } 524 busy = 0; 525 /* Start global timer or per-CPU timer of this CPU. */ 526 loadtimer(now, 1); 527 } else { 528 busy = 1; 529 /* Stop global timer or per-CPU timer of this CPU. */ 530 et_stop(timer); 531 } 532 ET_HW_UNLOCK(DPCPU_PTR(timerstate)); 533 #ifdef SMP 534 #ifdef EARLY_AP_STARTUP 535 /* If timer is global we are done. */ 536 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) { 537 #else 538 /* If timer is global or there is no other CPUs yet - we are done. */ 539 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) { 540 #endif 541 critical_exit(); 542 return; 543 } 544 /* Set reconfigure flags for other CPUs. */ 545 CPU_FOREACH(cpu) { 546 state = DPCPU_ID_PTR(cpu, timerstate); 547 atomic_store_rel_int(&state->action, 548 (cpu == curcpu) ? 0 : ( start ? 1 : 2)); 549 } 550 /* Broadcast reconfigure IPI. */ 551 ipi_all_but_self(IPI_HARDCLOCK); 552 /* Wait for reconfiguration completed. */ 553 restart: 554 cpu_spinwait(); 555 CPU_FOREACH(cpu) { 556 if (cpu == curcpu) 557 continue; 558 state = DPCPU_ID_PTR(cpu, timerstate); 559 if (atomic_load_acq_int(&state->action)) 560 goto restart; 561 } 562 #endif 563 critical_exit(); 564 } 565 566 /* 567 * Calculate nearest frequency supported by hardware timer. 568 */ 569 static int 570 round_freq(struct eventtimer *et, int freq) 571 { 572 uint64_t div; 573 574 if (et->et_frequency != 0) { 575 div = lmax((et->et_frequency + freq / 2) / freq, 1); 576 if (et->et_flags & ET_FLAGS_POW2DIV) 577 div = 1 << (flsl(div + div / 2) - 1); 578 freq = (et->et_frequency + div / 2) / div; 579 } 580 if (et->et_min_period > SBT_1S) 581 panic("Event timer \"%s\" doesn't support sub-second periods!", 582 et->et_name); 583 else if (et->et_min_period != 0) 584 freq = min(freq, SBT2FREQ(et->et_min_period)); 585 if (et->et_max_period < SBT_1S && et->et_max_period != 0) 586 freq = max(freq, SBT2FREQ(et->et_max_period)); 587 return (freq); 588 } 589 590 /* 591 * Configure and start event timers (BSP part). 592 */ 593 void 594 cpu_initclocks_bsp(void) 595 { 596 struct pcpu_state *state; 597 int base, div, cpu; 598 599 mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN); 600 CPU_FOREACH(cpu) { 601 state = DPCPU_ID_PTR(cpu, timerstate); 602 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN); 603 state->nextcall = SBT_MAX; 604 state->nextcallopt = SBT_MAX; 605 } 606 periodic = want_periodic; 607 /* Grab requested timer or the best of present. */ 608 if (timername[0]) 609 timer = et_find(timername, 0, 0); 610 if (timer == NULL && periodic) { 611 timer = et_find(NULL, 612 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 613 } 614 if (timer == NULL) { 615 timer = et_find(NULL, 616 ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT); 617 } 618 if (timer == NULL && !periodic) { 619 timer = et_find(NULL, 620 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC); 621 } 622 if (timer == NULL) 623 panic("No usable event timer found!"); 624 et_init(timer, timercb, NULL, NULL); 625 626 /* Adapt to timer capabilities. */ 627 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0) 628 periodic = 0; 629 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0) 630 periodic = 1; 631 if (timer->et_flags & ET_FLAGS_C3STOP) 632 cpu_disable_c3_sleep++; 633 634 /* 635 * We honor the requested 'hz' value. 636 * We want to run stathz in the neighborhood of 128hz. 637 * We would like profhz to run as often as possible. 638 */ 639 if (singlemul <= 0 || singlemul > 20) { 640 if (hz >= 1500 || (hz % 128) == 0) 641 singlemul = 1; 642 else if (hz >= 750) 643 singlemul = 2; 644 else 645 singlemul = 4; 646 } 647 if (periodic) { 648 base = round_freq(timer, hz * singlemul); 649 singlemul = max((base + hz / 2) / hz, 1); 650 hz = (base + singlemul / 2) / singlemul; 651 if (base <= 128) 652 stathz = base; 653 else { 654 div = base / 128; 655 if (div >= singlemul && (div % singlemul) == 0) 656 div++; 657 stathz = base / div; 658 } 659 profhz = stathz; 660 while ((profhz + stathz) <= 128 * 64) 661 profhz += stathz; 662 profhz = round_freq(timer, profhz); 663 } else { 664 hz = round_freq(timer, hz); 665 stathz = round_freq(timer, 127); 666 profhz = round_freq(timer, stathz * 64); 667 } 668 tick = 1000000 / hz; 669 tick_sbt = SBT_1S / hz; 670 tick_bt = sbttobt(tick_sbt); 671 statperiod = SBT_1S / stathz; 672 profperiod = SBT_1S / profhz; 673 ET_LOCK(); 674 configtimer(1); 675 ET_UNLOCK(); 676 } 677 678 /* 679 * Start per-CPU event timers on APs. 680 */ 681 void 682 cpu_initclocks_ap(void) 683 { 684 sbintime_t now; 685 struct pcpu_state *state; 686 struct thread *td; 687 688 state = DPCPU_PTR(timerstate); 689 now = sbinuptime(); 690 ET_HW_LOCK(state); 691 state->now = now; 692 hardclock_sync(curcpu); 693 spinlock_enter(); 694 ET_HW_UNLOCK(state); 695 td = curthread; 696 td->td_intr_nesting_level++; 697 handleevents(state->now, 2); 698 td->td_intr_nesting_level--; 699 spinlock_exit(); 700 } 701 702 void 703 suspendclock(void) 704 { 705 ET_LOCK(); 706 configtimer(0); 707 ET_UNLOCK(); 708 } 709 710 void 711 resumeclock(void) 712 { 713 ET_LOCK(); 714 configtimer(1); 715 ET_UNLOCK(); 716 } 717 718 /* 719 * Switch to profiling clock rates. 720 */ 721 void 722 cpu_startprofclock(void) 723 { 724 725 ET_LOCK(); 726 if (profiling == 0) { 727 if (periodic) { 728 configtimer(0); 729 profiling = 1; 730 configtimer(1); 731 } else 732 profiling = 1; 733 } else 734 profiling++; 735 ET_UNLOCK(); 736 } 737 738 /* 739 * Switch to regular clock rates. 740 */ 741 void 742 cpu_stopprofclock(void) 743 { 744 745 ET_LOCK(); 746 if (profiling == 1) { 747 if (periodic) { 748 configtimer(0); 749 profiling = 0; 750 configtimer(1); 751 } else 752 profiling = 0; 753 } else 754 profiling--; 755 ET_UNLOCK(); 756 } 757 758 /* 759 * Switch to idle mode (all ticks handled). 760 */ 761 sbintime_t 762 cpu_idleclock(void) 763 { 764 sbintime_t now, t; 765 struct pcpu_state *state; 766 767 if (idletick || busy || 768 (periodic && (timer->et_flags & ET_FLAGS_PERCPU)) 769 #ifdef DEVICE_POLLING 770 || curcpu == CPU_FIRST() 771 #endif 772 ) 773 return (-1); 774 state = DPCPU_PTR(timerstate); 775 if (periodic) 776 now = state->now; 777 else 778 now = sbinuptime(); 779 CTR3(KTR_SPARE2, "idle at %d: now %d.%08x", 780 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 781 t = getnextcpuevent(1); 782 ET_HW_LOCK(state); 783 state->idle = 1; 784 state->nextevent = t; 785 if (!periodic) 786 loadtimer(now, 0); 787 ET_HW_UNLOCK(state); 788 return (MAX(t - now, 0)); 789 } 790 791 /* 792 * Switch to active mode (skip empty ticks). 793 */ 794 void 795 cpu_activeclock(void) 796 { 797 sbintime_t now; 798 struct pcpu_state *state; 799 struct thread *td; 800 801 state = DPCPU_PTR(timerstate); 802 if (state->idle == 0 || busy) 803 return; 804 if (periodic) 805 now = state->now; 806 else 807 now = sbinuptime(); 808 CTR3(KTR_SPARE2, "active at %d: now %d.%08x", 809 curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff)); 810 spinlock_enter(); 811 td = curthread; 812 td->td_intr_nesting_level++; 813 handleevents(now, 1); 814 td->td_intr_nesting_level--; 815 spinlock_exit(); 816 } 817 818 /* 819 * Change the frequency of the given timer. This changes et->et_frequency and 820 * if et is the active timer it reconfigures the timer on all CPUs. This is 821 * intended to be a private interface for the use of et_change_frequency() only. 822 */ 823 void 824 cpu_et_frequency(struct eventtimer *et, uint64_t newfreq) 825 { 826 827 ET_LOCK(); 828 if (et == timer) { 829 configtimer(0); 830 et->et_frequency = newfreq; 831 configtimer(1); 832 } else 833 et->et_frequency = newfreq; 834 ET_UNLOCK(); 835 } 836 837 void 838 cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt) 839 { 840 struct pcpu_state *state; 841 842 /* Do not touch anything if somebody reconfiguring timers. */ 843 if (busy) 844 return; 845 CTR6(KTR_SPARE2, "new co at %d: on %d at %d.%08x - %d.%08x", 846 curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff), 847 (int)(bt >> 32), (u_int)(bt & 0xffffffff)); 848 849 KASSERT(!CPU_ABSENT(cpu), ("Absent CPU %d", cpu)); 850 state = DPCPU_ID_PTR(cpu, timerstate); 851 ET_HW_LOCK(state); 852 853 /* 854 * If there is callout time already set earlier -- do nothing. 855 * This check may appear redundant because we check already in 856 * callout_process() but this double check guarantees we're safe 857 * with respect to race conditions between interrupts execution 858 * and scheduling. 859 */ 860 state->nextcallopt = bt_opt; 861 if (bt >= state->nextcall) 862 goto done; 863 state->nextcall = bt; 864 /* If there is some other event set earlier -- do nothing. */ 865 if (bt >= state->nextevent) 866 goto done; 867 state->nextevent = bt; 868 /* If timer is periodic -- there is nothing to reprogram. */ 869 if (periodic) 870 goto done; 871 /* If timer is global or of the current CPU -- reprogram it. */ 872 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) { 873 loadtimer(sbinuptime(), 0); 874 done: 875 ET_HW_UNLOCK(state); 876 return; 877 } 878 /* Otherwise make other CPU to reprogram it. */ 879 state->handle = 1; 880 ET_HW_UNLOCK(state); 881 #ifdef SMP 882 ipi_cpu(cpu, IPI_HARDCLOCK); 883 #endif 884 } 885 886 /* 887 * Report or change the active event timers hardware. 888 */ 889 static int 890 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS) 891 { 892 char buf[32]; 893 struct eventtimer *et; 894 int error; 895 896 ET_LOCK(); 897 et = timer; 898 snprintf(buf, sizeof(buf), "%s", et->et_name); 899 ET_UNLOCK(); 900 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 901 ET_LOCK(); 902 et = timer; 903 if (error != 0 || req->newptr == NULL || 904 strcasecmp(buf, et->et_name) == 0) { 905 ET_UNLOCK(); 906 return (error); 907 } 908 et = et_find(buf, 0, 0); 909 if (et == NULL) { 910 ET_UNLOCK(); 911 return (ENOENT); 912 } 913 configtimer(0); 914 et_free(timer); 915 if (et->et_flags & ET_FLAGS_C3STOP) 916 cpu_disable_c3_sleep++; 917 if (timer->et_flags & ET_FLAGS_C3STOP) 918 cpu_disable_c3_sleep--; 919 periodic = want_periodic; 920 timer = et; 921 et_init(timer, timercb, NULL, NULL); 922 configtimer(1); 923 ET_UNLOCK(); 924 return (error); 925 } 926 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer, 927 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 928 0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer"); 929 930 /* 931 * Report or change the active event timer periodicity. 932 */ 933 static int 934 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS) 935 { 936 int error, val; 937 938 val = periodic; 939 error = sysctl_handle_int(oidp, &val, 0, req); 940 if (error != 0 || req->newptr == NULL) 941 return (error); 942 ET_LOCK(); 943 configtimer(0); 944 periodic = want_periodic = val; 945 configtimer(1); 946 ET_UNLOCK(); 947 return (error); 948 } 949 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic, 950 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 951 0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode"); 952 953 #include "opt_ddb.h" 954 955 #ifdef DDB 956 #include <ddb/ddb.h> 957 958 DB_SHOW_COMMAND(clocksource, db_show_clocksource) 959 { 960 struct pcpu_state *st; 961 int c; 962 963 CPU_FOREACH(c) { 964 st = DPCPU_ID_PTR(c, timerstate); 965 db_printf( 966 "CPU %2d: action %d handle %d ipi %d idle %d\n" 967 " now %#jx nevent %#jx (%jd)\n" 968 " ntick %#jx (%jd) nhard %#jx (%jd)\n" 969 " nstat %#jx (%jd) nprof %#jx (%jd)\n" 970 " ncall %#jx (%jd) ncallopt %#jx (%jd)\n", 971 c, st->action, st->handle, st->ipi, st->idle, 972 (uintmax_t)st->now, 973 (uintmax_t)st->nextevent, 974 (uintmax_t)(st->nextevent - st->now) / tick_sbt, 975 (uintmax_t)st->nexttick, 976 (uintmax_t)(st->nexttick - st->now) / tick_sbt, 977 (uintmax_t)st->nexthard, 978 (uintmax_t)(st->nexthard - st->now) / tick_sbt, 979 (uintmax_t)st->nextstat, 980 (uintmax_t)(st->nextstat - st->now) / tick_sbt, 981 (uintmax_t)st->nextprof, 982 (uintmax_t)(st->nextprof - st->now) / tick_sbt, 983 (uintmax_t)st->nextcall, 984 (uintmax_t)(st->nextcall - st->now) / tick_sbt, 985 (uintmax_t)st->nextcallopt, 986 (uintmax_t)(st->nextcallopt - st->now) / tick_sbt); 987 } 988 } 989 990 #endif 991