1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #define PSMI_1_6 27 #include <sys/smp_impldefs.h> 28 #include <sys/psm.h> 29 #include <sys/psm_modctl.h> 30 #include <sys/pit.h> 31 #include <sys/cmn_err.h> 32 #include <sys/strlog.h> 33 #include <sys/clock.h> 34 #include <sys/debug.h> 35 #include <sys/rtc.h> 36 #include <sys/x86_archext.h> 37 #include <sys/cpupart.h> 38 #include <sys/cpuvar.h> 39 #include <sys/cpu_event.h> 40 #include <sys/cmt.h> 41 #include <sys/cpu.h> 42 #include <sys/disp.h> 43 #include <sys/archsystm.h> 44 #include <sys/machsystm.h> 45 #include <sys/sysmacros.h> 46 #include <sys/memlist.h> 47 #include <sys/param.h> 48 #include <sys/promif.h> 49 #include <sys/cpu_pm.h> 50 #if defined(__xpv) 51 #include <sys/hypervisor.h> 52 #endif 53 #include <sys/mach_intr.h> 54 #include <vm/hat_i86.h> 55 #include <sys/kdi_machimpl.h> 56 #include <sys/sdt.h> 57 #include <sys/hpet.h> 58 59 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 60 61 /* 62 * Local function prototypes 63 */ 64 static int mp_disable_intr(processorid_t cpun); 65 static void mp_enable_intr(processorid_t cpun); 66 static void mach_init(); 67 static void mach_picinit(); 68 static int machhztomhz(uint64_t cpu_freq_hz); 69 static uint64_t mach_getcpufreq(void); 70 static void mach_fixcpufreq(void); 71 static int mach_clkinit(int, int *); 72 static void mach_smpinit(void); 73 static int mach_softlvl_to_vect(int ipl); 74 static void mach_get_platform(int owner); 75 static void mach_construct_info(); 76 static int mach_translate_irq(dev_info_t *dip, int irqno); 77 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 78 psm_intr_op_t, int *); 79 static void mach_notify_error(int level, char *errmsg); 80 static hrtime_t dummy_hrtime(void); 81 static void dummy_scalehrtime(hrtime_t *); 82 void cpu_idle(void); 83 static void cpu_wakeup(cpu_t *, int); 84 #ifndef __xpv 85 void cpu_idle_mwait(void); 86 static void cpu_wakeup_mwait(cpu_t *, int); 87 #endif 88 /* 89 * External reference functions 90 */ 91 extern void return_instr(); 92 extern uint64_t freq_tsc(uint32_t *); 93 #if defined(__i386) 94 extern uint64_t freq_notsc(uint32_t *); 95 #endif 96 extern void pc_gethrestime(timestruc_t *); 97 extern int cpuid_get_coreid(cpu_t *); 98 extern int cpuid_get_chipid(cpu_t *); 99 100 /* 101 * PSM functions initialization 102 */ 103 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 104 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 105 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 106 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 107 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 108 void (*psminitf)() = mach_init; 109 void (*picinitf)() = return_instr; 110 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 111 int (*ap_mlsetup)() = (int (*)(void))return_instr; 112 void (*send_dirintf)() = return_instr; 113 void (*setspl)(int) = (void (*)(int))return_instr; 114 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 115 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 116 void (*kdisetsoftint)(int, struct av_softinfo *)= 117 (void (*)(int, struct av_softinfo *))return_instr; 118 void (*setsoftint)(int, struct av_softinfo *)= 119 (void (*)(int, struct av_softinfo *))return_instr; 120 int (*slvltovect)(int) = (int (*)(int))return_instr; 121 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 122 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 123 int (*psm_disable_intr)(int) = mp_disable_intr; 124 void (*psm_enable_intr)(int) = mp_enable_intr; 125 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 126 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 127 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 128 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 129 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 130 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 131 int (*psm_get_clockirq)(int) = NULL; 132 int (*psm_get_ipivect)(int, int) = NULL; 133 134 int (*psm_clkinit)(int) = NULL; 135 void (*psm_timer_reprogram)(hrtime_t) = NULL; 136 void (*psm_timer_enable)(void) = NULL; 137 void (*psm_timer_disable)(void) = NULL; 138 void (*psm_post_cyclic_setup)(void *arg) = NULL; 139 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 140 int *) = mach_intr_ops; 141 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) 142 return_instr; 143 144 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 145 void (*hrtime_tick)(void) = return_instr; 146 147 /* 148 * True if the generic TSC code is our source of hrtime, rather than whatever 149 * the PSM can provide. 150 */ 151 #ifdef __xpv 152 int tsc_gethrtime_enable = 0; 153 #else 154 int tsc_gethrtime_enable = 1; 155 #endif 156 int tsc_gethrtime_initted = 0; 157 158 /* 159 * True if the hrtime implementation is "hires"; namely, better than microdata. 160 */ 161 int gethrtime_hires = 0; 162 163 /* 164 * Local Static Data 165 */ 166 static struct psm_ops mach_ops; 167 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 168 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 169 170 /* 171 * virtualization support for psm 172 */ 173 void *psm_vt_ops = NULL; 174 /* 175 * If non-zero, idle cpus will become "halted" when there's 176 * no work to do. 177 */ 178 int idle_cpu_use_hlt = 1; 179 180 #ifndef __xpv 181 /* 182 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 183 */ 184 int idle_cpu_prefer_mwait = 1; 185 /* 186 * Set to 0 to avoid MONITOR+CLFLUSH assertion. 187 */ 188 int idle_cpu_assert_cflush_monitor = 1; 189 190 /* 191 * If non-zero, idle cpus will not use power saving Deep C-States idle loop. 192 */ 193 int idle_cpu_no_deep_c = 0; 194 /* 195 * Non-power saving idle loop and wakeup pointers. 196 * Allows user to toggle Deep Idle power saving feature on/off. 197 */ 198 void (*non_deep_idle_cpu)() = cpu_idle; 199 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); 200 201 /* 202 * Object for the kernel to access the HPET. 203 */ 204 hpet_t hpet; 205 206 #endif /* ifndef __xpv */ 207 208 /*ARGSUSED*/ 209 int 210 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 211 { 212 switch (hw) { 213 case PGHW_IPIPE: 214 if (x86_feature & (X86_HTT)) { 215 /* 216 * Hyper-threading is SMT 217 */ 218 return (1); 219 } else { 220 return (0); 221 } 222 case PGHW_CHIP: 223 if (x86_feature & (X86_CMP|X86_HTT)) 224 return (1); 225 else 226 return (0); 227 case PGHW_CACHE: 228 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 229 return (1); 230 else 231 return (0); 232 case PGHW_POW_ACTIVE: 233 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) 234 return (1); 235 else 236 return (0); 237 case PGHW_POW_IDLE: 238 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) 239 return (1); 240 else 241 return (0); 242 default: 243 return (0); 244 } 245 } 246 247 /* 248 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 249 * If pghw_type_t is an unsupported hardware type, then return -1 250 */ 251 int 252 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 253 { 254 id_t pgp_a, pgp_b; 255 256 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 257 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 258 259 if (pgp_a == -1 || pgp_b == -1) 260 return (-1); 261 262 return (pgp_a == pgp_b); 263 } 264 265 /* 266 * Return a physical instance identifier for known hardware sharing 267 * relationships 268 */ 269 id_t 270 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 271 { 272 switch (hw) { 273 case PGHW_IPIPE: 274 return (cpuid_get_coreid(cpu)); 275 case PGHW_CACHE: 276 return (cpuid_get_last_lvl_cacheid(cpu)); 277 case PGHW_CHIP: 278 return (cpuid_get_chipid(cpu)); 279 case PGHW_POW_ACTIVE: 280 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); 281 case PGHW_POW_IDLE: 282 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); 283 default: 284 return (-1); 285 } 286 } 287 288 /* 289 * Express preference for optimizing for sharing relationship 290 * hw1 vs hw2 291 */ 292 pghw_type_t 293 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) 294 { 295 int i, rank1, rank2; 296 297 static pghw_type_t hw_hier[] = { 298 PGHW_IPIPE, 299 PGHW_CACHE, 300 PGHW_CHIP, 301 PGHW_POW_IDLE, 302 PGHW_POW_ACTIVE, 303 PGHW_NUM_COMPONENTS 304 }; 305 306 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 307 if (hw_hier[i] == hw1) 308 rank1 = i; 309 if (hw_hier[i] == hw2) 310 rank2 = i; 311 } 312 313 if (rank1 > rank2) 314 return (hw1); 315 else 316 return (hw2); 317 } 318 319 /* 320 * Override the default CMT dispatcher policy for the specified 321 * hardware sharing relationship 322 */ 323 pg_cmt_policy_t 324 pg_plat_cmt_policy(pghw_type_t hw) 325 { 326 /* 327 * For shared caches, also load balance across them to 328 * maximize aggregate cache capacity 329 */ 330 switch (hw) { 331 case PGHW_CACHE: 332 return (CMT_BALANCE|CMT_AFFINITY); 333 default: 334 return (CMT_NO_POLICY); 335 } 336 } 337 338 id_t 339 pg_plat_get_core_id(cpu_t *cpu) 340 { 341 return ((id_t)cpuid_get_coreid(cpu)); 342 } 343 344 void 345 cmp_set_nosteal_interval(void) 346 { 347 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 348 nosteal_nsec = 100000UL; 349 } 350 351 /* 352 * Routine to ensure initial callers to hrtime gets 0 as return 353 */ 354 static hrtime_t 355 dummy_hrtime(void) 356 { 357 return (0); 358 } 359 360 /* ARGSUSED */ 361 static void 362 dummy_scalehrtime(hrtime_t *ticks) 363 {} 364 365 /* 366 * Supports Deep C-State power saving idle loop. 367 */ 368 void 369 cpu_idle_adaptive(void) 370 { 371 (*CPU->cpu_m.mcpu_idle_cpu)(); 372 } 373 374 /* 375 * Function called by CPU idle notification framework to check whether CPU 376 * has been awakened. It will be called with interrupt disabled. 377 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 378 * notification framework. 379 */ 380 /*ARGSUSED*/ 381 static void 382 cpu_idle_check_wakeup(void *arg) 383 { 384 /* 385 * Toggle interrupt flag to detect pending interrupts. 386 * If interrupt happened, do_interrupt() will notify CPU idle 387 * notification framework so no need to call cpu_idle_exit() here. 388 */ 389 sti(); 390 SMT_PAUSE(); 391 cli(); 392 } 393 394 /* 395 * Idle the present CPU until wakened via an interrupt 396 */ 397 void 398 cpu_idle(void) 399 { 400 cpu_t *cpup = CPU; 401 processorid_t cpu_sid = cpup->cpu_seqid; 402 cpupart_t *cp = cpup->cpu_part; 403 int hset_update = 1; 404 405 /* 406 * If this CPU is online, and there's multiple CPUs 407 * in the system, then we should notate our halting 408 * by adding ourselves to the partition's halted CPU 409 * bitmap. This allows other CPUs to find/awaken us when 410 * work becomes available. 411 */ 412 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 413 hset_update = 0; 414 415 /* 416 * Add ourselves to the partition's halted CPUs bitmap 417 * and set our HALTED flag, if necessary. 418 * 419 * When a thread becomes runnable, it is placed on the queue 420 * and then the halted CPU bitmap is checked to determine who 421 * (if anyone) should be awakened. We therefore need to first 422 * add ourselves to the bitmap, and and then check if there 423 * is any work available. The order is important to prevent a race 424 * that can lead to work languishing on a run queue somewhere while 425 * this CPU remains halted. 426 * 427 * Either the producing CPU will see we're halted and will awaken us, 428 * or this CPU will see the work available in disp_anywork(). 429 * 430 * Note that memory barriers after updating the HALTED flag 431 * are not necessary since an atomic operation (updating the bitset) 432 * immediately follows. On x86 the atomic operation acts as a 433 * memory barrier for the update of cpu_disp_flags. 434 */ 435 if (hset_update) { 436 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 437 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 438 } 439 440 /* 441 * Check to make sure there's really nothing to do. 442 * Work destined for this CPU may become available after 443 * this check. We'll be notified through the clearing of our 444 * bit in the halted CPU bitmap, and a poke. 445 */ 446 if (disp_anywork()) { 447 if (hset_update) { 448 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 449 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 450 } 451 return; 452 } 453 454 /* 455 * We're on our way to being halted. 456 * 457 * Disable interrupts now, so that we'll awaken immediately 458 * after halting if someone tries to poke us between now and 459 * the time we actually halt. 460 * 461 * We check for the presence of our bit after disabling interrupts. 462 * If it's cleared, we'll return. If the bit is cleared after 463 * we check then the poke will pop us out of the halted state. 464 * 465 * This means that the ordering of the poke and the clearing 466 * of the bit by cpu_wakeup is important. 467 * cpu_wakeup() must clear, then poke. 468 * cpu_idle() must disable interrupts, then check for the bit. 469 */ 470 cli(); 471 472 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { 473 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 474 sti(); 475 return; 476 } 477 478 /* 479 * The check for anything locally runnable is here for performance 480 * and isn't needed for correctness. disp_nrunnable ought to be 481 * in our cache still, so it's inexpensive to check, and if there 482 * is anything runnable we won't have to wait for the poke. 483 */ 484 if (cpup->cpu_disp->disp_nrunnable != 0) { 485 if (hset_update) { 486 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 487 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 488 } 489 sti(); 490 return; 491 } 492 493 if (cpu_idle_enter(IDLE_STATE_C1, 0, 494 cpu_idle_check_wakeup, NULL) == 0) { 495 mach_cpu_idle(); 496 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 497 } 498 499 /* 500 * We're no longer halted 501 */ 502 if (hset_update) { 503 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 504 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 505 } 506 } 507 508 509 /* 510 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 511 * Otherwise, see if other CPUs in the cpu partition are halted and need to 512 * be woken up so that they can steal the thread we placed on this CPU. 513 * This function is only used on MP systems. 514 */ 515 static void 516 cpu_wakeup(cpu_t *cpu, int bound) 517 { 518 uint_t cpu_found; 519 processorid_t cpu_sid; 520 cpupart_t *cp; 521 522 cp = cpu->cpu_part; 523 cpu_sid = cpu->cpu_seqid; 524 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 525 /* 526 * Clear the halted bit for that CPU since it will be 527 * poked in a moment. 528 */ 529 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 530 /* 531 * We may find the current CPU present in the halted cpuset 532 * if we're in the context of an interrupt that occurred 533 * before we had a chance to clear our bit in cpu_idle(). 534 * Poking ourself is obviously unnecessary, since if 535 * we're here, we're not halted. 536 */ 537 if (cpu != CPU) 538 poke_cpu(cpu->cpu_id); 539 return; 540 } else { 541 /* 542 * This cpu isn't halted, but it's idle or undergoing a 543 * context switch. No need to awaken anyone else. 544 */ 545 if (cpu->cpu_thread == cpu->cpu_idle_thread || 546 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 547 return; 548 } 549 550 /* 551 * No need to wake up other CPUs if this is for a bound thread. 552 */ 553 if (bound) 554 return; 555 556 /* 557 * The CPU specified for wakeup isn't currently halted, so check 558 * to see if there are any other halted CPUs in the partition, 559 * and if there are then awaken one. 560 */ 561 do { 562 cpu_found = bitset_find(&cp->cp_haltset); 563 if (cpu_found == (uint_t)-1) 564 return; 565 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 566 567 if (cpu_found != CPU->cpu_seqid) { 568 poke_cpu(cpu_seq[cpu_found]->cpu_id); 569 } 570 } 571 572 #ifndef __xpv 573 /* 574 * Function called by CPU idle notification framework to check whether CPU 575 * has been awakened. It will be called with interrupt disabled. 576 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 577 * notification framework. 578 */ 579 static void 580 cpu_idle_mwait_check_wakeup(void *arg) 581 { 582 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; 583 584 ASSERT(arg != NULL); 585 if (*mcpu_mwait != MWAIT_HALTED) { 586 /* 587 * CPU has been awakened, notify CPU idle notification system. 588 */ 589 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 590 } else { 591 /* 592 * Toggle interrupt flag to detect pending interrupts. 593 * If interrupt happened, do_interrupt() will notify CPU idle 594 * notification framework so no need to call cpu_idle_exit() 595 * here. 596 */ 597 sti(); 598 SMT_PAUSE(); 599 cli(); 600 } 601 } 602 603 /* 604 * Idle the present CPU until awakened via touching its monitored line 605 */ 606 void 607 cpu_idle_mwait(void) 608 { 609 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 610 cpu_t *cpup = CPU; 611 processorid_t cpu_sid = cpup->cpu_seqid; 612 cpupart_t *cp = cpup->cpu_part; 613 int hset_update = 1; 614 615 /* 616 * Set our mcpu_mwait here, so we can tell if anyone tries to 617 * wake us between now and when we call mwait. No other cpu will 618 * attempt to set our mcpu_mwait until we add ourself to the halted 619 * CPU bitmap. 620 */ 621 *mcpu_mwait = MWAIT_HALTED; 622 623 /* 624 * If this CPU is online, and there's multiple CPUs 625 * in the system, then we should note our halting 626 * by adding ourselves to the partition's halted CPU 627 * bitmap. This allows other CPUs to find/awaken us when 628 * work becomes available. 629 */ 630 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 631 hset_update = 0; 632 633 /* 634 * Add ourselves to the partition's halted CPUs bitmap 635 * and set our HALTED flag, if necessary. 636 * 637 * When a thread becomes runnable, it is placed on the queue 638 * and then the halted CPU bitmap is checked to determine who 639 * (if anyone) should be awakened. We therefore need to first 640 * add ourselves to the bitmap, and and then check if there 641 * is any work available. 642 * 643 * Note that memory barriers after updating the HALTED flag 644 * are not necessary since an atomic operation (updating the bitmap) 645 * immediately follows. On x86 the atomic operation acts as a 646 * memory barrier for the update of cpu_disp_flags. 647 */ 648 if (hset_update) { 649 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 650 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 651 } 652 653 /* 654 * Check to make sure there's really nothing to do. 655 * Work destined for this CPU may become available after 656 * this check. We'll be notified through the clearing of our 657 * bit in the halted CPU bitmap, and a write to our mcpu_mwait. 658 * 659 * disp_anywork() checks disp_nrunnable, so we do not have to later. 660 */ 661 if (disp_anywork()) { 662 if (hset_update) { 663 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 664 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 665 } 666 return; 667 } 668 669 /* 670 * We're on our way to being halted. 671 * To avoid a lost wakeup, arm the monitor before checking if another 672 * cpu wrote to mcpu_mwait to wake us up. 673 */ 674 i86_monitor(mcpu_mwait, 0, 0); 675 if (*mcpu_mwait == MWAIT_HALTED) { 676 if (cpu_idle_enter(IDLE_STATE_C1, 0, 677 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { 678 if (*mcpu_mwait == MWAIT_HALTED) { 679 i86_mwait(0, 0); 680 } 681 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 682 } 683 } 684 685 /* 686 * We're no longer halted 687 */ 688 if (hset_update) { 689 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 690 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 691 } 692 } 693 694 /* 695 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 696 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 697 * need to be woken up so that they can steal the thread we placed on this CPU. 698 * This function is only used on MP systems. 699 */ 700 static void 701 cpu_wakeup_mwait(cpu_t *cp, int bound) 702 { 703 cpupart_t *cpu_part; 704 uint_t cpu_found; 705 processorid_t cpu_sid; 706 707 cpu_part = cp->cpu_part; 708 cpu_sid = cp->cpu_seqid; 709 710 /* 711 * Clear the halted bit for that CPU since it will be woken up 712 * in a moment. 713 */ 714 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { 715 /* 716 * Clear the halted bit for that CPU since it will be 717 * poked in a moment. 718 */ 719 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); 720 /* 721 * We may find the current CPU present in the halted cpuset 722 * if we're in the context of an interrupt that occurred 723 * before we had a chance to clear our bit in cpu_idle(). 724 * Waking ourself is obviously unnecessary, since if 725 * we're here, we're not halted. 726 * 727 * monitor/mwait wakeup via writing to our cache line is 728 * harmless and less expensive than always checking if we 729 * are waking ourself which is an uncommon case. 730 */ 731 MWAIT_WAKEUP(cp); /* write to monitored line */ 732 return; 733 } else { 734 /* 735 * This cpu isn't halted, but it's idle or undergoing a 736 * context switch. No need to awaken anyone else. 737 */ 738 if (cp->cpu_thread == cp->cpu_idle_thread || 739 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 740 return; 741 } 742 743 /* 744 * No need to wake up other CPUs if the thread we just enqueued 745 * is bound. 746 */ 747 if (bound || ncpus == 1) 748 return; 749 750 /* 751 * See if there's any other halted CPUs. If there are, then 752 * select one, and awaken it. 753 * It's possible that after we find a CPU, somebody else 754 * will awaken it before we get the chance. 755 * In that case, look again. 756 */ 757 do { 758 cpu_found = bitset_find(&cpu_part->cp_haltset); 759 if (cpu_found == (uint_t)-1) 760 return; 761 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, 762 cpu_found) < 0); 763 764 /* 765 * Do not check if cpu_found is ourself as monitor/mwait 766 * wakeup is cheap. 767 */ 768 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ 769 } 770 771 #endif 772 773 void (*cpu_pause_handler)(volatile char *) = NULL; 774 775 static int 776 mp_disable_intr(int cpun) 777 { 778 /* 779 * switch to the offline cpu 780 */ 781 affinity_set(cpun); 782 /* 783 * raise ipl to just below cross call 784 */ 785 splx(XC_SYS_PIL - 1); 786 /* 787 * set base spl to prevent the next swtch to idle from 788 * lowering back to ipl 0 789 */ 790 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); 791 set_base_spl(); 792 affinity_clear(); 793 return (DDI_SUCCESS); 794 } 795 796 static void 797 mp_enable_intr(int cpun) 798 { 799 /* 800 * switch to the online cpu 801 */ 802 affinity_set(cpun); 803 /* 804 * clear the interrupt active mask 805 */ 806 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); 807 set_base_spl(); 808 (void) spl0(); 809 affinity_clear(); 810 } 811 812 static void 813 mach_get_platform(int owner) 814 { 815 void **srv_opsp; 816 void **clt_opsp; 817 int i; 818 int total_ops; 819 820 /* fix up psm ops */ 821 srv_opsp = (void **)mach_set[0]; 822 clt_opsp = (void **)mach_set[owner]; 823 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 824 total_ops = sizeof (struct psm_ops_ver01) / 825 sizeof (void (*)(void)); 826 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 827 /* no psm_notify_func */ 828 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 829 sizeof (void (*)(void)); 830 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 831 /* no psm_timer funcs */ 832 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 833 sizeof (void (*)(void)); 834 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 835 /* no psm_preshutdown function */ 836 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 837 sizeof (void (*)(void)); 838 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 839 /* no psm_preshutdown function */ 840 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 841 sizeof (void (*)(void)); 842 else 843 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 844 845 /* 846 * Save the version of the PSM module, in case we need to 847 * behave differently based on version. 848 */ 849 mach_ver[0] = mach_ver[owner]; 850 851 for (i = 0; i < total_ops; i++) 852 if (clt_opsp[i] != NULL) 853 srv_opsp[i] = clt_opsp[i]; 854 } 855 856 static void 857 mach_construct_info() 858 { 859 struct psm_sw *swp; 860 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 861 int conflict_owner = 0; 862 863 if (psmsw->psw_forw == psmsw) 864 panic("No valid PSM modules found"); 865 mutex_enter(&psmsw_lock); 866 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 867 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 868 continue; 869 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 870 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 871 mach_cnt[swp->psw_infop->p_owner]++; 872 } 873 mutex_exit(&psmsw_lock); 874 875 mach_get_platform(PSM_OWN_SYS_DEFAULT); 876 877 /* check to see are there any conflicts */ 878 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 879 conflict_owner = PSM_OWN_EXCLUSIVE; 880 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 881 conflict_owner = PSM_OWN_OVERRIDE; 882 if (conflict_owner) { 883 /* remove all psm modules except uppc */ 884 cmn_err(CE_WARN, 885 "Conflicts detected on the following PSM modules:"); 886 mutex_enter(&psmsw_lock); 887 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 888 if (swp->psw_infop->p_owner == conflict_owner) 889 cmn_err(CE_WARN, "%s ", 890 swp->psw_infop->p_mach_idstring); 891 } 892 mutex_exit(&psmsw_lock); 893 cmn_err(CE_WARN, 894 "Setting the system back to SINGLE processor mode!"); 895 cmn_err(CE_WARN, 896 "Please edit /etc/mach to remove the invalid PSM module."); 897 return; 898 } 899 900 if (mach_set[PSM_OWN_EXCLUSIVE]) 901 mach_get_platform(PSM_OWN_EXCLUSIVE); 902 903 if (mach_set[PSM_OWN_OVERRIDE]) 904 mach_get_platform(PSM_OWN_OVERRIDE); 905 } 906 907 static void 908 mach_init() 909 { 910 struct psm_ops *pops; 911 912 mach_construct_info(); 913 914 pops = mach_set[0]; 915 916 /* register the interrupt and clock initialization rotuines */ 917 picinitf = mach_picinit; 918 clkinitf = mach_clkinit; 919 psm_get_clockirq = pops->psm_get_clockirq; 920 921 /* register the interrupt setup code */ 922 slvltovect = mach_softlvl_to_vect; 923 addspl = pops->psm_addspl; 924 delspl = pops->psm_delspl; 925 926 if (pops->psm_translate_irq) 927 psm_translate_irq = pops->psm_translate_irq; 928 if (pops->psm_intr_ops) 929 psm_intr_ops = pops->psm_intr_ops; 930 931 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 932 /* 933 * Time-of-day functionality now handled in TOD modules. 934 * (Warn about PSM modules that think that we're going to use 935 * their ops vectors.) 936 */ 937 if (pops->psm_tod_get) 938 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 939 (void *)pops->psm_tod_get); 940 941 if (pops->psm_tod_set) 942 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 943 (void *)pops->psm_tod_set); 944 #endif 945 946 if (pops->psm_notify_error) { 947 psm_notify_error = mach_notify_error; 948 notify_error = pops->psm_notify_error; 949 } 950 951 (*pops->psm_softinit)(); 952 953 /* 954 * Initialize the dispatcher's function hooks to enable CPU halting 955 * when idle. Set both the deep-idle and non-deep-idle hooks. 956 * 957 * Assume we can use power saving deep-idle loop cpu_idle_adaptive. 958 * Platform deep-idle driver will reset our idle loop to 959 * non_deep_idle_cpu if power saving deep-idle feature is not available. 960 * 961 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) 962 * or idle_cpu_prefer_mwait is not set. 963 * Allocate monitor/mwait buffer for cpu0. 964 */ 965 #ifndef __xpv 966 non_deep_idle_disp_enq_thread = disp_enq_thread; 967 #endif 968 if (idle_cpu_use_hlt) { 969 idle_cpu = cpu_idle_adaptive; 970 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 971 #ifndef __xpv 972 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) { 973 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); 974 /* 975 * Protect ourself from insane mwait size. 976 */ 977 if (CPU->cpu_m.mcpu_mwait == NULL) { 978 #ifdef DEBUG 979 cmn_err(CE_NOTE, "Using hlt idle. Cannot " 980 "handle cpu 0 mwait size."); 981 #endif 982 idle_cpu_prefer_mwait = 0; 983 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 984 } else { 985 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; 986 } 987 } else { 988 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 989 } 990 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; 991 992 /* 993 * Disable power saving deep idle loop? 994 */ 995 if (idle_cpu_no_deep_c) { 996 idle_cpu = non_deep_idle_cpu; 997 } 998 #endif 999 } 1000 1001 mach_smpinit(); 1002 } 1003 1004 static void 1005 mach_smpinit(void) 1006 { 1007 struct psm_ops *pops; 1008 processorid_t cpu_id; 1009 int cnt; 1010 cpuset_t cpumask; 1011 1012 pops = mach_set[0]; 1013 CPUSET_ZERO(cpumask); 1014 1015 cpu_id = -1; 1016 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1017 for (cnt = 0; cpu_id != -1; cnt++) { 1018 CPUSET_ADD(cpumask, cpu_id); 1019 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1020 } 1021 1022 mp_cpus = cpumask; 1023 1024 /* MP related routines */ 1025 ap_mlsetup = pops->psm_post_cpu_start; 1026 send_dirintf = pops->psm_send_ipi; 1027 1028 /* optional MP related routines */ 1029 if (pops->psm_shutdown) 1030 psm_shutdownf = pops->psm_shutdown; 1031 if (pops->psm_preshutdown) 1032 psm_preshutdownf = pops->psm_preshutdown; 1033 if (pops->psm_notify_func) 1034 psm_notifyf = pops->psm_notify_func; 1035 if (pops->psm_set_idlecpu) 1036 psm_set_idle_cpuf = pops->psm_set_idlecpu; 1037 if (pops->psm_unset_idlecpu) 1038 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 1039 1040 psm_clkinit = pops->psm_clkinit; 1041 1042 if (pops->psm_timer_reprogram) 1043 psm_timer_reprogram = pops->psm_timer_reprogram; 1044 1045 if (pops->psm_timer_enable) 1046 psm_timer_enable = pops->psm_timer_enable; 1047 1048 if (pops->psm_timer_disable) 1049 psm_timer_disable = pops->psm_timer_disable; 1050 1051 if (pops->psm_post_cyclic_setup) 1052 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 1053 1054 if (pops->psm_state) 1055 psm_state = pops->psm_state; 1056 1057 /* 1058 * Set these vectors here so they can be used by Suspend/Resume 1059 * on UP machines. 1060 */ 1061 if (pops->psm_disable_intr) 1062 psm_disable_intr = pops->psm_disable_intr; 1063 if (pops->psm_enable_intr) 1064 psm_enable_intr = pops->psm_enable_intr; 1065 1066 /* check for multiple CPUs */ 1067 if (cnt < 2) 1068 return; 1069 1070 /* check for MP platforms */ 1071 if (pops->psm_cpu_start == NULL) 1072 return; 1073 1074 /* 1075 * Set the dispatcher hook to enable cpu "wake up" 1076 * when a thread becomes runnable. 1077 */ 1078 if (idle_cpu_use_hlt) { 1079 disp_enq_thread = cpu_wakeup; 1080 #ifndef __xpv 1081 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) 1082 disp_enq_thread = cpu_wakeup_mwait; 1083 non_deep_idle_disp_enq_thread = disp_enq_thread; 1084 #endif 1085 } 1086 1087 psm_get_ipivect = pops->psm_get_ipivect; 1088 1089 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", 1090 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 1091 NULL, NULL, NULL, NULL); 1092 1093 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 1094 } 1095 1096 static void 1097 mach_picinit() 1098 { 1099 struct psm_ops *pops; 1100 1101 pops = mach_set[0]; 1102 1103 /* register the interrupt handlers */ 1104 setlvl = pops->psm_intr_enter; 1105 setlvlx = pops->psm_intr_exit; 1106 1107 /* initialize the interrupt hardware */ 1108 (*pops->psm_picinit)(); 1109 1110 /* set interrupt mask for current ipl */ 1111 setspl = pops->psm_setspl; 1112 cli(); 1113 setspl(CPU->cpu_pri); 1114 } 1115 1116 uint_t cpu_freq; /* MHz */ 1117 uint64_t cpu_freq_hz; /* measured (in hertz) */ 1118 1119 #define MEGA_HZ 1000000 1120 1121 #ifdef __xpv 1122 1123 int xpv_cpufreq_workaround = 1; 1124 int xpv_cpufreq_verbose = 0; 1125 1126 #else /* __xpv */ 1127 1128 static uint64_t 1129 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 1130 { 1131 uint64_t cpu_hz; 1132 1133 if ((pit_counter == 0) || (*processor_clks == 0) || 1134 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 1135 return (0); 1136 1137 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 1138 1139 return (cpu_hz); 1140 } 1141 1142 #endif /* __xpv */ 1143 1144 static uint64_t 1145 mach_getcpufreq(void) 1146 { 1147 #if defined(__xpv) 1148 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; 1149 uint64_t cpu_hz; 1150 1151 /* 1152 * During dom0 bringup, it was noted that on at least one older 1153 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul 1154 * value that is quite wrong (the 3.06GHz clock was reported 1155 * as 4.77GHz) 1156 * 1157 * The curious thing is, that if you stop the kernel at entry, 1158 * breakpoint here and inspect the value with kmdb, the value 1159 * is correct - but if you don't stop and simply enable the 1160 * printf statement (below), you can see the bad value printed 1161 * here. Almost as if something kmdb did caused the hypervisor to 1162 * figure it out correctly. And, note that the hypervisor 1163 * eventually -does- figure it out correctly ... if you look at 1164 * the field later in the life of dom0, it is correct. 1165 * 1166 * For now, on dom0, we employ a slightly cheesy workaround of 1167 * using the DOM0_PHYSINFO hypercall. 1168 */ 1169 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { 1170 xen_sysctl_t op0, *op = &op0; 1171 1172 op->cmd = XEN_SYSCTL_physinfo; 1173 op->interface_version = XEN_SYSCTL_INTERFACE_VERSION; 1174 if (HYPERVISOR_sysctl(op) != 0) 1175 panic("physinfo op refused"); 1176 1177 cpu_hz = 1000 * (uint64_t)op->u.physinfo.cpu_khz; 1178 } else { 1179 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; 1180 1181 if (vti->tsc_shift < 0) 1182 cpu_hz <<= -vti->tsc_shift; 1183 else 1184 cpu_hz >>= vti->tsc_shift; 1185 } 1186 1187 if (xpv_cpufreq_verbose) 1188 printf("mach_getcpufreq: system_mul 0x%x, shift %d, " 1189 "cpu_hz %" PRId64 "Hz\n", 1190 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); 1191 1192 return (cpu_hz); 1193 #else /* __xpv */ 1194 uint32_t pit_counter; 1195 uint64_t processor_clks; 1196 1197 if (x86_feature & X86_TSC) { 1198 /* 1199 * We have a TSC. freq_tsc() knows how to measure the number 1200 * of clock cycles sampled against the PIT. 1201 */ 1202 ulong_t flags = clear_int_flag(); 1203 processor_clks = freq_tsc(&pit_counter); 1204 restore_int_flag(flags); 1205 return (mach_calchz(pit_counter, &processor_clks)); 1206 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1207 #if defined(__amd64) 1208 panic("mach_getcpufreq: no TSC!"); 1209 #elif defined(__i386) 1210 /* 1211 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1212 * for which freq_notsc() knows how to measure the number of 1213 * elapsed clock cycles sampled against the PIT 1214 */ 1215 ulong_t flags = clear_int_flag(); 1216 processor_clks = freq_notsc(&pit_counter); 1217 restore_int_flag(flags); 1218 return (mach_calchz(pit_counter, &processor_clks)); 1219 #endif /* __i386 */ 1220 } 1221 1222 /* We do not know how to calculate cpu frequency for this cpu. */ 1223 return (0); 1224 #endif /* __xpv */ 1225 } 1226 1227 /* 1228 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1229 * to this array, instead improve the accuracy of the algorithm that determines 1230 * the clock speed of the processor or extend the implementation to support the 1231 * vendor as appropriate. This is here only to support adjusting the speed on 1232 * older slower processors that mach_fixcpufreq() would not be able to account 1233 * for otherwise. 1234 */ 1235 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1236 1237 /* 1238 * On fast processors the clock frequency that is measured may be off by 1239 * a few MHz from the value printed on the part. This is a combination of 1240 * the factors that for such fast parts being off by this much is within 1241 * the tolerances for manufacture and because of the difficulties in the 1242 * measurement that can lead to small error. This function uses some 1243 * heuristics in order to tweak the value that was measured to match what 1244 * is most likely printed on the part. 1245 * 1246 * Some examples: 1247 * AMD Athlon 1000 mhz measured as 998 mhz 1248 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1249 * Intel Pentium IV 1500 mhz measured as 1495mhz 1250 * 1251 * If in the future this function is no longer sufficient to correct 1252 * for the error in the measurement, then the algorithm used to perform 1253 * the measurement will have to be improved in order to increase accuracy 1254 * rather than adding horrible and questionable kludges here. 1255 * 1256 * This is called after the cyclics subsystem because of the potential 1257 * that the heuristics within may give a worse estimate of the clock 1258 * frequency than the value that was measured. 1259 */ 1260 static void 1261 mach_fixcpufreq(void) 1262 { 1263 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1264 1265 freq = (uint32_t)cpu_freq; 1266 1267 /* 1268 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1269 * measured speed taking into account that the 667 MHz parts were 1270 * the first to round-up. 1271 */ 1272 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1273 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1274 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1275 1276 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1277 mul = (freq + 25) / 50; 1278 near50 = mul * 50; 1279 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1280 1281 /* Find the closer of the two */ 1282 if (delta66 < delta50) { 1283 fixed = near66; 1284 delta = delta66; 1285 } else { 1286 fixed = near50; 1287 delta = delta50; 1288 } 1289 1290 if (fixed > INT_MAX) 1291 return; 1292 1293 /* 1294 * Some older parts have a core clock frequency that is not an 1295 * integral multiple of 50 or 66 MHz. Check if one of the old 1296 * clock frequencies is closer to the measured value than any 1297 * of the integral multiples of 50 an 66, and if so set fixed 1298 * and delta appropriately to represent the closest value. 1299 */ 1300 i = sizeof (x86_cpu_freq) / sizeof (int); 1301 while (i > 0) { 1302 i--; 1303 1304 if (x86_cpu_freq[i] <= freq) { 1305 mul = freq - x86_cpu_freq[i]; 1306 1307 if (mul < delta) { 1308 fixed = x86_cpu_freq[i]; 1309 delta = mul; 1310 } 1311 1312 break; 1313 } 1314 1315 mul = x86_cpu_freq[i] - freq; 1316 1317 if (mul < delta) { 1318 fixed = x86_cpu_freq[i]; 1319 delta = mul; 1320 } 1321 } 1322 1323 /* 1324 * Set a reasonable maximum for how much to correct the measured 1325 * result by. This check is here to prevent the adjustment made 1326 * by this function from being more harm than good. It is entirely 1327 * possible that in the future parts will be made that are not 1328 * integral multiples of 66 or 50 in clock frequency or that 1329 * someone may overclock a part to some odd frequency. If the 1330 * measured value is farther from the corrected value than 1331 * allowed, then assume the corrected value is in error and use 1332 * the measured value. 1333 */ 1334 if (6 < delta) 1335 return; 1336 1337 cpu_freq = (int)fixed; 1338 } 1339 1340 1341 static int 1342 machhztomhz(uint64_t cpu_freq_hz) 1343 { 1344 uint64_t cpu_mhz; 1345 1346 /* Round to nearest MHZ */ 1347 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1348 1349 if (cpu_mhz > INT_MAX) 1350 return (0); 1351 1352 return ((int)cpu_mhz); 1353 1354 } 1355 1356 1357 static int 1358 mach_clkinit(int preferred_mode, int *set_mode) 1359 { 1360 struct psm_ops *pops; 1361 int resolution; 1362 1363 pops = mach_set[0]; 1364 1365 cpu_freq_hz = mach_getcpufreq(); 1366 1367 cpu_freq = machhztomhz(cpu_freq_hz); 1368 1369 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 1370 tsc_gethrtime_enable = 0; 1371 1372 #ifndef __xpv 1373 if (tsc_gethrtime_enable) { 1374 tsc_hrtimeinit(cpu_freq_hz); 1375 } else 1376 #endif 1377 { 1378 if (pops->psm_hrtimeinit) 1379 (*pops->psm_hrtimeinit)(); 1380 gethrtimef = pops->psm_gethrtime; 1381 gethrtimeunscaledf = gethrtimef; 1382 /* scalehrtimef will remain dummy */ 1383 } 1384 1385 mach_fixcpufreq(); 1386 1387 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1388 if (preferred_mode == TIMER_ONESHOT) { 1389 1390 resolution = (*pops->psm_clkinit)(0); 1391 if (resolution != 0) { 1392 *set_mode = TIMER_ONESHOT; 1393 return (resolution); 1394 } 1395 } 1396 1397 /* 1398 * either periodic mode was requested or could not set to 1399 * one-shot mode 1400 */ 1401 resolution = (*pops->psm_clkinit)(hz); 1402 /* 1403 * psm should be able to do periodic, so we do not check 1404 * for return value of psm_clkinit here. 1405 */ 1406 *set_mode = TIMER_PERIODIC; 1407 return (resolution); 1408 } else { 1409 /* 1410 * PSMI interface prior to PSMI_3 does not define a return 1411 * value for psm_clkinit, so the return value is ignored. 1412 */ 1413 (void) (*pops->psm_clkinit)(hz); 1414 *set_mode = TIMER_PERIODIC; 1415 return (nsec_per_tick); 1416 } 1417 } 1418 1419 1420 /*ARGSUSED*/ 1421 static int 1422 mach_softlvl_to_vect(int ipl) 1423 { 1424 setsoftint = av_set_softint_pending; 1425 kdisetsoftint = kdi_av_set_softint_pending; 1426 1427 return (PSM_SV_SOFTWARE); 1428 } 1429 1430 #ifdef DEBUG 1431 /* 1432 * This is here to allow us to simulate cpus that refuse to start. 1433 */ 1434 cpuset_t cpufailset; 1435 #endif 1436 1437 int 1438 mach_cpu_start(struct cpu *cp, void *ctx) 1439 { 1440 struct psm_ops *pops = mach_set[0]; 1441 processorid_t id = cp->cpu_id; 1442 1443 #ifdef DEBUG 1444 if (CPU_IN_SET(cpufailset, id)) 1445 return (0); 1446 #endif 1447 return ((*pops->psm_cpu_start)(id, ctx)); 1448 } 1449 1450 int 1451 mach_cpuid_start(processorid_t id, void *ctx) 1452 { 1453 struct psm_ops *pops = mach_set[0]; 1454 1455 #ifdef DEBUG 1456 if (CPU_IN_SET(cpufailset, id)) 1457 return (0); 1458 #endif 1459 return ((*pops->psm_cpu_start)(id, ctx)); 1460 } 1461 1462 /*ARGSUSED*/ 1463 static int 1464 mach_translate_irq(dev_info_t *dip, int irqno) 1465 { 1466 return (irqno); /* default to NO translation */ 1467 } 1468 1469 static void 1470 mach_notify_error(int level, char *errmsg) 1471 { 1472 /* 1473 * SL_FATAL is pass in once panicstr is set, deliver it 1474 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1475 * codes for the psmi handler 1476 */ 1477 if (level & SL_FATAL) 1478 (*notify_error)(CE_PANIC, errmsg); 1479 else if (level & SL_WARN) 1480 (*notify_error)(CE_WARN, errmsg); 1481 else if (level & SL_NOTE) 1482 (*notify_error)(CE_NOTE, errmsg); 1483 else if (level & SL_CONSOLE) 1484 (*notify_error)(CE_CONT, errmsg); 1485 } 1486 1487 /* 1488 * It provides the default basic intr_ops interface for the new DDI 1489 * interrupt framework if the PSM doesn't have one. 1490 * 1491 * Input: 1492 * dip - pointer to the dev_info structure of the requested device 1493 * hdlp - pointer to the internal interrupt handle structure for the 1494 * requested interrupt 1495 * intr_op - opcode for this call 1496 * result - pointer to the integer that will hold the result to be 1497 * passed back if return value is PSM_SUCCESS 1498 * 1499 * Output: 1500 * return value is either PSM_SUCCESS or PSM_FAILURE 1501 */ 1502 static int 1503 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1504 psm_intr_op_t intr_op, int *result) 1505 { 1506 struct intrspec *ispec; 1507 1508 switch (intr_op) { 1509 case PSM_INTR_OP_CHECK_MSI: 1510 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1511 DDI_INTR_TYPE_MSIX); 1512 break; 1513 case PSM_INTR_OP_ALLOC_VECTORS: 1514 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1515 *result = 1; 1516 else 1517 *result = 0; 1518 break; 1519 case PSM_INTR_OP_FREE_VECTORS: 1520 break; 1521 case PSM_INTR_OP_NAVAIL_VECTORS: 1522 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1523 *result = 1; 1524 else 1525 *result = 0; 1526 break; 1527 case PSM_INTR_OP_XLATE_VECTOR: 1528 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1529 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1530 break; 1531 case PSM_INTR_OP_GET_CAP: 1532 *result = 0; 1533 break; 1534 case PSM_INTR_OP_GET_PENDING: 1535 case PSM_INTR_OP_CLEAR_MASK: 1536 case PSM_INTR_OP_SET_MASK: 1537 case PSM_INTR_OP_GET_SHARED: 1538 case PSM_INTR_OP_SET_PRI: 1539 case PSM_INTR_OP_SET_CAP: 1540 case PSM_INTR_OP_SET_CPU: 1541 case PSM_INTR_OP_GET_INTR: 1542 default: 1543 return (PSM_FAILURE); 1544 } 1545 return (PSM_SUCCESS); 1546 } 1547 /* 1548 * Return 1 if CMT load balancing policies should be 1549 * implemented across instances of the specified hardware 1550 * sharing relationship. 1551 */ 1552 int 1553 pg_cmt_load_bal_hw(pghw_type_t hw) 1554 { 1555 if (hw == PGHW_IPIPE || 1556 hw == PGHW_FPU || 1557 hw == PGHW_CHIP) 1558 return (1); 1559 else 1560 return (0); 1561 } 1562 /* 1563 * Return 1 if thread affinity polices should be implemented 1564 * for instances of the specifed hardware sharing relationship. 1565 */ 1566 int 1567 pg_cmt_affinity_hw(pghw_type_t hw) 1568 { 1569 if (hw == PGHW_CACHE) 1570 return (1); 1571 else 1572 return (0); 1573 } 1574