1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright (c) 2009, Intel Corporation. 27 * All rights reserved. 28 */ 29 30 #define PSMI_1_6 31 #include <sys/smp_impldefs.h> 32 #include <sys/psm.h> 33 #include <sys/psm_modctl.h> 34 #include <sys/pit.h> 35 #include <sys/cmn_err.h> 36 #include <sys/strlog.h> 37 #include <sys/clock.h> 38 #include <sys/debug.h> 39 #include <sys/rtc.h> 40 #include <sys/x86_archext.h> 41 #include <sys/cpupart.h> 42 #include <sys/cpuvar.h> 43 #include <sys/cpu_event.h> 44 #include <sys/cmt.h> 45 #include <sys/cpu.h> 46 #include <sys/disp.h> 47 #include <sys/archsystm.h> 48 #include <sys/machsystm.h> 49 #include <sys/sysmacros.h> 50 #include <sys/memlist.h> 51 #include <sys/param.h> 52 #include <sys/promif.h> 53 #include <sys/cpu_pm.h> 54 #if defined(__xpv) 55 #include <sys/hypervisor.h> 56 #endif 57 #include <sys/mach_intr.h> 58 #include <vm/hat_i86.h> 59 #include <sys/kdi_machimpl.h> 60 #include <sys/sdt.h> 61 #include <sys/hpet.h> 62 #include <sys/sunddi.h> 63 #include <sys/sunndi.h> 64 65 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 66 67 /* 68 * Local function prototypes 69 */ 70 static int mp_disable_intr(processorid_t cpun); 71 static void mp_enable_intr(processorid_t cpun); 72 static void mach_init(); 73 static void mach_picinit(); 74 static int machhztomhz(uint64_t cpu_freq_hz); 75 static uint64_t mach_getcpufreq(void); 76 static void mach_fixcpufreq(void); 77 static int mach_clkinit(int, int *); 78 static void mach_smpinit(void); 79 static int mach_softlvl_to_vect(int ipl); 80 static void mach_get_platform(int owner); 81 static void mach_construct_info(); 82 static int mach_translate_irq(dev_info_t *dip, int irqno); 83 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 84 psm_intr_op_t, int *); 85 static void mach_notify_error(int level, char *errmsg); 86 static hrtime_t dummy_hrtime(void); 87 static void dummy_scalehrtime(hrtime_t *); 88 static uint64_t dummy_unscalehrtime(hrtime_t); 89 void cpu_idle(void); 90 static void cpu_wakeup(cpu_t *, int); 91 #ifndef __xpv 92 void cpu_idle_mwait(void); 93 static void cpu_wakeup_mwait(cpu_t *, int); 94 #endif 95 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp); 96 97 /* 98 * External reference functions 99 */ 100 extern void return_instr(); 101 extern uint64_t freq_tsc(uint32_t *); 102 #if defined(__i386) 103 extern uint64_t freq_notsc(uint32_t *); 104 #endif 105 extern void pc_gethrestime(timestruc_t *); 106 extern int cpuid_get_coreid(cpu_t *); 107 extern int cpuid_get_chipid(cpu_t *); 108 109 /* 110 * PSM functions initialization 111 */ 112 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 113 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 114 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 115 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 116 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 117 void (*psminitf)() = mach_init; 118 void (*picinitf)() = return_instr; 119 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 120 int (*ap_mlsetup)() = (int (*)(void))return_instr; 121 void (*send_dirintf)() = return_instr; 122 void (*setspl)(int) = (void (*)(int))return_instr; 123 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 124 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 125 void (*kdisetsoftint)(int, struct av_softinfo *)= 126 (void (*)(int, struct av_softinfo *))return_instr; 127 void (*setsoftint)(int, struct av_softinfo *)= 128 (void (*)(int, struct av_softinfo *))return_instr; 129 int (*slvltovect)(int) = (int (*)(int))return_instr; 130 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 131 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 132 int (*psm_disable_intr)(int) = mp_disable_intr; 133 void (*psm_enable_intr)(int) = mp_enable_intr; 134 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 135 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 136 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 137 uint64_t (*unscalehrtimef)(hrtime_t) = dummy_unscalehrtime; 138 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 139 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 140 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 141 int (*psm_get_clockirq)(int) = NULL; 142 int (*psm_get_ipivect)(int, int) = NULL; 143 144 int (*psm_clkinit)(int) = NULL; 145 void (*psm_timer_reprogram)(hrtime_t) = NULL; 146 void (*psm_timer_enable)(void) = NULL; 147 void (*psm_timer_disable)(void) = NULL; 148 void (*psm_post_cyclic_setup)(void *arg) = NULL; 149 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 150 int *) = mach_intr_ops; 151 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) 152 return_instr; 153 154 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 155 void (*hrtime_tick)(void) = return_instr; 156 157 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo; 158 159 /* 160 * True if the generic TSC code is our source of hrtime, rather than whatever 161 * the PSM can provide. 162 */ 163 #ifdef __xpv 164 int tsc_gethrtime_enable = 0; 165 #else 166 int tsc_gethrtime_enable = 1; 167 #endif 168 int tsc_gethrtime_initted = 0; 169 170 /* 171 * True if the hrtime implementation is "hires"; namely, better than microdata. 172 */ 173 int gethrtime_hires = 0; 174 175 /* 176 * Local Static Data 177 */ 178 static struct psm_ops mach_ops; 179 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 180 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 181 182 /* 183 * virtualization support for psm 184 */ 185 void *psm_vt_ops = NULL; 186 /* 187 * If non-zero, idle cpus will become "halted" when there's 188 * no work to do. 189 */ 190 int idle_cpu_use_hlt = 1; 191 192 #ifndef __xpv 193 /* 194 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 195 */ 196 int idle_cpu_prefer_mwait = 1; 197 /* 198 * Set to 0 to avoid MONITOR+CLFLUSH assertion. 199 */ 200 int idle_cpu_assert_cflush_monitor = 1; 201 202 /* 203 * If non-zero, idle cpus will not use power saving Deep C-States idle loop. 204 */ 205 int idle_cpu_no_deep_c = 0; 206 /* 207 * Non-power saving idle loop and wakeup pointers. 208 * Allows user to toggle Deep Idle power saving feature on/off. 209 */ 210 void (*non_deep_idle_cpu)() = cpu_idle; 211 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); 212 213 /* 214 * Object for the kernel to access the HPET. 215 */ 216 hpet_t hpet; 217 218 #endif /* ifndef __xpv */ 219 220 /*ARGSUSED*/ 221 int 222 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 223 { 224 switch (hw) { 225 case PGHW_IPIPE: 226 if (x86_feature & (X86_HTT)) { 227 /* 228 * Hyper-threading is SMT 229 */ 230 return (1); 231 } else { 232 return (0); 233 } 234 case PGHW_PROCNODE: 235 if (cpuid_get_procnodes_per_pkg(cp) > 1) 236 return (1); 237 else 238 return (0); 239 case PGHW_CHIP: 240 if (x86_feature & (X86_CMP|X86_HTT)) 241 return (1); 242 else 243 return (0); 244 case PGHW_CACHE: 245 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 246 return (1); 247 else 248 return (0); 249 case PGHW_POW_ACTIVE: 250 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) 251 return (1); 252 else 253 return (0); 254 case PGHW_POW_IDLE: 255 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) 256 return (1); 257 else 258 return (0); 259 default: 260 return (0); 261 } 262 } 263 264 /* 265 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 266 * If pghw_type_t is an unsupported hardware type, then return -1 267 */ 268 int 269 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 270 { 271 id_t pgp_a, pgp_b; 272 273 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 274 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 275 276 if (pgp_a == -1 || pgp_b == -1) 277 return (-1); 278 279 return (pgp_a == pgp_b); 280 } 281 282 /* 283 * Return a physical instance identifier for known hardware sharing 284 * relationships 285 */ 286 id_t 287 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 288 { 289 switch (hw) { 290 case PGHW_IPIPE: 291 return (cpuid_get_coreid(cpu)); 292 case PGHW_CACHE: 293 return (cpuid_get_last_lvl_cacheid(cpu)); 294 case PGHW_PROCNODE: 295 return (cpuid_get_procnodeid(cpu)); 296 case PGHW_CHIP: 297 return (cpuid_get_chipid(cpu)); 298 case PGHW_POW_ACTIVE: 299 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); 300 case PGHW_POW_IDLE: 301 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); 302 default: 303 return (-1); 304 } 305 } 306 307 /* 308 * Express preference for optimizing for sharing relationship 309 * hw1 vs hw2 310 */ 311 pghw_type_t 312 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) 313 { 314 int i, rank1, rank2; 315 316 static pghw_type_t hw_hier[] = { 317 PGHW_IPIPE, 318 PGHW_CACHE, 319 PGHW_PROCNODE, 320 PGHW_CHIP, 321 PGHW_POW_IDLE, 322 PGHW_POW_ACTIVE, 323 PGHW_NUM_COMPONENTS 324 }; 325 326 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 327 if (hw_hier[i] == hw1) 328 rank1 = i; 329 if (hw_hier[i] == hw2) 330 rank2 = i; 331 } 332 333 if (rank1 > rank2) 334 return (hw1); 335 else 336 return (hw2); 337 } 338 339 /* 340 * Override the default CMT dispatcher policy for the specified 341 * hardware sharing relationship 342 */ 343 pg_cmt_policy_t 344 pg_plat_cmt_policy(pghw_type_t hw) 345 { 346 /* 347 * For shared caches, also load balance across them to 348 * maximize aggregate cache capacity 349 */ 350 switch (hw) { 351 case PGHW_CACHE: 352 return (CMT_BALANCE|CMT_AFFINITY); 353 default: 354 return (CMT_NO_POLICY); 355 } 356 } 357 358 id_t 359 pg_plat_get_core_id(cpu_t *cpu) 360 { 361 return ((id_t)cpuid_get_coreid(cpu)); 362 } 363 364 void 365 cmp_set_nosteal_interval(void) 366 { 367 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 368 nosteal_nsec = 100000UL; 369 } 370 371 /* 372 * Routine to ensure initial callers to hrtime gets 0 as return 373 */ 374 static hrtime_t 375 dummy_hrtime(void) 376 { 377 return (0); 378 } 379 380 /* ARGSUSED */ 381 static void 382 dummy_scalehrtime(hrtime_t *ticks) 383 {} 384 385 static uint64_t 386 dummy_unscalehrtime(hrtime_t nsecs) 387 { 388 return ((uint64_t)nsecs); 389 } 390 391 /* 392 * Supports Deep C-State power saving idle loop. 393 */ 394 void 395 cpu_idle_adaptive(void) 396 { 397 (*CPU->cpu_m.mcpu_idle_cpu)(); 398 } 399 400 /* 401 * Function called by CPU idle notification framework to check whether CPU 402 * has been awakened. It will be called with interrupt disabled. 403 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 404 * notification framework. 405 */ 406 /*ARGSUSED*/ 407 static void 408 cpu_idle_check_wakeup(void *arg) 409 { 410 /* 411 * Toggle interrupt flag to detect pending interrupts. 412 * If interrupt happened, do_interrupt() will notify CPU idle 413 * notification framework so no need to call cpu_idle_exit() here. 414 */ 415 sti(); 416 SMT_PAUSE(); 417 cli(); 418 } 419 420 /* 421 * Idle the present CPU until wakened via an interrupt 422 */ 423 void 424 cpu_idle(void) 425 { 426 cpu_t *cpup = CPU; 427 processorid_t cpu_sid = cpup->cpu_seqid; 428 cpupart_t *cp = cpup->cpu_part; 429 int hset_update = 1; 430 431 /* 432 * If this CPU is online, and there's multiple CPUs 433 * in the system, then we should notate our halting 434 * by adding ourselves to the partition's halted CPU 435 * bitmap. This allows other CPUs to find/awaken us when 436 * work becomes available. 437 */ 438 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 439 hset_update = 0; 440 441 /* 442 * Add ourselves to the partition's halted CPUs bitmap 443 * and set our HALTED flag, if necessary. 444 * 445 * When a thread becomes runnable, it is placed on the queue 446 * and then the halted CPU bitmap is checked to determine who 447 * (if anyone) should be awakened. We therefore need to first 448 * add ourselves to the bitmap, and and then check if there 449 * is any work available. The order is important to prevent a race 450 * that can lead to work languishing on a run queue somewhere while 451 * this CPU remains halted. 452 * 453 * Either the producing CPU will see we're halted and will awaken us, 454 * or this CPU will see the work available in disp_anywork(). 455 * 456 * Note that memory barriers after updating the HALTED flag 457 * are not necessary since an atomic operation (updating the bitset) 458 * immediately follows. On x86 the atomic operation acts as a 459 * memory barrier for the update of cpu_disp_flags. 460 */ 461 if (hset_update) { 462 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 463 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 464 } 465 466 /* 467 * Check to make sure there's really nothing to do. 468 * Work destined for this CPU may become available after 469 * this check. We'll be notified through the clearing of our 470 * bit in the halted CPU bitmap, and a poke. 471 */ 472 if (disp_anywork()) { 473 if (hset_update) { 474 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 475 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 476 } 477 return; 478 } 479 480 /* 481 * We're on our way to being halted. 482 * 483 * Disable interrupts now, so that we'll awaken immediately 484 * after halting if someone tries to poke us between now and 485 * the time we actually halt. 486 * 487 * We check for the presence of our bit after disabling interrupts. 488 * If it's cleared, we'll return. If the bit is cleared after 489 * we check then the poke will pop us out of the halted state. 490 * 491 * This means that the ordering of the poke and the clearing 492 * of the bit by cpu_wakeup is important. 493 * cpu_wakeup() must clear, then poke. 494 * cpu_idle() must disable interrupts, then check for the bit. 495 */ 496 cli(); 497 498 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { 499 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 500 sti(); 501 return; 502 } 503 504 /* 505 * The check for anything locally runnable is here for performance 506 * and isn't needed for correctness. disp_nrunnable ought to be 507 * in our cache still, so it's inexpensive to check, and if there 508 * is anything runnable we won't have to wait for the poke. 509 */ 510 if (cpup->cpu_disp->disp_nrunnable != 0) { 511 if (hset_update) { 512 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 513 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 514 } 515 sti(); 516 return; 517 } 518 519 if (cpu_idle_enter(IDLE_STATE_C1, 0, 520 cpu_idle_check_wakeup, NULL) == 0) { 521 mach_cpu_idle(); 522 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 523 } 524 525 /* 526 * We're no longer halted 527 */ 528 if (hset_update) { 529 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 530 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 531 } 532 } 533 534 535 /* 536 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 537 * Otherwise, see if other CPUs in the cpu partition are halted and need to 538 * be woken up so that they can steal the thread we placed on this CPU. 539 * This function is only used on MP systems. 540 */ 541 static void 542 cpu_wakeup(cpu_t *cpu, int bound) 543 { 544 uint_t cpu_found; 545 processorid_t cpu_sid; 546 cpupart_t *cp; 547 548 cp = cpu->cpu_part; 549 cpu_sid = cpu->cpu_seqid; 550 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 551 /* 552 * Clear the halted bit for that CPU since it will be 553 * poked in a moment. 554 */ 555 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 556 /* 557 * We may find the current CPU present in the halted cpuset 558 * if we're in the context of an interrupt that occurred 559 * before we had a chance to clear our bit in cpu_idle(). 560 * Poking ourself is obviously unnecessary, since if 561 * we're here, we're not halted. 562 */ 563 if (cpu != CPU) 564 poke_cpu(cpu->cpu_id); 565 return; 566 } else { 567 /* 568 * This cpu isn't halted, but it's idle or undergoing a 569 * context switch. No need to awaken anyone else. 570 */ 571 if (cpu->cpu_thread == cpu->cpu_idle_thread || 572 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 573 return; 574 } 575 576 /* 577 * No need to wake up other CPUs if this is for a bound thread. 578 */ 579 if (bound) 580 return; 581 582 /* 583 * The CPU specified for wakeup isn't currently halted, so check 584 * to see if there are any other halted CPUs in the partition, 585 * and if there are then awaken one. 586 */ 587 do { 588 cpu_found = bitset_find(&cp->cp_haltset); 589 if (cpu_found == (uint_t)-1) 590 return; 591 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 592 593 if (cpu_found != CPU->cpu_seqid) { 594 poke_cpu(cpu_seq[cpu_found]->cpu_id); 595 } 596 } 597 598 #ifndef __xpv 599 /* 600 * Function called by CPU idle notification framework to check whether CPU 601 * has been awakened. It will be called with interrupt disabled. 602 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 603 * notification framework. 604 */ 605 static void 606 cpu_idle_mwait_check_wakeup(void *arg) 607 { 608 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; 609 610 ASSERT(arg != NULL); 611 if (*mcpu_mwait != MWAIT_HALTED) { 612 /* 613 * CPU has been awakened, notify CPU idle notification system. 614 */ 615 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 616 } else { 617 /* 618 * Toggle interrupt flag to detect pending interrupts. 619 * If interrupt happened, do_interrupt() will notify CPU idle 620 * notification framework so no need to call cpu_idle_exit() 621 * here. 622 */ 623 sti(); 624 SMT_PAUSE(); 625 cli(); 626 } 627 } 628 629 /* 630 * Idle the present CPU until awakened via touching its monitored line 631 */ 632 void 633 cpu_idle_mwait(void) 634 { 635 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 636 cpu_t *cpup = CPU; 637 processorid_t cpu_sid = cpup->cpu_seqid; 638 cpupart_t *cp = cpup->cpu_part; 639 int hset_update = 1; 640 641 /* 642 * Set our mcpu_mwait here, so we can tell if anyone tries to 643 * wake us between now and when we call mwait. No other cpu will 644 * attempt to set our mcpu_mwait until we add ourself to the halted 645 * CPU bitmap. 646 */ 647 *mcpu_mwait = MWAIT_HALTED; 648 649 /* 650 * If this CPU is online, and there's multiple CPUs 651 * in the system, then we should note our halting 652 * by adding ourselves to the partition's halted CPU 653 * bitmap. This allows other CPUs to find/awaken us when 654 * work becomes available. 655 */ 656 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 657 hset_update = 0; 658 659 /* 660 * Add ourselves to the partition's halted CPUs bitmap 661 * and set our HALTED flag, if necessary. 662 * 663 * When a thread becomes runnable, it is placed on the queue 664 * and then the halted CPU bitmap is checked to determine who 665 * (if anyone) should be awakened. We therefore need to first 666 * add ourselves to the bitmap, and and then check if there 667 * is any work available. 668 * 669 * Note that memory barriers after updating the HALTED flag 670 * are not necessary since an atomic operation (updating the bitmap) 671 * immediately follows. On x86 the atomic operation acts as a 672 * memory barrier for the update of cpu_disp_flags. 673 */ 674 if (hset_update) { 675 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 676 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 677 } 678 679 /* 680 * Check to make sure there's really nothing to do. 681 * Work destined for this CPU may become available after 682 * this check. We'll be notified through the clearing of our 683 * bit in the halted CPU bitmap, and a write to our mcpu_mwait. 684 * 685 * disp_anywork() checks disp_nrunnable, so we do not have to later. 686 */ 687 if (disp_anywork()) { 688 if (hset_update) { 689 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 690 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 691 } 692 return; 693 } 694 695 /* 696 * We're on our way to being halted. 697 * To avoid a lost wakeup, arm the monitor before checking if another 698 * cpu wrote to mcpu_mwait to wake us up. 699 */ 700 i86_monitor(mcpu_mwait, 0, 0); 701 if (*mcpu_mwait == MWAIT_HALTED) { 702 if (cpu_idle_enter(IDLE_STATE_C1, 0, 703 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { 704 if (*mcpu_mwait == MWAIT_HALTED) { 705 i86_mwait(0, 0); 706 } 707 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 708 } 709 } 710 711 /* 712 * We're no longer halted 713 */ 714 if (hset_update) { 715 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 716 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 717 } 718 } 719 720 /* 721 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 722 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 723 * need to be woken up so that they can steal the thread we placed on this CPU. 724 * This function is only used on MP systems. 725 */ 726 static void 727 cpu_wakeup_mwait(cpu_t *cp, int bound) 728 { 729 cpupart_t *cpu_part; 730 uint_t cpu_found; 731 processorid_t cpu_sid; 732 733 cpu_part = cp->cpu_part; 734 cpu_sid = cp->cpu_seqid; 735 736 /* 737 * Clear the halted bit for that CPU since it will be woken up 738 * in a moment. 739 */ 740 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { 741 /* 742 * Clear the halted bit for that CPU since it will be 743 * poked in a moment. 744 */ 745 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); 746 /* 747 * We may find the current CPU present in the halted cpuset 748 * if we're in the context of an interrupt that occurred 749 * before we had a chance to clear our bit in cpu_idle(). 750 * Waking ourself is obviously unnecessary, since if 751 * we're here, we're not halted. 752 * 753 * monitor/mwait wakeup via writing to our cache line is 754 * harmless and less expensive than always checking if we 755 * are waking ourself which is an uncommon case. 756 */ 757 MWAIT_WAKEUP(cp); /* write to monitored line */ 758 return; 759 } else { 760 /* 761 * This cpu isn't halted, but it's idle or undergoing a 762 * context switch. No need to awaken anyone else. 763 */ 764 if (cp->cpu_thread == cp->cpu_idle_thread || 765 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 766 return; 767 } 768 769 /* 770 * No need to wake up other CPUs if the thread we just enqueued 771 * is bound. 772 */ 773 if (bound || ncpus == 1) 774 return; 775 776 /* 777 * See if there's any other halted CPUs. If there are, then 778 * select one, and awaken it. 779 * It's possible that after we find a CPU, somebody else 780 * will awaken it before we get the chance. 781 * In that case, look again. 782 */ 783 do { 784 cpu_found = bitset_find(&cpu_part->cp_haltset); 785 if (cpu_found == (uint_t)-1) 786 return; 787 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, 788 cpu_found) < 0); 789 790 /* 791 * Do not check if cpu_found is ourself as monitor/mwait 792 * wakeup is cheap. 793 */ 794 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ 795 } 796 797 #endif 798 799 void (*cpu_pause_handler)(volatile char *) = NULL; 800 801 static int 802 mp_disable_intr(int cpun) 803 { 804 /* 805 * switch to the offline cpu 806 */ 807 affinity_set(cpun); 808 /* 809 * raise ipl to just below cross call 810 */ 811 splx(XC_SYS_PIL - 1); 812 /* 813 * set base spl to prevent the next swtch to idle from 814 * lowering back to ipl 0 815 */ 816 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); 817 set_base_spl(); 818 affinity_clear(); 819 return (DDI_SUCCESS); 820 } 821 822 static void 823 mp_enable_intr(int cpun) 824 { 825 /* 826 * switch to the online cpu 827 */ 828 affinity_set(cpun); 829 /* 830 * clear the interrupt active mask 831 */ 832 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); 833 set_base_spl(); 834 (void) spl0(); 835 affinity_clear(); 836 } 837 838 static void 839 mach_get_platform(int owner) 840 { 841 void **srv_opsp; 842 void **clt_opsp; 843 int i; 844 int total_ops; 845 846 /* fix up psm ops */ 847 srv_opsp = (void **)mach_set[0]; 848 clt_opsp = (void **)mach_set[owner]; 849 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 850 total_ops = sizeof (struct psm_ops_ver01) / 851 sizeof (void (*)(void)); 852 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 853 /* no psm_notify_func */ 854 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 855 sizeof (void (*)(void)); 856 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 857 /* no psm_timer funcs */ 858 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 859 sizeof (void (*)(void)); 860 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 861 /* no psm_preshutdown function */ 862 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 863 sizeof (void (*)(void)); 864 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 865 /* no psm_preshutdown function */ 866 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 867 sizeof (void (*)(void)); 868 else 869 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 870 871 /* 872 * Save the version of the PSM module, in case we need to 873 * behave differently based on version. 874 */ 875 mach_ver[0] = mach_ver[owner]; 876 877 for (i = 0; i < total_ops; i++) 878 if (clt_opsp[i] != NULL) 879 srv_opsp[i] = clt_opsp[i]; 880 } 881 882 static void 883 mach_construct_info() 884 { 885 struct psm_sw *swp; 886 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 887 int conflict_owner = 0; 888 889 if (psmsw->psw_forw == psmsw) 890 panic("No valid PSM modules found"); 891 mutex_enter(&psmsw_lock); 892 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 893 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 894 continue; 895 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 896 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 897 mach_cnt[swp->psw_infop->p_owner]++; 898 } 899 mutex_exit(&psmsw_lock); 900 901 mach_get_platform(PSM_OWN_SYS_DEFAULT); 902 903 /* check to see are there any conflicts */ 904 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 905 conflict_owner = PSM_OWN_EXCLUSIVE; 906 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 907 conflict_owner = PSM_OWN_OVERRIDE; 908 if (conflict_owner) { 909 /* remove all psm modules except uppc */ 910 cmn_err(CE_WARN, 911 "Conflicts detected on the following PSM modules:"); 912 mutex_enter(&psmsw_lock); 913 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 914 if (swp->psw_infop->p_owner == conflict_owner) 915 cmn_err(CE_WARN, "%s ", 916 swp->psw_infop->p_mach_idstring); 917 } 918 mutex_exit(&psmsw_lock); 919 cmn_err(CE_WARN, 920 "Setting the system back to SINGLE processor mode!"); 921 cmn_err(CE_WARN, 922 "Please edit /etc/mach to remove the invalid PSM module."); 923 return; 924 } 925 926 if (mach_set[PSM_OWN_EXCLUSIVE]) 927 mach_get_platform(PSM_OWN_EXCLUSIVE); 928 929 if (mach_set[PSM_OWN_OVERRIDE]) 930 mach_get_platform(PSM_OWN_OVERRIDE); 931 } 932 933 static void 934 mach_init() 935 { 936 struct psm_ops *pops; 937 938 mach_construct_info(); 939 940 pops = mach_set[0]; 941 942 /* register the interrupt and clock initialization rotuines */ 943 picinitf = mach_picinit; 944 clkinitf = mach_clkinit; 945 psm_get_clockirq = pops->psm_get_clockirq; 946 947 /* register the interrupt setup code */ 948 slvltovect = mach_softlvl_to_vect; 949 addspl = pops->psm_addspl; 950 delspl = pops->psm_delspl; 951 952 if (pops->psm_translate_irq) 953 psm_translate_irq = pops->psm_translate_irq; 954 if (pops->psm_intr_ops) 955 psm_intr_ops = pops->psm_intr_ops; 956 957 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 958 /* 959 * Time-of-day functionality now handled in TOD modules. 960 * (Warn about PSM modules that think that we're going to use 961 * their ops vectors.) 962 */ 963 if (pops->psm_tod_get) 964 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 965 (void *)pops->psm_tod_get); 966 967 if (pops->psm_tod_set) 968 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 969 (void *)pops->psm_tod_set); 970 #endif 971 972 if (pops->psm_notify_error) { 973 psm_notify_error = mach_notify_error; 974 notify_error = pops->psm_notify_error; 975 } 976 977 (*pops->psm_softinit)(); 978 979 /* 980 * Initialize the dispatcher's function hooks to enable CPU halting 981 * when idle. Set both the deep-idle and non-deep-idle hooks. 982 * 983 * Assume we can use power saving deep-idle loop cpu_idle_adaptive. 984 * Platform deep-idle driver will reset our idle loop to 985 * non_deep_idle_cpu if power saving deep-idle feature is not available. 986 * 987 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) 988 * or idle_cpu_prefer_mwait is not set. 989 * Allocate monitor/mwait buffer for cpu0. 990 */ 991 #ifndef __xpv 992 non_deep_idle_disp_enq_thread = disp_enq_thread; 993 #endif 994 if (idle_cpu_use_hlt) { 995 idle_cpu = cpu_idle_adaptive; 996 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 997 #ifndef __xpv 998 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) { 999 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); 1000 /* 1001 * Protect ourself from insane mwait size. 1002 */ 1003 if (CPU->cpu_m.mcpu_mwait == NULL) { 1004 #ifdef DEBUG 1005 cmn_err(CE_NOTE, "Using hlt idle. Cannot " 1006 "handle cpu 0 mwait size."); 1007 #endif 1008 idle_cpu_prefer_mwait = 0; 1009 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1010 } else { 1011 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; 1012 } 1013 } else { 1014 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1015 } 1016 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; 1017 1018 /* 1019 * Disable power saving deep idle loop? 1020 */ 1021 if (idle_cpu_no_deep_c) { 1022 idle_cpu = non_deep_idle_cpu; 1023 } 1024 #endif 1025 } 1026 1027 mach_smpinit(); 1028 } 1029 1030 static void 1031 mach_smpinit(void) 1032 { 1033 struct psm_ops *pops; 1034 processorid_t cpu_id; 1035 int cnt; 1036 cpuset_t cpumask; 1037 1038 pops = mach_set[0]; 1039 CPUSET_ZERO(cpumask); 1040 1041 cpu_id = -1; 1042 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1043 for (cnt = 0; cpu_id != -1; cnt++) { 1044 CPUSET_ADD(cpumask, cpu_id); 1045 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1046 } 1047 1048 mp_cpus = cpumask; 1049 1050 /* MP related routines */ 1051 ap_mlsetup = pops->psm_post_cpu_start; 1052 send_dirintf = pops->psm_send_ipi; 1053 1054 /* optional MP related routines */ 1055 if (pops->psm_shutdown) 1056 psm_shutdownf = pops->psm_shutdown; 1057 if (pops->psm_preshutdown) 1058 psm_preshutdownf = pops->psm_preshutdown; 1059 if (pops->psm_notify_func) 1060 psm_notifyf = pops->psm_notify_func; 1061 if (pops->psm_set_idlecpu) 1062 psm_set_idle_cpuf = pops->psm_set_idlecpu; 1063 if (pops->psm_unset_idlecpu) 1064 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 1065 1066 psm_clkinit = pops->psm_clkinit; 1067 1068 if (pops->psm_timer_reprogram) 1069 psm_timer_reprogram = pops->psm_timer_reprogram; 1070 1071 if (pops->psm_timer_enable) 1072 psm_timer_enable = pops->psm_timer_enable; 1073 1074 if (pops->psm_timer_disable) 1075 psm_timer_disable = pops->psm_timer_disable; 1076 1077 if (pops->psm_post_cyclic_setup) 1078 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 1079 1080 if (pops->psm_state) 1081 psm_state = pops->psm_state; 1082 1083 /* 1084 * Set these vectors here so they can be used by Suspend/Resume 1085 * on UP machines. 1086 */ 1087 if (pops->psm_disable_intr) 1088 psm_disable_intr = pops->psm_disable_intr; 1089 if (pops->psm_enable_intr) 1090 psm_enable_intr = pops->psm_enable_intr; 1091 1092 /* check for multiple CPUs */ 1093 if (cnt < 2) 1094 return; 1095 1096 /* check for MP platforms */ 1097 if (pops->psm_cpu_start == NULL) 1098 return; 1099 1100 /* 1101 * Set the dispatcher hook to enable cpu "wake up" 1102 * when a thread becomes runnable. 1103 */ 1104 if (idle_cpu_use_hlt) { 1105 disp_enq_thread = cpu_wakeup; 1106 #ifndef __xpv 1107 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) 1108 disp_enq_thread = cpu_wakeup_mwait; 1109 non_deep_idle_disp_enq_thread = disp_enq_thread; 1110 #endif 1111 } 1112 1113 psm_get_ipivect = pops->psm_get_ipivect; 1114 1115 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", 1116 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 1117 NULL, NULL, NULL, NULL); 1118 1119 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 1120 } 1121 1122 static void 1123 mach_picinit() 1124 { 1125 struct psm_ops *pops; 1126 1127 pops = mach_set[0]; 1128 1129 /* register the interrupt handlers */ 1130 setlvl = pops->psm_intr_enter; 1131 setlvlx = pops->psm_intr_exit; 1132 1133 /* initialize the interrupt hardware */ 1134 (*pops->psm_picinit)(); 1135 1136 /* set interrupt mask for current ipl */ 1137 setspl = pops->psm_setspl; 1138 cli(); 1139 setspl(CPU->cpu_pri); 1140 } 1141 1142 uint_t cpu_freq; /* MHz */ 1143 uint64_t cpu_freq_hz; /* measured (in hertz) */ 1144 1145 #define MEGA_HZ 1000000 1146 1147 #ifdef __xpv 1148 1149 int xpv_cpufreq_workaround = 1; 1150 int xpv_cpufreq_verbose = 0; 1151 1152 #else /* __xpv */ 1153 1154 static uint64_t 1155 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 1156 { 1157 uint64_t cpu_hz; 1158 1159 if ((pit_counter == 0) || (*processor_clks == 0) || 1160 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 1161 return (0); 1162 1163 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 1164 1165 return (cpu_hz); 1166 } 1167 1168 #endif /* __xpv */ 1169 1170 static uint64_t 1171 mach_getcpufreq(void) 1172 { 1173 #if defined(__xpv) 1174 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; 1175 uint64_t cpu_hz; 1176 1177 /* 1178 * During dom0 bringup, it was noted that on at least one older 1179 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul 1180 * value that is quite wrong (the 3.06GHz clock was reported 1181 * as 4.77GHz) 1182 * 1183 * The curious thing is, that if you stop the kernel at entry, 1184 * breakpoint here and inspect the value with kmdb, the value 1185 * is correct - but if you don't stop and simply enable the 1186 * printf statement (below), you can see the bad value printed 1187 * here. Almost as if something kmdb did caused the hypervisor to 1188 * figure it out correctly. And, note that the hypervisor 1189 * eventually -does- figure it out correctly ... if you look at 1190 * the field later in the life of dom0, it is correct. 1191 * 1192 * For now, on dom0, we employ a slightly cheesy workaround of 1193 * using the DOM0_PHYSINFO hypercall. 1194 */ 1195 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { 1196 cpu_hz = 1000 * xpv_cpu_khz(); 1197 } else { 1198 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; 1199 1200 if (vti->tsc_shift < 0) 1201 cpu_hz <<= -vti->tsc_shift; 1202 else 1203 cpu_hz >>= vti->tsc_shift; 1204 } 1205 1206 if (xpv_cpufreq_verbose) 1207 printf("mach_getcpufreq: system_mul 0x%x, shift %d, " 1208 "cpu_hz %" PRId64 "Hz\n", 1209 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); 1210 1211 return (cpu_hz); 1212 #else /* __xpv */ 1213 uint32_t pit_counter; 1214 uint64_t processor_clks; 1215 1216 if (x86_feature & X86_TSC) { 1217 /* 1218 * We have a TSC. freq_tsc() knows how to measure the number 1219 * of clock cycles sampled against the PIT. 1220 */ 1221 ulong_t flags = clear_int_flag(); 1222 processor_clks = freq_tsc(&pit_counter); 1223 restore_int_flag(flags); 1224 return (mach_calchz(pit_counter, &processor_clks)); 1225 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1226 #if defined(__amd64) 1227 panic("mach_getcpufreq: no TSC!"); 1228 #elif defined(__i386) 1229 /* 1230 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1231 * for which freq_notsc() knows how to measure the number of 1232 * elapsed clock cycles sampled against the PIT 1233 */ 1234 ulong_t flags = clear_int_flag(); 1235 processor_clks = freq_notsc(&pit_counter); 1236 restore_int_flag(flags); 1237 return (mach_calchz(pit_counter, &processor_clks)); 1238 #endif /* __i386 */ 1239 } 1240 1241 /* We do not know how to calculate cpu frequency for this cpu. */ 1242 return (0); 1243 #endif /* __xpv */ 1244 } 1245 1246 /* 1247 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1248 * to this array, instead improve the accuracy of the algorithm that determines 1249 * the clock speed of the processor or extend the implementation to support the 1250 * vendor as appropriate. This is here only to support adjusting the speed on 1251 * older slower processors that mach_fixcpufreq() would not be able to account 1252 * for otherwise. 1253 */ 1254 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1255 1256 /* 1257 * On fast processors the clock frequency that is measured may be off by 1258 * a few MHz from the value printed on the part. This is a combination of 1259 * the factors that for such fast parts being off by this much is within 1260 * the tolerances for manufacture and because of the difficulties in the 1261 * measurement that can lead to small error. This function uses some 1262 * heuristics in order to tweak the value that was measured to match what 1263 * is most likely printed on the part. 1264 * 1265 * Some examples: 1266 * AMD Athlon 1000 mhz measured as 998 mhz 1267 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1268 * Intel Pentium IV 1500 mhz measured as 1495mhz 1269 * 1270 * If in the future this function is no longer sufficient to correct 1271 * for the error in the measurement, then the algorithm used to perform 1272 * the measurement will have to be improved in order to increase accuracy 1273 * rather than adding horrible and questionable kludges here. 1274 * 1275 * This is called after the cyclics subsystem because of the potential 1276 * that the heuristics within may give a worse estimate of the clock 1277 * frequency than the value that was measured. 1278 */ 1279 static void 1280 mach_fixcpufreq(void) 1281 { 1282 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1283 1284 freq = (uint32_t)cpu_freq; 1285 1286 /* 1287 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1288 * measured speed taking into account that the 667 MHz parts were 1289 * the first to round-up. 1290 */ 1291 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1292 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1293 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1294 1295 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1296 mul = (freq + 25) / 50; 1297 near50 = mul * 50; 1298 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1299 1300 /* Find the closer of the two */ 1301 if (delta66 < delta50) { 1302 fixed = near66; 1303 delta = delta66; 1304 } else { 1305 fixed = near50; 1306 delta = delta50; 1307 } 1308 1309 if (fixed > INT_MAX) 1310 return; 1311 1312 /* 1313 * Some older parts have a core clock frequency that is not an 1314 * integral multiple of 50 or 66 MHz. Check if one of the old 1315 * clock frequencies is closer to the measured value than any 1316 * of the integral multiples of 50 an 66, and if so set fixed 1317 * and delta appropriately to represent the closest value. 1318 */ 1319 i = sizeof (x86_cpu_freq) / sizeof (int); 1320 while (i > 0) { 1321 i--; 1322 1323 if (x86_cpu_freq[i] <= freq) { 1324 mul = freq - x86_cpu_freq[i]; 1325 1326 if (mul < delta) { 1327 fixed = x86_cpu_freq[i]; 1328 delta = mul; 1329 } 1330 1331 break; 1332 } 1333 1334 mul = x86_cpu_freq[i] - freq; 1335 1336 if (mul < delta) { 1337 fixed = x86_cpu_freq[i]; 1338 delta = mul; 1339 } 1340 } 1341 1342 /* 1343 * Set a reasonable maximum for how much to correct the measured 1344 * result by. This check is here to prevent the adjustment made 1345 * by this function from being more harm than good. It is entirely 1346 * possible that in the future parts will be made that are not 1347 * integral multiples of 66 or 50 in clock frequency or that 1348 * someone may overclock a part to some odd frequency. If the 1349 * measured value is farther from the corrected value than 1350 * allowed, then assume the corrected value is in error and use 1351 * the measured value. 1352 */ 1353 if (6 < delta) 1354 return; 1355 1356 cpu_freq = (int)fixed; 1357 } 1358 1359 1360 static int 1361 machhztomhz(uint64_t cpu_freq_hz) 1362 { 1363 uint64_t cpu_mhz; 1364 1365 /* Round to nearest MHZ */ 1366 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1367 1368 if (cpu_mhz > INT_MAX) 1369 return (0); 1370 1371 return ((int)cpu_mhz); 1372 1373 } 1374 1375 1376 static int 1377 mach_clkinit(int preferred_mode, int *set_mode) 1378 { 1379 struct psm_ops *pops; 1380 int resolution; 1381 1382 pops = mach_set[0]; 1383 1384 cpu_freq_hz = mach_getcpufreq(); 1385 1386 cpu_freq = machhztomhz(cpu_freq_hz); 1387 1388 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 1389 tsc_gethrtime_enable = 0; 1390 1391 #ifndef __xpv 1392 if (tsc_gethrtime_enable) { 1393 tsc_hrtimeinit(cpu_freq_hz); 1394 } else 1395 #endif 1396 { 1397 if (pops->psm_hrtimeinit) 1398 (*pops->psm_hrtimeinit)(); 1399 gethrtimef = pops->psm_gethrtime; 1400 gethrtimeunscaledf = gethrtimef; 1401 /* scalehrtimef will remain dummy */ 1402 } 1403 1404 mach_fixcpufreq(); 1405 1406 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1407 if (preferred_mode == TIMER_ONESHOT) { 1408 1409 resolution = (*pops->psm_clkinit)(0); 1410 if (resolution != 0) { 1411 *set_mode = TIMER_ONESHOT; 1412 return (resolution); 1413 } 1414 } 1415 1416 /* 1417 * either periodic mode was requested or could not set to 1418 * one-shot mode 1419 */ 1420 resolution = (*pops->psm_clkinit)(hz); 1421 /* 1422 * psm should be able to do periodic, so we do not check 1423 * for return value of psm_clkinit here. 1424 */ 1425 *set_mode = TIMER_PERIODIC; 1426 return (resolution); 1427 } else { 1428 /* 1429 * PSMI interface prior to PSMI_3 does not define a return 1430 * value for psm_clkinit, so the return value is ignored. 1431 */ 1432 (void) (*pops->psm_clkinit)(hz); 1433 *set_mode = TIMER_PERIODIC; 1434 return (nsec_per_tick); 1435 } 1436 } 1437 1438 1439 /*ARGSUSED*/ 1440 static int 1441 mach_softlvl_to_vect(int ipl) 1442 { 1443 setsoftint = av_set_softint_pending; 1444 kdisetsoftint = kdi_av_set_softint_pending; 1445 1446 return (PSM_SV_SOFTWARE); 1447 } 1448 1449 #ifdef DEBUG 1450 /* 1451 * This is here to allow us to simulate cpus that refuse to start. 1452 */ 1453 cpuset_t cpufailset; 1454 #endif 1455 1456 int 1457 mach_cpu_start(struct cpu *cp, void *ctx) 1458 { 1459 struct psm_ops *pops = mach_set[0]; 1460 processorid_t id = cp->cpu_id; 1461 1462 #ifdef DEBUG 1463 if (CPU_IN_SET(cpufailset, id)) 1464 return (0); 1465 #endif 1466 return ((*pops->psm_cpu_start)(id, ctx)); 1467 } 1468 1469 int 1470 mach_cpuid_start(processorid_t id, void *ctx) 1471 { 1472 struct psm_ops *pops = mach_set[0]; 1473 1474 #ifdef DEBUG 1475 if (CPU_IN_SET(cpufailset, id)) 1476 return (0); 1477 #endif 1478 return ((*pops->psm_cpu_start)(id, ctx)); 1479 } 1480 1481 /* 1482 * Default handler to create device node for CPU. 1483 * One reference count will be held on created device node. 1484 */ 1485 static int 1486 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp) 1487 { 1488 int rv, circ; 1489 dev_info_t *dip; 1490 static kmutex_t cpu_node_lock; 1491 static dev_info_t *cpu_nex_devi = NULL; 1492 1493 ASSERT(cp != NULL); 1494 ASSERT(dipp != NULL); 1495 *dipp = NULL; 1496 1497 if (cpu_nex_devi == NULL) { 1498 mutex_enter(&cpu_node_lock); 1499 /* First check whether cpus exists. */ 1500 cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0); 1501 /* Create cpus if it doesn't exist. */ 1502 if (cpu_nex_devi == NULL) { 1503 ndi_devi_enter(ddi_root_node(), &circ); 1504 rv = ndi_devi_alloc(ddi_root_node(), "cpus", 1505 (pnode_t)DEVI_SID_NODEID, &dip); 1506 if (rv != NDI_SUCCESS) { 1507 mutex_exit(&cpu_node_lock); 1508 cmn_err(CE_CONT, 1509 "?failed to create cpu nexus device.\n"); 1510 return (PSM_FAILURE); 1511 } 1512 ASSERT(dip != NULL); 1513 (void) ndi_devi_online(dip, 0); 1514 ndi_devi_exit(ddi_root_node(), circ); 1515 cpu_nex_devi = dip; 1516 } 1517 mutex_exit(&cpu_node_lock); 1518 } 1519 1520 /* 1521 * create a child node for cpu identified as 'cpu_id' 1522 */ 1523 ndi_devi_enter(cpu_nex_devi, &circ); 1524 dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1); 1525 if (dip == NULL) { 1526 cmn_err(CE_CONT, 1527 "?failed to create device node for cpu%d.\n", cp->cpu_id); 1528 rv = PSM_FAILURE; 1529 } else { 1530 *dipp = dip; 1531 (void) ndi_hold_devi(dip); 1532 rv = PSM_SUCCESS; 1533 } 1534 ndi_devi_exit(cpu_nex_devi, circ); 1535 1536 return (rv); 1537 } 1538 1539 /* 1540 * Create cpu device node in device tree and online it. 1541 * Return created dip with reference count held if requested. 1542 */ 1543 int 1544 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp) 1545 { 1546 int rv; 1547 dev_info_t *dip = NULL; 1548 1549 ASSERT(psm_cpu_create_devinfo != NULL); 1550 rv = psm_cpu_create_devinfo(cp, &dip); 1551 if (rv == PSM_SUCCESS) { 1552 cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi); 1553 /* Recursively attach driver for parent nexus device. */ 1554 if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) == 1555 DDI_SUCCESS) { 1556 /* Configure cpu itself and descendants. */ 1557 (void) ndi_devi_online(dip, 1558 NDI_ONLINE_ATTACH | NDI_CONFIG); 1559 } 1560 if (dipp != NULL) { 1561 *dipp = dip; 1562 } else { 1563 (void) ndi_rele_devi(dip); 1564 } 1565 } 1566 1567 return (rv); 1568 } 1569 1570 /*ARGSUSED*/ 1571 static int 1572 mach_translate_irq(dev_info_t *dip, int irqno) 1573 { 1574 return (irqno); /* default to NO translation */ 1575 } 1576 1577 static void 1578 mach_notify_error(int level, char *errmsg) 1579 { 1580 /* 1581 * SL_FATAL is pass in once panicstr is set, deliver it 1582 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1583 * codes for the psmi handler 1584 */ 1585 if (level & SL_FATAL) 1586 (*notify_error)(CE_PANIC, errmsg); 1587 else if (level & SL_WARN) 1588 (*notify_error)(CE_WARN, errmsg); 1589 else if (level & SL_NOTE) 1590 (*notify_error)(CE_NOTE, errmsg); 1591 else if (level & SL_CONSOLE) 1592 (*notify_error)(CE_CONT, errmsg); 1593 } 1594 1595 /* 1596 * It provides the default basic intr_ops interface for the new DDI 1597 * interrupt framework if the PSM doesn't have one. 1598 * 1599 * Input: 1600 * dip - pointer to the dev_info structure of the requested device 1601 * hdlp - pointer to the internal interrupt handle structure for the 1602 * requested interrupt 1603 * intr_op - opcode for this call 1604 * result - pointer to the integer that will hold the result to be 1605 * passed back if return value is PSM_SUCCESS 1606 * 1607 * Output: 1608 * return value is either PSM_SUCCESS or PSM_FAILURE 1609 */ 1610 static int 1611 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1612 psm_intr_op_t intr_op, int *result) 1613 { 1614 struct intrspec *ispec; 1615 1616 switch (intr_op) { 1617 case PSM_INTR_OP_CHECK_MSI: 1618 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1619 DDI_INTR_TYPE_MSIX); 1620 break; 1621 case PSM_INTR_OP_ALLOC_VECTORS: 1622 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1623 *result = 1; 1624 else 1625 *result = 0; 1626 break; 1627 case PSM_INTR_OP_FREE_VECTORS: 1628 break; 1629 case PSM_INTR_OP_NAVAIL_VECTORS: 1630 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1631 *result = 1; 1632 else 1633 *result = 0; 1634 break; 1635 case PSM_INTR_OP_XLATE_VECTOR: 1636 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1637 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1638 break; 1639 case PSM_INTR_OP_GET_CAP: 1640 *result = 0; 1641 break; 1642 case PSM_INTR_OP_GET_PENDING: 1643 case PSM_INTR_OP_CLEAR_MASK: 1644 case PSM_INTR_OP_SET_MASK: 1645 case PSM_INTR_OP_GET_SHARED: 1646 case PSM_INTR_OP_SET_PRI: 1647 case PSM_INTR_OP_SET_CAP: 1648 case PSM_INTR_OP_SET_CPU: 1649 case PSM_INTR_OP_GET_INTR: 1650 default: 1651 return (PSM_FAILURE); 1652 } 1653 return (PSM_SUCCESS); 1654 } 1655 /* 1656 * Return 1 if CMT load balancing policies should be 1657 * implemented across instances of the specified hardware 1658 * sharing relationship. 1659 */ 1660 int 1661 pg_cmt_load_bal_hw(pghw_type_t hw) 1662 { 1663 if (hw == PGHW_IPIPE || 1664 hw == PGHW_FPU || 1665 hw == PGHW_PROCNODE || 1666 hw == PGHW_CHIP) 1667 return (1); 1668 else 1669 return (0); 1670 } 1671 /* 1672 * Return 1 if thread affinity polices should be implemented 1673 * for instances of the specifed hardware sharing relationship. 1674 */ 1675 int 1676 pg_cmt_affinity_hw(pghw_type_t hw) 1677 { 1678 if (hw == PGHW_CACHE) 1679 return (1); 1680 else 1681 return (0); 1682 } 1683