1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright (c) 2009, Intel Corporation. 27 * All rights reserved. 28 */ 29 30 #define PSMI_1_6 31 #include <sys/smp_impldefs.h> 32 #include <sys/psm.h> 33 #include <sys/psm_modctl.h> 34 #include <sys/pit.h> 35 #include <sys/cmn_err.h> 36 #include <sys/strlog.h> 37 #include <sys/clock.h> 38 #include <sys/debug.h> 39 #include <sys/rtc.h> 40 #include <sys/x86_archext.h> 41 #include <sys/cpupart.h> 42 #include <sys/cpuvar.h> 43 #include <sys/cpu_event.h> 44 #include <sys/cmt.h> 45 #include <sys/cpu.h> 46 #include <sys/disp.h> 47 #include <sys/archsystm.h> 48 #include <sys/machsystm.h> 49 #include <sys/sysmacros.h> 50 #include <sys/memlist.h> 51 #include <sys/param.h> 52 #include <sys/promif.h> 53 #include <sys/cpu_pm.h> 54 #if defined(__xpv) 55 #include <sys/hypervisor.h> 56 #endif 57 #include <sys/mach_intr.h> 58 #include <vm/hat_i86.h> 59 #include <sys/kdi_machimpl.h> 60 #include <sys/sdt.h> 61 #include <sys/hpet.h> 62 #include <sys/sunddi.h> 63 #include <sys/sunndi.h> 64 65 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 66 67 /* 68 * Local function prototypes 69 */ 70 static int mp_disable_intr(processorid_t cpun); 71 static void mp_enable_intr(processorid_t cpun); 72 static void mach_init(); 73 static void mach_picinit(); 74 static int machhztomhz(uint64_t cpu_freq_hz); 75 static uint64_t mach_getcpufreq(void); 76 static void mach_fixcpufreq(void); 77 static int mach_clkinit(int, int *); 78 static void mach_smpinit(void); 79 static int mach_softlvl_to_vect(int ipl); 80 static void mach_get_platform(int owner); 81 static void mach_construct_info(); 82 static int mach_translate_irq(dev_info_t *dip, int irqno); 83 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 84 psm_intr_op_t, int *); 85 static void mach_notify_error(int level, char *errmsg); 86 static hrtime_t dummy_hrtime(void); 87 static void dummy_scalehrtime(hrtime_t *); 88 void cpu_idle(void); 89 static void cpu_wakeup(cpu_t *, int); 90 #ifndef __xpv 91 void cpu_idle_mwait(void); 92 static void cpu_wakeup_mwait(cpu_t *, int); 93 #endif 94 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp); 95 96 /* 97 * External reference functions 98 */ 99 extern void return_instr(); 100 extern uint64_t freq_tsc(uint32_t *); 101 #if defined(__i386) 102 extern uint64_t freq_notsc(uint32_t *); 103 #endif 104 extern void pc_gethrestime(timestruc_t *); 105 extern int cpuid_get_coreid(cpu_t *); 106 extern int cpuid_get_chipid(cpu_t *); 107 108 /* 109 * PSM functions initialization 110 */ 111 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 112 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 113 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 114 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 115 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 116 void (*psminitf)() = mach_init; 117 void (*picinitf)() = return_instr; 118 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 119 int (*ap_mlsetup)() = (int (*)(void))return_instr; 120 void (*send_dirintf)() = return_instr; 121 void (*setspl)(int) = (void (*)(int))return_instr; 122 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 123 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 124 void (*kdisetsoftint)(int, struct av_softinfo *)= 125 (void (*)(int, struct av_softinfo *))return_instr; 126 void (*setsoftint)(int, struct av_softinfo *)= 127 (void (*)(int, struct av_softinfo *))return_instr; 128 int (*slvltovect)(int) = (int (*)(int))return_instr; 129 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 130 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 131 int (*psm_disable_intr)(int) = mp_disable_intr; 132 void (*psm_enable_intr)(int) = mp_enable_intr; 133 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 134 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 135 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 136 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 137 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 138 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 139 int (*psm_get_clockirq)(int) = NULL; 140 int (*psm_get_ipivect)(int, int) = NULL; 141 142 int (*psm_clkinit)(int) = NULL; 143 void (*psm_timer_reprogram)(hrtime_t) = NULL; 144 void (*psm_timer_enable)(void) = NULL; 145 void (*psm_timer_disable)(void) = NULL; 146 void (*psm_post_cyclic_setup)(void *arg) = NULL; 147 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 148 int *) = mach_intr_ops; 149 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) 150 return_instr; 151 152 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 153 void (*hrtime_tick)(void) = return_instr; 154 155 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo; 156 157 /* 158 * True if the generic TSC code is our source of hrtime, rather than whatever 159 * the PSM can provide. 160 */ 161 #ifdef __xpv 162 int tsc_gethrtime_enable = 0; 163 #else 164 int tsc_gethrtime_enable = 1; 165 #endif 166 int tsc_gethrtime_initted = 0; 167 168 /* 169 * True if the hrtime implementation is "hires"; namely, better than microdata. 170 */ 171 int gethrtime_hires = 0; 172 173 /* 174 * Local Static Data 175 */ 176 static struct psm_ops mach_ops; 177 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 178 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 179 180 /* 181 * virtualization support for psm 182 */ 183 void *psm_vt_ops = NULL; 184 /* 185 * If non-zero, idle cpus will become "halted" when there's 186 * no work to do. 187 */ 188 int idle_cpu_use_hlt = 1; 189 190 #ifndef __xpv 191 /* 192 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 193 */ 194 int idle_cpu_prefer_mwait = 1; 195 /* 196 * Set to 0 to avoid MONITOR+CLFLUSH assertion. 197 */ 198 int idle_cpu_assert_cflush_monitor = 1; 199 200 /* 201 * If non-zero, idle cpus will not use power saving Deep C-States idle loop. 202 */ 203 int idle_cpu_no_deep_c = 0; 204 /* 205 * Non-power saving idle loop and wakeup pointers. 206 * Allows user to toggle Deep Idle power saving feature on/off. 207 */ 208 void (*non_deep_idle_cpu)() = cpu_idle; 209 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); 210 211 /* 212 * Object for the kernel to access the HPET. 213 */ 214 hpet_t hpet; 215 216 #endif /* ifndef __xpv */ 217 218 /*ARGSUSED*/ 219 int 220 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 221 { 222 switch (hw) { 223 case PGHW_IPIPE: 224 if (x86_feature & (X86_HTT)) { 225 /* 226 * Hyper-threading is SMT 227 */ 228 return (1); 229 } else { 230 return (0); 231 } 232 case PGHW_CHIP: 233 if (x86_feature & (X86_CMP|X86_HTT)) 234 return (1); 235 else 236 return (0); 237 case PGHW_CACHE: 238 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 239 return (1); 240 else 241 return (0); 242 case PGHW_POW_ACTIVE: 243 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) 244 return (1); 245 else 246 return (0); 247 case PGHW_POW_IDLE: 248 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) 249 return (1); 250 else 251 return (0); 252 default: 253 return (0); 254 } 255 } 256 257 /* 258 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 259 * If pghw_type_t is an unsupported hardware type, then return -1 260 */ 261 int 262 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 263 { 264 id_t pgp_a, pgp_b; 265 266 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 267 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 268 269 if (pgp_a == -1 || pgp_b == -1) 270 return (-1); 271 272 return (pgp_a == pgp_b); 273 } 274 275 /* 276 * Return a physical instance identifier for known hardware sharing 277 * relationships 278 */ 279 id_t 280 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 281 { 282 switch (hw) { 283 case PGHW_IPIPE: 284 return (cpuid_get_coreid(cpu)); 285 case PGHW_CACHE: 286 return (cpuid_get_last_lvl_cacheid(cpu)); 287 case PGHW_CHIP: 288 return (cpuid_get_chipid(cpu)); 289 case PGHW_POW_ACTIVE: 290 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); 291 case PGHW_POW_IDLE: 292 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); 293 default: 294 return (-1); 295 } 296 } 297 298 /* 299 * Express preference for optimizing for sharing relationship 300 * hw1 vs hw2 301 */ 302 pghw_type_t 303 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) 304 { 305 int i, rank1, rank2; 306 307 static pghw_type_t hw_hier[] = { 308 PGHW_IPIPE, 309 PGHW_CACHE, 310 PGHW_CHIP, 311 PGHW_POW_IDLE, 312 PGHW_POW_ACTIVE, 313 PGHW_NUM_COMPONENTS 314 }; 315 316 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 317 if (hw_hier[i] == hw1) 318 rank1 = i; 319 if (hw_hier[i] == hw2) 320 rank2 = i; 321 } 322 323 if (rank1 > rank2) 324 return (hw1); 325 else 326 return (hw2); 327 } 328 329 /* 330 * Override the default CMT dispatcher policy for the specified 331 * hardware sharing relationship 332 */ 333 pg_cmt_policy_t 334 pg_plat_cmt_policy(pghw_type_t hw) 335 { 336 /* 337 * For shared caches, also load balance across them to 338 * maximize aggregate cache capacity 339 */ 340 switch (hw) { 341 case PGHW_CACHE: 342 return (CMT_BALANCE|CMT_AFFINITY); 343 default: 344 return (CMT_NO_POLICY); 345 } 346 } 347 348 id_t 349 pg_plat_get_core_id(cpu_t *cpu) 350 { 351 return ((id_t)cpuid_get_coreid(cpu)); 352 } 353 354 void 355 cmp_set_nosteal_interval(void) 356 { 357 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 358 nosteal_nsec = 100000UL; 359 } 360 361 /* 362 * Routine to ensure initial callers to hrtime gets 0 as return 363 */ 364 static hrtime_t 365 dummy_hrtime(void) 366 { 367 return (0); 368 } 369 370 /* ARGSUSED */ 371 static void 372 dummy_scalehrtime(hrtime_t *ticks) 373 {} 374 375 /* 376 * Supports Deep C-State power saving idle loop. 377 */ 378 void 379 cpu_idle_adaptive(void) 380 { 381 (*CPU->cpu_m.mcpu_idle_cpu)(); 382 } 383 384 /* 385 * Function called by CPU idle notification framework to check whether CPU 386 * has been awakened. It will be called with interrupt disabled. 387 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 388 * notification framework. 389 */ 390 /*ARGSUSED*/ 391 static void 392 cpu_idle_check_wakeup(void *arg) 393 { 394 /* 395 * Toggle interrupt flag to detect pending interrupts. 396 * If interrupt happened, do_interrupt() will notify CPU idle 397 * notification framework so no need to call cpu_idle_exit() here. 398 */ 399 sti(); 400 SMT_PAUSE(); 401 cli(); 402 } 403 404 /* 405 * Idle the present CPU until wakened via an interrupt 406 */ 407 void 408 cpu_idle(void) 409 { 410 cpu_t *cpup = CPU; 411 processorid_t cpu_sid = cpup->cpu_seqid; 412 cpupart_t *cp = cpup->cpu_part; 413 int hset_update = 1; 414 415 /* 416 * If this CPU is online, and there's multiple CPUs 417 * in the system, then we should notate our halting 418 * by adding ourselves to the partition's halted CPU 419 * bitmap. This allows other CPUs to find/awaken us when 420 * work becomes available. 421 */ 422 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 423 hset_update = 0; 424 425 /* 426 * Add ourselves to the partition's halted CPUs bitmap 427 * and set our HALTED flag, if necessary. 428 * 429 * When a thread becomes runnable, it is placed on the queue 430 * and then the halted CPU bitmap is checked to determine who 431 * (if anyone) should be awakened. We therefore need to first 432 * add ourselves to the bitmap, and and then check if there 433 * is any work available. The order is important to prevent a race 434 * that can lead to work languishing on a run queue somewhere while 435 * this CPU remains halted. 436 * 437 * Either the producing CPU will see we're halted and will awaken us, 438 * or this CPU will see the work available in disp_anywork(). 439 * 440 * Note that memory barriers after updating the HALTED flag 441 * are not necessary since an atomic operation (updating the bitset) 442 * immediately follows. On x86 the atomic operation acts as a 443 * memory barrier for the update of cpu_disp_flags. 444 */ 445 if (hset_update) { 446 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 447 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 448 } 449 450 /* 451 * Check to make sure there's really nothing to do. 452 * Work destined for this CPU may become available after 453 * this check. We'll be notified through the clearing of our 454 * bit in the halted CPU bitmap, and a poke. 455 */ 456 if (disp_anywork()) { 457 if (hset_update) { 458 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 459 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 460 } 461 return; 462 } 463 464 /* 465 * We're on our way to being halted. 466 * 467 * Disable interrupts now, so that we'll awaken immediately 468 * after halting if someone tries to poke us between now and 469 * the time we actually halt. 470 * 471 * We check for the presence of our bit after disabling interrupts. 472 * If it's cleared, we'll return. If the bit is cleared after 473 * we check then the poke will pop us out of the halted state. 474 * 475 * This means that the ordering of the poke and the clearing 476 * of the bit by cpu_wakeup is important. 477 * cpu_wakeup() must clear, then poke. 478 * cpu_idle() must disable interrupts, then check for the bit. 479 */ 480 cli(); 481 482 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { 483 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 484 sti(); 485 return; 486 } 487 488 /* 489 * The check for anything locally runnable is here for performance 490 * and isn't needed for correctness. disp_nrunnable ought to be 491 * in our cache still, so it's inexpensive to check, and if there 492 * is anything runnable we won't have to wait for the poke. 493 */ 494 if (cpup->cpu_disp->disp_nrunnable != 0) { 495 if (hset_update) { 496 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 497 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 498 } 499 sti(); 500 return; 501 } 502 503 if (cpu_idle_enter(IDLE_STATE_C1, 0, 504 cpu_idle_check_wakeup, NULL) == 0) { 505 mach_cpu_idle(); 506 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 507 } 508 509 /* 510 * We're no longer halted 511 */ 512 if (hset_update) { 513 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 514 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 515 } 516 } 517 518 519 /* 520 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 521 * Otherwise, see if other CPUs in the cpu partition are halted and need to 522 * be woken up so that they can steal the thread we placed on this CPU. 523 * This function is only used on MP systems. 524 */ 525 static void 526 cpu_wakeup(cpu_t *cpu, int bound) 527 { 528 uint_t cpu_found; 529 processorid_t cpu_sid; 530 cpupart_t *cp; 531 532 cp = cpu->cpu_part; 533 cpu_sid = cpu->cpu_seqid; 534 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 535 /* 536 * Clear the halted bit for that CPU since it will be 537 * poked in a moment. 538 */ 539 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 540 /* 541 * We may find the current CPU present in the halted cpuset 542 * if we're in the context of an interrupt that occurred 543 * before we had a chance to clear our bit in cpu_idle(). 544 * Poking ourself is obviously unnecessary, since if 545 * we're here, we're not halted. 546 */ 547 if (cpu != CPU) 548 poke_cpu(cpu->cpu_id); 549 return; 550 } else { 551 /* 552 * This cpu isn't halted, but it's idle or undergoing a 553 * context switch. No need to awaken anyone else. 554 */ 555 if (cpu->cpu_thread == cpu->cpu_idle_thread || 556 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 557 return; 558 } 559 560 /* 561 * No need to wake up other CPUs if this is for a bound thread. 562 */ 563 if (bound) 564 return; 565 566 /* 567 * The CPU specified for wakeup isn't currently halted, so check 568 * to see if there are any other halted CPUs in the partition, 569 * and if there are then awaken one. 570 */ 571 do { 572 cpu_found = bitset_find(&cp->cp_haltset); 573 if (cpu_found == (uint_t)-1) 574 return; 575 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 576 577 if (cpu_found != CPU->cpu_seqid) { 578 poke_cpu(cpu_seq[cpu_found]->cpu_id); 579 } 580 } 581 582 #ifndef __xpv 583 /* 584 * Function called by CPU idle notification framework to check whether CPU 585 * has been awakened. It will be called with interrupt disabled. 586 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 587 * notification framework. 588 */ 589 static void 590 cpu_idle_mwait_check_wakeup(void *arg) 591 { 592 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; 593 594 ASSERT(arg != NULL); 595 if (*mcpu_mwait != MWAIT_HALTED) { 596 /* 597 * CPU has been awakened, notify CPU idle notification system. 598 */ 599 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 600 } else { 601 /* 602 * Toggle interrupt flag to detect pending interrupts. 603 * If interrupt happened, do_interrupt() will notify CPU idle 604 * notification framework so no need to call cpu_idle_exit() 605 * here. 606 */ 607 sti(); 608 SMT_PAUSE(); 609 cli(); 610 } 611 } 612 613 /* 614 * Idle the present CPU until awakened via touching its monitored line 615 */ 616 void 617 cpu_idle_mwait(void) 618 { 619 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 620 cpu_t *cpup = CPU; 621 processorid_t cpu_sid = cpup->cpu_seqid; 622 cpupart_t *cp = cpup->cpu_part; 623 int hset_update = 1; 624 625 /* 626 * Set our mcpu_mwait here, so we can tell if anyone tries to 627 * wake us between now and when we call mwait. No other cpu will 628 * attempt to set our mcpu_mwait until we add ourself to the halted 629 * CPU bitmap. 630 */ 631 *mcpu_mwait = MWAIT_HALTED; 632 633 /* 634 * If this CPU is online, and there's multiple CPUs 635 * in the system, then we should note our halting 636 * by adding ourselves to the partition's halted CPU 637 * bitmap. This allows other CPUs to find/awaken us when 638 * work becomes available. 639 */ 640 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 641 hset_update = 0; 642 643 /* 644 * Add ourselves to the partition's halted CPUs bitmap 645 * and set our HALTED flag, if necessary. 646 * 647 * When a thread becomes runnable, it is placed on the queue 648 * and then the halted CPU bitmap is checked to determine who 649 * (if anyone) should be awakened. We therefore need to first 650 * add ourselves to the bitmap, and and then check if there 651 * is any work available. 652 * 653 * Note that memory barriers after updating the HALTED flag 654 * are not necessary since an atomic operation (updating the bitmap) 655 * immediately follows. On x86 the atomic operation acts as a 656 * memory barrier for the update of cpu_disp_flags. 657 */ 658 if (hset_update) { 659 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 660 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 661 } 662 663 /* 664 * Check to make sure there's really nothing to do. 665 * Work destined for this CPU may become available after 666 * this check. We'll be notified through the clearing of our 667 * bit in the halted CPU bitmap, and a write to our mcpu_mwait. 668 * 669 * disp_anywork() checks disp_nrunnable, so we do not have to later. 670 */ 671 if (disp_anywork()) { 672 if (hset_update) { 673 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 674 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 675 } 676 return; 677 } 678 679 /* 680 * We're on our way to being halted. 681 * To avoid a lost wakeup, arm the monitor before checking if another 682 * cpu wrote to mcpu_mwait to wake us up. 683 */ 684 i86_monitor(mcpu_mwait, 0, 0); 685 if (*mcpu_mwait == MWAIT_HALTED) { 686 if (cpu_idle_enter(IDLE_STATE_C1, 0, 687 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { 688 if (*mcpu_mwait == MWAIT_HALTED) { 689 i86_mwait(0, 0); 690 } 691 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 692 } 693 } 694 695 /* 696 * We're no longer halted 697 */ 698 if (hset_update) { 699 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 700 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 701 } 702 } 703 704 /* 705 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 706 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 707 * need to be woken up so that they can steal the thread we placed on this CPU. 708 * This function is only used on MP systems. 709 */ 710 static void 711 cpu_wakeup_mwait(cpu_t *cp, int bound) 712 { 713 cpupart_t *cpu_part; 714 uint_t cpu_found; 715 processorid_t cpu_sid; 716 717 cpu_part = cp->cpu_part; 718 cpu_sid = cp->cpu_seqid; 719 720 /* 721 * Clear the halted bit for that CPU since it will be woken up 722 * in a moment. 723 */ 724 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { 725 /* 726 * Clear the halted bit for that CPU since it will be 727 * poked in a moment. 728 */ 729 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); 730 /* 731 * We may find the current CPU present in the halted cpuset 732 * if we're in the context of an interrupt that occurred 733 * before we had a chance to clear our bit in cpu_idle(). 734 * Waking ourself is obviously unnecessary, since if 735 * we're here, we're not halted. 736 * 737 * monitor/mwait wakeup via writing to our cache line is 738 * harmless and less expensive than always checking if we 739 * are waking ourself which is an uncommon case. 740 */ 741 MWAIT_WAKEUP(cp); /* write to monitored line */ 742 return; 743 } else { 744 /* 745 * This cpu isn't halted, but it's idle or undergoing a 746 * context switch. No need to awaken anyone else. 747 */ 748 if (cp->cpu_thread == cp->cpu_idle_thread || 749 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 750 return; 751 } 752 753 /* 754 * No need to wake up other CPUs if the thread we just enqueued 755 * is bound. 756 */ 757 if (bound || ncpus == 1) 758 return; 759 760 /* 761 * See if there's any other halted CPUs. If there are, then 762 * select one, and awaken it. 763 * It's possible that after we find a CPU, somebody else 764 * will awaken it before we get the chance. 765 * In that case, look again. 766 */ 767 do { 768 cpu_found = bitset_find(&cpu_part->cp_haltset); 769 if (cpu_found == (uint_t)-1) 770 return; 771 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, 772 cpu_found) < 0); 773 774 /* 775 * Do not check if cpu_found is ourself as monitor/mwait 776 * wakeup is cheap. 777 */ 778 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ 779 } 780 781 #endif 782 783 void (*cpu_pause_handler)(volatile char *) = NULL; 784 785 static int 786 mp_disable_intr(int cpun) 787 { 788 /* 789 * switch to the offline cpu 790 */ 791 affinity_set(cpun); 792 /* 793 * raise ipl to just below cross call 794 */ 795 splx(XC_SYS_PIL - 1); 796 /* 797 * set base spl to prevent the next swtch to idle from 798 * lowering back to ipl 0 799 */ 800 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); 801 set_base_spl(); 802 affinity_clear(); 803 return (DDI_SUCCESS); 804 } 805 806 static void 807 mp_enable_intr(int cpun) 808 { 809 /* 810 * switch to the online cpu 811 */ 812 affinity_set(cpun); 813 /* 814 * clear the interrupt active mask 815 */ 816 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); 817 set_base_spl(); 818 (void) spl0(); 819 affinity_clear(); 820 } 821 822 static void 823 mach_get_platform(int owner) 824 { 825 void **srv_opsp; 826 void **clt_opsp; 827 int i; 828 int total_ops; 829 830 /* fix up psm ops */ 831 srv_opsp = (void **)mach_set[0]; 832 clt_opsp = (void **)mach_set[owner]; 833 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 834 total_ops = sizeof (struct psm_ops_ver01) / 835 sizeof (void (*)(void)); 836 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 837 /* no psm_notify_func */ 838 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 839 sizeof (void (*)(void)); 840 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 841 /* no psm_timer funcs */ 842 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 843 sizeof (void (*)(void)); 844 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 845 /* no psm_preshutdown function */ 846 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 847 sizeof (void (*)(void)); 848 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 849 /* no psm_preshutdown function */ 850 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 851 sizeof (void (*)(void)); 852 else 853 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 854 855 /* 856 * Save the version of the PSM module, in case we need to 857 * behave differently based on version. 858 */ 859 mach_ver[0] = mach_ver[owner]; 860 861 for (i = 0; i < total_ops; i++) 862 if (clt_opsp[i] != NULL) 863 srv_opsp[i] = clt_opsp[i]; 864 } 865 866 static void 867 mach_construct_info() 868 { 869 struct psm_sw *swp; 870 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 871 int conflict_owner = 0; 872 873 if (psmsw->psw_forw == psmsw) 874 panic("No valid PSM modules found"); 875 mutex_enter(&psmsw_lock); 876 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 877 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 878 continue; 879 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 880 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 881 mach_cnt[swp->psw_infop->p_owner]++; 882 } 883 mutex_exit(&psmsw_lock); 884 885 mach_get_platform(PSM_OWN_SYS_DEFAULT); 886 887 /* check to see are there any conflicts */ 888 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 889 conflict_owner = PSM_OWN_EXCLUSIVE; 890 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 891 conflict_owner = PSM_OWN_OVERRIDE; 892 if (conflict_owner) { 893 /* remove all psm modules except uppc */ 894 cmn_err(CE_WARN, 895 "Conflicts detected on the following PSM modules:"); 896 mutex_enter(&psmsw_lock); 897 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 898 if (swp->psw_infop->p_owner == conflict_owner) 899 cmn_err(CE_WARN, "%s ", 900 swp->psw_infop->p_mach_idstring); 901 } 902 mutex_exit(&psmsw_lock); 903 cmn_err(CE_WARN, 904 "Setting the system back to SINGLE processor mode!"); 905 cmn_err(CE_WARN, 906 "Please edit /etc/mach to remove the invalid PSM module."); 907 return; 908 } 909 910 if (mach_set[PSM_OWN_EXCLUSIVE]) 911 mach_get_platform(PSM_OWN_EXCLUSIVE); 912 913 if (mach_set[PSM_OWN_OVERRIDE]) 914 mach_get_platform(PSM_OWN_OVERRIDE); 915 } 916 917 static void 918 mach_init() 919 { 920 struct psm_ops *pops; 921 922 mach_construct_info(); 923 924 pops = mach_set[0]; 925 926 /* register the interrupt and clock initialization rotuines */ 927 picinitf = mach_picinit; 928 clkinitf = mach_clkinit; 929 psm_get_clockirq = pops->psm_get_clockirq; 930 931 /* register the interrupt setup code */ 932 slvltovect = mach_softlvl_to_vect; 933 addspl = pops->psm_addspl; 934 delspl = pops->psm_delspl; 935 936 if (pops->psm_translate_irq) 937 psm_translate_irq = pops->psm_translate_irq; 938 if (pops->psm_intr_ops) 939 psm_intr_ops = pops->psm_intr_ops; 940 941 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 942 /* 943 * Time-of-day functionality now handled in TOD modules. 944 * (Warn about PSM modules that think that we're going to use 945 * their ops vectors.) 946 */ 947 if (pops->psm_tod_get) 948 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 949 (void *)pops->psm_tod_get); 950 951 if (pops->psm_tod_set) 952 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 953 (void *)pops->psm_tod_set); 954 #endif 955 956 if (pops->psm_notify_error) { 957 psm_notify_error = mach_notify_error; 958 notify_error = pops->psm_notify_error; 959 } 960 961 (*pops->psm_softinit)(); 962 963 /* 964 * Initialize the dispatcher's function hooks to enable CPU halting 965 * when idle. Set both the deep-idle and non-deep-idle hooks. 966 * 967 * Assume we can use power saving deep-idle loop cpu_idle_adaptive. 968 * Platform deep-idle driver will reset our idle loop to 969 * non_deep_idle_cpu if power saving deep-idle feature is not available. 970 * 971 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) 972 * or idle_cpu_prefer_mwait is not set. 973 * Allocate monitor/mwait buffer for cpu0. 974 */ 975 #ifndef __xpv 976 non_deep_idle_disp_enq_thread = disp_enq_thread; 977 #endif 978 if (idle_cpu_use_hlt) { 979 idle_cpu = cpu_idle_adaptive; 980 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 981 #ifndef __xpv 982 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) { 983 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); 984 /* 985 * Protect ourself from insane mwait size. 986 */ 987 if (CPU->cpu_m.mcpu_mwait == NULL) { 988 #ifdef DEBUG 989 cmn_err(CE_NOTE, "Using hlt idle. Cannot " 990 "handle cpu 0 mwait size."); 991 #endif 992 idle_cpu_prefer_mwait = 0; 993 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 994 } else { 995 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; 996 } 997 } else { 998 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 999 } 1000 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; 1001 1002 /* 1003 * Disable power saving deep idle loop? 1004 */ 1005 if (idle_cpu_no_deep_c) { 1006 idle_cpu = non_deep_idle_cpu; 1007 } 1008 #endif 1009 } 1010 1011 mach_smpinit(); 1012 } 1013 1014 static void 1015 mach_smpinit(void) 1016 { 1017 struct psm_ops *pops; 1018 processorid_t cpu_id; 1019 int cnt; 1020 cpuset_t cpumask; 1021 1022 pops = mach_set[0]; 1023 CPUSET_ZERO(cpumask); 1024 1025 cpu_id = -1; 1026 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1027 for (cnt = 0; cpu_id != -1; cnt++) { 1028 CPUSET_ADD(cpumask, cpu_id); 1029 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1030 } 1031 1032 mp_cpus = cpumask; 1033 1034 /* MP related routines */ 1035 ap_mlsetup = pops->psm_post_cpu_start; 1036 send_dirintf = pops->psm_send_ipi; 1037 1038 /* optional MP related routines */ 1039 if (pops->psm_shutdown) 1040 psm_shutdownf = pops->psm_shutdown; 1041 if (pops->psm_preshutdown) 1042 psm_preshutdownf = pops->psm_preshutdown; 1043 if (pops->psm_notify_func) 1044 psm_notifyf = pops->psm_notify_func; 1045 if (pops->psm_set_idlecpu) 1046 psm_set_idle_cpuf = pops->psm_set_idlecpu; 1047 if (pops->psm_unset_idlecpu) 1048 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 1049 1050 psm_clkinit = pops->psm_clkinit; 1051 1052 if (pops->psm_timer_reprogram) 1053 psm_timer_reprogram = pops->psm_timer_reprogram; 1054 1055 if (pops->psm_timer_enable) 1056 psm_timer_enable = pops->psm_timer_enable; 1057 1058 if (pops->psm_timer_disable) 1059 psm_timer_disable = pops->psm_timer_disable; 1060 1061 if (pops->psm_post_cyclic_setup) 1062 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 1063 1064 if (pops->psm_state) 1065 psm_state = pops->psm_state; 1066 1067 /* 1068 * Set these vectors here so they can be used by Suspend/Resume 1069 * on UP machines. 1070 */ 1071 if (pops->psm_disable_intr) 1072 psm_disable_intr = pops->psm_disable_intr; 1073 if (pops->psm_enable_intr) 1074 psm_enable_intr = pops->psm_enable_intr; 1075 1076 /* check for multiple CPUs */ 1077 if (cnt < 2) 1078 return; 1079 1080 /* check for MP platforms */ 1081 if (pops->psm_cpu_start == NULL) 1082 return; 1083 1084 /* 1085 * Set the dispatcher hook to enable cpu "wake up" 1086 * when a thread becomes runnable. 1087 */ 1088 if (idle_cpu_use_hlt) { 1089 disp_enq_thread = cpu_wakeup; 1090 #ifndef __xpv 1091 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) 1092 disp_enq_thread = cpu_wakeup_mwait; 1093 non_deep_idle_disp_enq_thread = disp_enq_thread; 1094 #endif 1095 } 1096 1097 psm_get_ipivect = pops->psm_get_ipivect; 1098 1099 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", 1100 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 1101 NULL, NULL, NULL, NULL); 1102 1103 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 1104 } 1105 1106 static void 1107 mach_picinit() 1108 { 1109 struct psm_ops *pops; 1110 1111 pops = mach_set[0]; 1112 1113 /* register the interrupt handlers */ 1114 setlvl = pops->psm_intr_enter; 1115 setlvlx = pops->psm_intr_exit; 1116 1117 /* initialize the interrupt hardware */ 1118 (*pops->psm_picinit)(); 1119 1120 /* set interrupt mask for current ipl */ 1121 setspl = pops->psm_setspl; 1122 cli(); 1123 setspl(CPU->cpu_pri); 1124 } 1125 1126 uint_t cpu_freq; /* MHz */ 1127 uint64_t cpu_freq_hz; /* measured (in hertz) */ 1128 1129 #define MEGA_HZ 1000000 1130 1131 #ifdef __xpv 1132 1133 int xpv_cpufreq_workaround = 1; 1134 int xpv_cpufreq_verbose = 0; 1135 1136 #else /* __xpv */ 1137 1138 static uint64_t 1139 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 1140 { 1141 uint64_t cpu_hz; 1142 1143 if ((pit_counter == 0) || (*processor_clks == 0) || 1144 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 1145 return (0); 1146 1147 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 1148 1149 return (cpu_hz); 1150 } 1151 1152 #endif /* __xpv */ 1153 1154 static uint64_t 1155 mach_getcpufreq(void) 1156 { 1157 #if defined(__xpv) 1158 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; 1159 uint64_t cpu_hz; 1160 1161 /* 1162 * During dom0 bringup, it was noted that on at least one older 1163 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul 1164 * value that is quite wrong (the 3.06GHz clock was reported 1165 * as 4.77GHz) 1166 * 1167 * The curious thing is, that if you stop the kernel at entry, 1168 * breakpoint here and inspect the value with kmdb, the value 1169 * is correct - but if you don't stop and simply enable the 1170 * printf statement (below), you can see the bad value printed 1171 * here. Almost as if something kmdb did caused the hypervisor to 1172 * figure it out correctly. And, note that the hypervisor 1173 * eventually -does- figure it out correctly ... if you look at 1174 * the field later in the life of dom0, it is correct. 1175 * 1176 * For now, on dom0, we employ a slightly cheesy workaround of 1177 * using the DOM0_PHYSINFO hypercall. 1178 */ 1179 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { 1180 xen_sysctl_t op0, *op = &op0; 1181 1182 op->cmd = XEN_SYSCTL_physinfo; 1183 op->interface_version = XEN_SYSCTL_INTERFACE_VERSION; 1184 if (HYPERVISOR_sysctl(op) != 0) 1185 panic("physinfo op refused"); 1186 1187 cpu_hz = 1000 * (uint64_t)op->u.physinfo.cpu_khz; 1188 } else { 1189 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; 1190 1191 if (vti->tsc_shift < 0) 1192 cpu_hz <<= -vti->tsc_shift; 1193 else 1194 cpu_hz >>= vti->tsc_shift; 1195 } 1196 1197 if (xpv_cpufreq_verbose) 1198 printf("mach_getcpufreq: system_mul 0x%x, shift %d, " 1199 "cpu_hz %" PRId64 "Hz\n", 1200 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); 1201 1202 return (cpu_hz); 1203 #else /* __xpv */ 1204 uint32_t pit_counter; 1205 uint64_t processor_clks; 1206 1207 if (x86_feature & X86_TSC) { 1208 /* 1209 * We have a TSC. freq_tsc() knows how to measure the number 1210 * of clock cycles sampled against the PIT. 1211 */ 1212 ulong_t flags = clear_int_flag(); 1213 processor_clks = freq_tsc(&pit_counter); 1214 restore_int_flag(flags); 1215 return (mach_calchz(pit_counter, &processor_clks)); 1216 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1217 #if defined(__amd64) 1218 panic("mach_getcpufreq: no TSC!"); 1219 #elif defined(__i386) 1220 /* 1221 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1222 * for which freq_notsc() knows how to measure the number of 1223 * elapsed clock cycles sampled against the PIT 1224 */ 1225 ulong_t flags = clear_int_flag(); 1226 processor_clks = freq_notsc(&pit_counter); 1227 restore_int_flag(flags); 1228 return (mach_calchz(pit_counter, &processor_clks)); 1229 #endif /* __i386 */ 1230 } 1231 1232 /* We do not know how to calculate cpu frequency for this cpu. */ 1233 return (0); 1234 #endif /* __xpv */ 1235 } 1236 1237 /* 1238 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1239 * to this array, instead improve the accuracy of the algorithm that determines 1240 * the clock speed of the processor or extend the implementation to support the 1241 * vendor as appropriate. This is here only to support adjusting the speed on 1242 * older slower processors that mach_fixcpufreq() would not be able to account 1243 * for otherwise. 1244 */ 1245 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1246 1247 /* 1248 * On fast processors the clock frequency that is measured may be off by 1249 * a few MHz from the value printed on the part. This is a combination of 1250 * the factors that for such fast parts being off by this much is within 1251 * the tolerances for manufacture and because of the difficulties in the 1252 * measurement that can lead to small error. This function uses some 1253 * heuristics in order to tweak the value that was measured to match what 1254 * is most likely printed on the part. 1255 * 1256 * Some examples: 1257 * AMD Athlon 1000 mhz measured as 998 mhz 1258 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1259 * Intel Pentium IV 1500 mhz measured as 1495mhz 1260 * 1261 * If in the future this function is no longer sufficient to correct 1262 * for the error in the measurement, then the algorithm used to perform 1263 * the measurement will have to be improved in order to increase accuracy 1264 * rather than adding horrible and questionable kludges here. 1265 * 1266 * This is called after the cyclics subsystem because of the potential 1267 * that the heuristics within may give a worse estimate of the clock 1268 * frequency than the value that was measured. 1269 */ 1270 static void 1271 mach_fixcpufreq(void) 1272 { 1273 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1274 1275 freq = (uint32_t)cpu_freq; 1276 1277 /* 1278 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1279 * measured speed taking into account that the 667 MHz parts were 1280 * the first to round-up. 1281 */ 1282 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1283 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1284 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1285 1286 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1287 mul = (freq + 25) / 50; 1288 near50 = mul * 50; 1289 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1290 1291 /* Find the closer of the two */ 1292 if (delta66 < delta50) { 1293 fixed = near66; 1294 delta = delta66; 1295 } else { 1296 fixed = near50; 1297 delta = delta50; 1298 } 1299 1300 if (fixed > INT_MAX) 1301 return; 1302 1303 /* 1304 * Some older parts have a core clock frequency that is not an 1305 * integral multiple of 50 or 66 MHz. Check if one of the old 1306 * clock frequencies is closer to the measured value than any 1307 * of the integral multiples of 50 an 66, and if so set fixed 1308 * and delta appropriately to represent the closest value. 1309 */ 1310 i = sizeof (x86_cpu_freq) / sizeof (int); 1311 while (i > 0) { 1312 i--; 1313 1314 if (x86_cpu_freq[i] <= freq) { 1315 mul = freq - x86_cpu_freq[i]; 1316 1317 if (mul < delta) { 1318 fixed = x86_cpu_freq[i]; 1319 delta = mul; 1320 } 1321 1322 break; 1323 } 1324 1325 mul = x86_cpu_freq[i] - freq; 1326 1327 if (mul < delta) { 1328 fixed = x86_cpu_freq[i]; 1329 delta = mul; 1330 } 1331 } 1332 1333 /* 1334 * Set a reasonable maximum for how much to correct the measured 1335 * result by. This check is here to prevent the adjustment made 1336 * by this function from being more harm than good. It is entirely 1337 * possible that in the future parts will be made that are not 1338 * integral multiples of 66 or 50 in clock frequency or that 1339 * someone may overclock a part to some odd frequency. If the 1340 * measured value is farther from the corrected value than 1341 * allowed, then assume the corrected value is in error and use 1342 * the measured value. 1343 */ 1344 if (6 < delta) 1345 return; 1346 1347 cpu_freq = (int)fixed; 1348 } 1349 1350 1351 static int 1352 machhztomhz(uint64_t cpu_freq_hz) 1353 { 1354 uint64_t cpu_mhz; 1355 1356 /* Round to nearest MHZ */ 1357 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1358 1359 if (cpu_mhz > INT_MAX) 1360 return (0); 1361 1362 return ((int)cpu_mhz); 1363 1364 } 1365 1366 1367 static int 1368 mach_clkinit(int preferred_mode, int *set_mode) 1369 { 1370 struct psm_ops *pops; 1371 int resolution; 1372 1373 pops = mach_set[0]; 1374 1375 cpu_freq_hz = mach_getcpufreq(); 1376 1377 cpu_freq = machhztomhz(cpu_freq_hz); 1378 1379 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 1380 tsc_gethrtime_enable = 0; 1381 1382 #ifndef __xpv 1383 if (tsc_gethrtime_enable) { 1384 tsc_hrtimeinit(cpu_freq_hz); 1385 } else 1386 #endif 1387 { 1388 if (pops->psm_hrtimeinit) 1389 (*pops->psm_hrtimeinit)(); 1390 gethrtimef = pops->psm_gethrtime; 1391 gethrtimeunscaledf = gethrtimef; 1392 /* scalehrtimef will remain dummy */ 1393 } 1394 1395 mach_fixcpufreq(); 1396 1397 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1398 if (preferred_mode == TIMER_ONESHOT) { 1399 1400 resolution = (*pops->psm_clkinit)(0); 1401 if (resolution != 0) { 1402 *set_mode = TIMER_ONESHOT; 1403 return (resolution); 1404 } 1405 } 1406 1407 /* 1408 * either periodic mode was requested or could not set to 1409 * one-shot mode 1410 */ 1411 resolution = (*pops->psm_clkinit)(hz); 1412 /* 1413 * psm should be able to do periodic, so we do not check 1414 * for return value of psm_clkinit here. 1415 */ 1416 *set_mode = TIMER_PERIODIC; 1417 return (resolution); 1418 } else { 1419 /* 1420 * PSMI interface prior to PSMI_3 does not define a return 1421 * value for psm_clkinit, so the return value is ignored. 1422 */ 1423 (void) (*pops->psm_clkinit)(hz); 1424 *set_mode = TIMER_PERIODIC; 1425 return (nsec_per_tick); 1426 } 1427 } 1428 1429 1430 /*ARGSUSED*/ 1431 static int 1432 mach_softlvl_to_vect(int ipl) 1433 { 1434 setsoftint = av_set_softint_pending; 1435 kdisetsoftint = kdi_av_set_softint_pending; 1436 1437 return (PSM_SV_SOFTWARE); 1438 } 1439 1440 #ifdef DEBUG 1441 /* 1442 * This is here to allow us to simulate cpus that refuse to start. 1443 */ 1444 cpuset_t cpufailset; 1445 #endif 1446 1447 int 1448 mach_cpu_start(struct cpu *cp, void *ctx) 1449 { 1450 struct psm_ops *pops = mach_set[0]; 1451 processorid_t id = cp->cpu_id; 1452 1453 #ifdef DEBUG 1454 if (CPU_IN_SET(cpufailset, id)) 1455 return (0); 1456 #endif 1457 return ((*pops->psm_cpu_start)(id, ctx)); 1458 } 1459 1460 int 1461 mach_cpuid_start(processorid_t id, void *ctx) 1462 { 1463 struct psm_ops *pops = mach_set[0]; 1464 1465 #ifdef DEBUG 1466 if (CPU_IN_SET(cpufailset, id)) 1467 return (0); 1468 #endif 1469 return ((*pops->psm_cpu_start)(id, ctx)); 1470 } 1471 1472 /* 1473 * Default handler to create device node for CPU. 1474 * One reference count will be held on created device node. 1475 */ 1476 static int 1477 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp) 1478 { 1479 int rv, circ; 1480 dev_info_t *dip; 1481 static kmutex_t cpu_node_lock; 1482 static dev_info_t *cpu_nex_devi = NULL; 1483 1484 ASSERT(cp != NULL); 1485 ASSERT(dipp != NULL); 1486 *dipp = NULL; 1487 1488 if (cpu_nex_devi == NULL) { 1489 mutex_enter(&cpu_node_lock); 1490 /* First check whether cpus exists. */ 1491 cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0); 1492 /* Create cpus if it doesn't exist. */ 1493 if (cpu_nex_devi == NULL) { 1494 ndi_devi_enter(ddi_root_node(), &circ); 1495 rv = ndi_devi_alloc(ddi_root_node(), "cpus", 1496 (pnode_t)DEVI_SID_NODEID, &dip); 1497 if (rv != NDI_SUCCESS) { 1498 mutex_exit(&cpu_node_lock); 1499 cmn_err(CE_CONT, 1500 "?failed to create cpu nexus device.\n"); 1501 return (PSM_FAILURE); 1502 } 1503 ASSERT(dip != NULL); 1504 (void) ndi_devi_online(dip, 0); 1505 ndi_devi_exit(ddi_root_node(), circ); 1506 cpu_nex_devi = dip; 1507 } 1508 mutex_exit(&cpu_node_lock); 1509 } 1510 1511 /* 1512 * create a child node for cpu identified as 'cpu_id' 1513 */ 1514 ndi_devi_enter(cpu_nex_devi, &circ); 1515 dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, cp->cpu_id); 1516 if (dip == NULL) { 1517 cmn_err(CE_CONT, 1518 "?failed to create device node for cpu%d.\n", cp->cpu_id); 1519 rv = PSM_FAILURE; 1520 } else { 1521 *dipp = dip; 1522 (void) ndi_hold_devi(dip); 1523 rv = PSM_SUCCESS; 1524 } 1525 ndi_devi_exit(cpu_nex_devi, circ); 1526 1527 return (rv); 1528 } 1529 1530 /* 1531 * Create cpu device node in device tree and online it. 1532 * Return created dip with reference count held if requested. 1533 */ 1534 int 1535 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp) 1536 { 1537 int rv; 1538 dev_info_t *dip = NULL; 1539 1540 ASSERT(psm_cpu_create_devinfo != NULL); 1541 rv = psm_cpu_create_devinfo(cp, &dip); 1542 if (rv == PSM_SUCCESS) { 1543 cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi); 1544 /* Recursively attach driver for parent nexus device. */ 1545 if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) == 1546 DDI_SUCCESS) { 1547 /* Configure cpu itself and descendants. */ 1548 (void) ndi_devi_online(dip, 1549 NDI_ONLINE_ATTACH | NDI_CONFIG); 1550 } 1551 if (dipp != NULL) { 1552 *dipp = dip; 1553 } else { 1554 (void) ndi_rele_devi(dip); 1555 } 1556 } 1557 1558 return (rv); 1559 } 1560 1561 /*ARGSUSED*/ 1562 static int 1563 mach_translate_irq(dev_info_t *dip, int irqno) 1564 { 1565 return (irqno); /* default to NO translation */ 1566 } 1567 1568 static void 1569 mach_notify_error(int level, char *errmsg) 1570 { 1571 /* 1572 * SL_FATAL is pass in once panicstr is set, deliver it 1573 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1574 * codes for the psmi handler 1575 */ 1576 if (level & SL_FATAL) 1577 (*notify_error)(CE_PANIC, errmsg); 1578 else if (level & SL_WARN) 1579 (*notify_error)(CE_WARN, errmsg); 1580 else if (level & SL_NOTE) 1581 (*notify_error)(CE_NOTE, errmsg); 1582 else if (level & SL_CONSOLE) 1583 (*notify_error)(CE_CONT, errmsg); 1584 } 1585 1586 /* 1587 * It provides the default basic intr_ops interface for the new DDI 1588 * interrupt framework if the PSM doesn't have one. 1589 * 1590 * Input: 1591 * dip - pointer to the dev_info structure of the requested device 1592 * hdlp - pointer to the internal interrupt handle structure for the 1593 * requested interrupt 1594 * intr_op - opcode for this call 1595 * result - pointer to the integer that will hold the result to be 1596 * passed back if return value is PSM_SUCCESS 1597 * 1598 * Output: 1599 * return value is either PSM_SUCCESS or PSM_FAILURE 1600 */ 1601 static int 1602 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1603 psm_intr_op_t intr_op, int *result) 1604 { 1605 struct intrspec *ispec; 1606 1607 switch (intr_op) { 1608 case PSM_INTR_OP_CHECK_MSI: 1609 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1610 DDI_INTR_TYPE_MSIX); 1611 break; 1612 case PSM_INTR_OP_ALLOC_VECTORS: 1613 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1614 *result = 1; 1615 else 1616 *result = 0; 1617 break; 1618 case PSM_INTR_OP_FREE_VECTORS: 1619 break; 1620 case PSM_INTR_OP_NAVAIL_VECTORS: 1621 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1622 *result = 1; 1623 else 1624 *result = 0; 1625 break; 1626 case PSM_INTR_OP_XLATE_VECTOR: 1627 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1628 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1629 break; 1630 case PSM_INTR_OP_GET_CAP: 1631 *result = 0; 1632 break; 1633 case PSM_INTR_OP_GET_PENDING: 1634 case PSM_INTR_OP_CLEAR_MASK: 1635 case PSM_INTR_OP_SET_MASK: 1636 case PSM_INTR_OP_GET_SHARED: 1637 case PSM_INTR_OP_SET_PRI: 1638 case PSM_INTR_OP_SET_CAP: 1639 case PSM_INTR_OP_SET_CPU: 1640 case PSM_INTR_OP_GET_INTR: 1641 default: 1642 return (PSM_FAILURE); 1643 } 1644 return (PSM_SUCCESS); 1645 } 1646 /* 1647 * Return 1 if CMT load balancing policies should be 1648 * implemented across instances of the specified hardware 1649 * sharing relationship. 1650 */ 1651 int 1652 pg_cmt_load_bal_hw(pghw_type_t hw) 1653 { 1654 if (hw == PGHW_IPIPE || 1655 hw == PGHW_FPU || 1656 hw == PGHW_CHIP) 1657 return (1); 1658 else 1659 return (0); 1660 } 1661 /* 1662 * Return 1 if thread affinity polices should be implemented 1663 * for instances of the specifed hardware sharing relationship. 1664 */ 1665 int 1666 pg_cmt_affinity_hw(pghw_type_t hw) 1667 { 1668 if (hw == PGHW_CACHE) 1669 return (1); 1670 else 1671 return (0); 1672 } 1673