1 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 /* 26 * Copyright (c) 2009-2010, Intel Corporation. 27 * All rights reserved. 28 */ 29 30 #define PSMI_1_7 31 #include <sys/smp_impldefs.h> 32 #include <sys/psm.h> 33 #include <sys/psm_modctl.h> 34 #include <sys/pit.h> 35 #include <sys/cmn_err.h> 36 #include <sys/strlog.h> 37 #include <sys/clock.h> 38 #include <sys/debug.h> 39 #include <sys/rtc.h> 40 #include <sys/x86_archext.h> 41 #include <sys/cpupart.h> 42 #include <sys/cpuvar.h> 43 #include <sys/cpu_event.h> 44 #include <sys/cmt.h> 45 #include <sys/cpu.h> 46 #include <sys/disp.h> 47 #include <sys/archsystm.h> 48 #include <sys/machsystm.h> 49 #include <sys/sysmacros.h> 50 #include <sys/memlist.h> 51 #include <sys/param.h> 52 #include <sys/promif.h> 53 #include <sys/cpu_pm.h> 54 #if defined(__xpv) 55 #include <sys/hypervisor.h> 56 #endif 57 #include <sys/mach_intr.h> 58 #include <vm/hat_i86.h> 59 #include <sys/kdi_machimpl.h> 60 #include <sys/sdt.h> 61 #include <sys/hpet.h> 62 #include <sys/sunddi.h> 63 #include <sys/sunndi.h> 64 #include <sys/cpc_pcbe.h> 65 66 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 67 68 /* 69 * Local function prototypes 70 */ 71 static int mp_disable_intr(processorid_t cpun); 72 static void mp_enable_intr(processorid_t cpun); 73 static void mach_init(); 74 static void mach_picinit(); 75 static int machhztomhz(uint64_t cpu_freq_hz); 76 static uint64_t mach_getcpufreq(void); 77 static void mach_fixcpufreq(void); 78 static int mach_clkinit(int, int *); 79 static void mach_smpinit(void); 80 static int mach_softlvl_to_vect(int ipl); 81 static void mach_get_platform(int owner); 82 static void mach_construct_info(); 83 static int mach_translate_irq(dev_info_t *dip, int irqno); 84 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 85 psm_intr_op_t, int *); 86 static void mach_notify_error(int level, char *errmsg); 87 static hrtime_t dummy_hrtime(void); 88 static void dummy_scalehrtime(hrtime_t *); 89 static uint64_t dummy_unscalehrtime(hrtime_t); 90 void cpu_idle(void); 91 static void cpu_wakeup(cpu_t *, int); 92 #ifndef __xpv 93 void cpu_idle_mwait(void); 94 static void cpu_wakeup_mwait(cpu_t *, int); 95 #endif 96 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp); 97 98 /* 99 * External reference functions 100 */ 101 extern void return_instr(); 102 extern uint64_t freq_tsc(uint32_t *); 103 #if defined(__i386) 104 extern uint64_t freq_notsc(uint32_t *); 105 #endif 106 extern void pc_gethrestime(timestruc_t *); 107 extern int cpuid_get_coreid(cpu_t *); 108 extern int cpuid_get_chipid(cpu_t *); 109 110 /* 111 * PSM functions initialization 112 */ 113 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 114 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 115 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 116 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 117 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 118 void (*psminitf)() = mach_init; 119 void (*picinitf)() = return_instr; 120 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 121 int (*ap_mlsetup)() = (int (*)(void))return_instr; 122 void (*send_dirintf)() = return_instr; 123 void (*setspl)(int) = (void (*)(int))return_instr; 124 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 125 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 126 void (*kdisetsoftint)(int, struct av_softinfo *)= 127 (void (*)(int, struct av_softinfo *))return_instr; 128 void (*setsoftint)(int, struct av_softinfo *)= 129 (void (*)(int, struct av_softinfo *))return_instr; 130 int (*slvltovect)(int) = (int (*)(int))return_instr; 131 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 132 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 133 int (*psm_disable_intr)(int) = mp_disable_intr; 134 void (*psm_enable_intr)(int) = mp_enable_intr; 135 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 136 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 137 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 138 uint64_t (*unscalehrtimef)(hrtime_t) = dummy_unscalehrtime; 139 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 140 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 141 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 142 int (*psm_get_clockirq)(int) = NULL; 143 int (*psm_get_ipivect)(int, int) = NULL; 144 145 int (*psm_clkinit)(int) = NULL; 146 void (*psm_timer_reprogram)(hrtime_t) = NULL; 147 void (*psm_timer_enable)(void) = NULL; 148 void (*psm_timer_disable)(void) = NULL; 149 void (*psm_post_cyclic_setup)(void *arg) = NULL; 150 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 151 int *) = mach_intr_ops; 152 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) 153 return_instr; 154 155 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 156 void (*hrtime_tick)(void) = return_instr; 157 158 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo; 159 int (*psm_cpu_get_devinfo)(cpu_t *, dev_info_t **) = NULL; 160 161 /* 162 * True if the generic TSC code is our source of hrtime, rather than whatever 163 * the PSM can provide. 164 */ 165 #ifdef __xpv 166 int tsc_gethrtime_enable = 0; 167 #else 168 int tsc_gethrtime_enable = 1; 169 #endif 170 int tsc_gethrtime_initted = 0; 171 172 /* 173 * True if the hrtime implementation is "hires"; namely, better than microdata. 174 */ 175 int gethrtime_hires = 0; 176 177 /* 178 * Local Static Data 179 */ 180 static struct psm_ops mach_ops; 181 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 182 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 183 184 /* 185 * virtualization support for psm 186 */ 187 void *psm_vt_ops = NULL; 188 /* 189 * If non-zero, idle cpus will become "halted" when there's 190 * no work to do. 191 */ 192 int idle_cpu_use_hlt = 1; 193 194 #ifndef __xpv 195 /* 196 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 197 */ 198 int idle_cpu_prefer_mwait = 1; 199 /* 200 * Set to 0 to avoid MONITOR+CLFLUSH assertion. 201 */ 202 int idle_cpu_assert_cflush_monitor = 1; 203 204 /* 205 * If non-zero, idle cpus will not use power saving Deep C-States idle loop. 206 */ 207 int idle_cpu_no_deep_c = 0; 208 /* 209 * Non-power saving idle loop and wakeup pointers. 210 * Allows user to toggle Deep Idle power saving feature on/off. 211 */ 212 void (*non_deep_idle_cpu)() = cpu_idle; 213 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); 214 215 /* 216 * Object for the kernel to access the HPET. 217 */ 218 hpet_t hpet; 219 220 #endif /* ifndef __xpv */ 221 222 uint_t cp_haltset_fanout = 0; 223 224 /*ARGSUSED*/ 225 int 226 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 227 { 228 switch (hw) { 229 case PGHW_IPIPE: 230 if (x86_feature & (X86_HTT)) { 231 /* 232 * Hyper-threading is SMT 233 */ 234 return (1); 235 } else { 236 return (0); 237 } 238 case PGHW_PROCNODE: 239 if (cpuid_get_procnodes_per_pkg(cp) > 1) 240 return (1); 241 else 242 return (0); 243 case PGHW_CHIP: 244 if (x86_feature & (X86_CMP|X86_HTT)) 245 return (1); 246 else 247 return (0); 248 case PGHW_CACHE: 249 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 250 return (1); 251 else 252 return (0); 253 case PGHW_POW_ACTIVE: 254 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) 255 return (1); 256 else 257 return (0); 258 case PGHW_POW_IDLE: 259 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) 260 return (1); 261 else 262 return (0); 263 default: 264 return (0); 265 } 266 } 267 268 /* 269 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 270 * If pghw_type_t is an unsupported hardware type, then return -1 271 */ 272 int 273 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 274 { 275 id_t pgp_a, pgp_b; 276 277 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 278 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 279 280 if (pgp_a == -1 || pgp_b == -1) 281 return (-1); 282 283 return (pgp_a == pgp_b); 284 } 285 286 /* 287 * Return a physical instance identifier for known hardware sharing 288 * relationships 289 */ 290 id_t 291 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 292 { 293 switch (hw) { 294 case PGHW_IPIPE: 295 return (cpuid_get_coreid(cpu)); 296 case PGHW_CACHE: 297 return (cpuid_get_last_lvl_cacheid(cpu)); 298 case PGHW_PROCNODE: 299 return (cpuid_get_procnodeid(cpu)); 300 case PGHW_CHIP: 301 return (cpuid_get_chipid(cpu)); 302 case PGHW_POW_ACTIVE: 303 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); 304 case PGHW_POW_IDLE: 305 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); 306 default: 307 return (-1); 308 } 309 } 310 311 /* 312 * Express preference for optimizing for sharing relationship 313 * hw1 vs hw2 314 */ 315 pghw_type_t 316 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) 317 { 318 int i, rank1, rank2; 319 320 static pghw_type_t hw_hier[] = { 321 PGHW_IPIPE, 322 PGHW_CACHE, 323 PGHW_PROCNODE, 324 PGHW_CHIP, 325 PGHW_POW_IDLE, 326 PGHW_POW_ACTIVE, 327 PGHW_NUM_COMPONENTS 328 }; 329 330 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 331 if (hw_hier[i] == hw1) 332 rank1 = i; 333 if (hw_hier[i] == hw2) 334 rank2 = i; 335 } 336 337 if (rank1 > rank2) 338 return (hw1); 339 else 340 return (hw2); 341 } 342 343 /* 344 * Override the default CMT dispatcher policy for the specified 345 * hardware sharing relationship 346 */ 347 pg_cmt_policy_t 348 pg_plat_cmt_policy(pghw_type_t hw) 349 { 350 /* 351 * For shared caches, also load balance across them to 352 * maximize aggregate cache capacity 353 */ 354 switch (hw) { 355 case PGHW_CACHE: 356 return (CMT_BALANCE|CMT_AFFINITY); 357 default: 358 return (CMT_NO_POLICY); 359 } 360 } 361 362 id_t 363 pg_plat_get_core_id(cpu_t *cpu) 364 { 365 return ((id_t)cpuid_get_coreid(cpu)); 366 } 367 368 void 369 cmp_set_nosteal_interval(void) 370 { 371 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 372 nosteal_nsec = 100000UL; 373 } 374 375 /* 376 * Routine to ensure initial callers to hrtime gets 0 as return 377 */ 378 static hrtime_t 379 dummy_hrtime(void) 380 { 381 return (0); 382 } 383 384 /* ARGSUSED */ 385 static void 386 dummy_scalehrtime(hrtime_t *ticks) 387 {} 388 389 static uint64_t 390 dummy_unscalehrtime(hrtime_t nsecs) 391 { 392 return ((uint64_t)nsecs); 393 } 394 395 /* 396 * Supports Deep C-State power saving idle loop. 397 */ 398 void 399 cpu_idle_adaptive(void) 400 { 401 (*CPU->cpu_m.mcpu_idle_cpu)(); 402 } 403 404 /* 405 * Function called by CPU idle notification framework to check whether CPU 406 * has been awakened. It will be called with interrupt disabled. 407 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 408 * notification framework. 409 */ 410 /*ARGSUSED*/ 411 static void 412 cpu_idle_check_wakeup(void *arg) 413 { 414 /* 415 * Toggle interrupt flag to detect pending interrupts. 416 * If interrupt happened, do_interrupt() will notify CPU idle 417 * notification framework so no need to call cpu_idle_exit() here. 418 */ 419 sti(); 420 SMT_PAUSE(); 421 cli(); 422 } 423 424 /* 425 * Idle the present CPU until wakened via an interrupt 426 */ 427 void 428 cpu_idle(void) 429 { 430 cpu_t *cpup = CPU; 431 processorid_t cpu_sid = cpup->cpu_seqid; 432 cpupart_t *cp = cpup->cpu_part; 433 int hset_update = 1; 434 435 /* 436 * If this CPU is online, and there's multiple CPUs 437 * in the system, then we should notate our halting 438 * by adding ourselves to the partition's halted CPU 439 * bitmap. This allows other CPUs to find/awaken us when 440 * work becomes available. 441 */ 442 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 443 hset_update = 0; 444 445 /* 446 * Add ourselves to the partition's halted CPUs bitmap 447 * and set our HALTED flag, if necessary. 448 * 449 * When a thread becomes runnable, it is placed on the queue 450 * and then the halted CPU bitmap is checked to determine who 451 * (if anyone) should be awakened. We therefore need to first 452 * add ourselves to the bitmap, and and then check if there 453 * is any work available. The order is important to prevent a race 454 * that can lead to work languishing on a run queue somewhere while 455 * this CPU remains halted. 456 * 457 * Either the producing CPU will see we're halted and will awaken us, 458 * or this CPU will see the work available in disp_anywork(). 459 * 460 * Note that memory barriers after updating the HALTED flag 461 * are not necessary since an atomic operation (updating the bitset) 462 * immediately follows. On x86 the atomic operation acts as a 463 * memory barrier for the update of cpu_disp_flags. 464 */ 465 if (hset_update) { 466 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 467 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 468 } 469 470 /* 471 * Check to make sure there's really nothing to do. 472 * Work destined for this CPU may become available after 473 * this check. We'll be notified through the clearing of our 474 * bit in the halted CPU bitmap, and a poke. 475 */ 476 if (disp_anywork()) { 477 if (hset_update) { 478 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 479 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 480 } 481 return; 482 } 483 484 /* 485 * We're on our way to being halted. 486 * 487 * Disable interrupts now, so that we'll awaken immediately 488 * after halting if someone tries to poke us between now and 489 * the time we actually halt. 490 * 491 * We check for the presence of our bit after disabling interrupts. 492 * If it's cleared, we'll return. If the bit is cleared after 493 * we check then the poke will pop us out of the halted state. 494 * 495 * This means that the ordering of the poke and the clearing 496 * of the bit by cpu_wakeup is important. 497 * cpu_wakeup() must clear, then poke. 498 * cpu_idle() must disable interrupts, then check for the bit. 499 */ 500 cli(); 501 502 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { 503 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 504 sti(); 505 return; 506 } 507 508 /* 509 * The check for anything locally runnable is here for performance 510 * and isn't needed for correctness. disp_nrunnable ought to be 511 * in our cache still, so it's inexpensive to check, and if there 512 * is anything runnable we won't have to wait for the poke. 513 */ 514 if (cpup->cpu_disp->disp_nrunnable != 0) { 515 if (hset_update) { 516 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 517 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 518 } 519 sti(); 520 return; 521 } 522 523 if (cpu_idle_enter(IDLE_STATE_C1, 0, 524 cpu_idle_check_wakeup, NULL) == 0) { 525 mach_cpu_idle(); 526 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 527 } 528 529 /* 530 * We're no longer halted 531 */ 532 if (hset_update) { 533 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 534 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 535 } 536 } 537 538 539 /* 540 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 541 * Otherwise, see if other CPUs in the cpu partition are halted and need to 542 * be woken up so that they can steal the thread we placed on this CPU. 543 * This function is only used on MP systems. 544 */ 545 static void 546 cpu_wakeup(cpu_t *cpu, int bound) 547 { 548 uint_t cpu_found; 549 processorid_t cpu_sid; 550 cpupart_t *cp; 551 552 cp = cpu->cpu_part; 553 cpu_sid = cpu->cpu_seqid; 554 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 555 /* 556 * Clear the halted bit for that CPU since it will be 557 * poked in a moment. 558 */ 559 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 560 /* 561 * We may find the current CPU present in the halted cpuset 562 * if we're in the context of an interrupt that occurred 563 * before we had a chance to clear our bit in cpu_idle(). 564 * Poking ourself is obviously unnecessary, since if 565 * we're here, we're not halted. 566 */ 567 if (cpu != CPU) 568 poke_cpu(cpu->cpu_id); 569 return; 570 } else { 571 /* 572 * This cpu isn't halted, but it's idle or undergoing a 573 * context switch. No need to awaken anyone else. 574 */ 575 if (cpu->cpu_thread == cpu->cpu_idle_thread || 576 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 577 return; 578 } 579 580 /* 581 * No need to wake up other CPUs if this is for a bound thread. 582 */ 583 if (bound) 584 return; 585 586 /* 587 * The CPU specified for wakeup isn't currently halted, so check 588 * to see if there are any other halted CPUs in the partition, 589 * and if there are then awaken one. 590 */ 591 do { 592 cpu_found = bitset_find(&cp->cp_haltset); 593 if (cpu_found == (uint_t)-1) 594 return; 595 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 596 597 if (cpu_found != CPU->cpu_seqid) { 598 poke_cpu(cpu_seq[cpu_found]->cpu_id); 599 } 600 } 601 602 #ifndef __xpv 603 /* 604 * Function called by CPU idle notification framework to check whether CPU 605 * has been awakened. It will be called with interrupt disabled. 606 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 607 * notification framework. 608 */ 609 static void 610 cpu_idle_mwait_check_wakeup(void *arg) 611 { 612 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; 613 614 ASSERT(arg != NULL); 615 if (*mcpu_mwait != MWAIT_HALTED) { 616 /* 617 * CPU has been awakened, notify CPU idle notification system. 618 */ 619 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 620 } else { 621 /* 622 * Toggle interrupt flag to detect pending interrupts. 623 * If interrupt happened, do_interrupt() will notify CPU idle 624 * notification framework so no need to call cpu_idle_exit() 625 * here. 626 */ 627 sti(); 628 SMT_PAUSE(); 629 cli(); 630 } 631 } 632 633 /* 634 * Idle the present CPU until awakened via touching its monitored line 635 */ 636 void 637 cpu_idle_mwait(void) 638 { 639 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 640 cpu_t *cpup = CPU; 641 processorid_t cpu_sid = cpup->cpu_seqid; 642 cpupart_t *cp = cpup->cpu_part; 643 int hset_update = 1; 644 645 /* 646 * Set our mcpu_mwait here, so we can tell if anyone tries to 647 * wake us between now and when we call mwait. No other cpu will 648 * attempt to set our mcpu_mwait until we add ourself to the halted 649 * CPU bitmap. 650 */ 651 *mcpu_mwait = MWAIT_HALTED; 652 653 /* 654 * If this CPU is online, and there's multiple CPUs 655 * in the system, then we should note our halting 656 * by adding ourselves to the partition's halted CPU 657 * bitmap. This allows other CPUs to find/awaken us when 658 * work becomes available. 659 */ 660 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 661 hset_update = 0; 662 663 /* 664 * Add ourselves to the partition's halted CPUs bitmap 665 * and set our HALTED flag, if necessary. 666 * 667 * When a thread becomes runnable, it is placed on the queue 668 * and then the halted CPU bitmap is checked to determine who 669 * (if anyone) should be awakened. We therefore need to first 670 * add ourselves to the bitmap, and and then check if there 671 * is any work available. 672 * 673 * Note that memory barriers after updating the HALTED flag 674 * are not necessary since an atomic operation (updating the bitmap) 675 * immediately follows. On x86 the atomic operation acts as a 676 * memory barrier for the update of cpu_disp_flags. 677 */ 678 if (hset_update) { 679 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 680 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 681 } 682 683 /* 684 * Check to make sure there's really nothing to do. 685 * Work destined for this CPU may become available after 686 * this check. We'll be notified through the clearing of our 687 * bit in the halted CPU bitmap, and a write to our mcpu_mwait. 688 * 689 * disp_anywork() checks disp_nrunnable, so we do not have to later. 690 */ 691 if (disp_anywork()) { 692 if (hset_update) { 693 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 694 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 695 } 696 return; 697 } 698 699 /* 700 * We're on our way to being halted. 701 * To avoid a lost wakeup, arm the monitor before checking if another 702 * cpu wrote to mcpu_mwait to wake us up. 703 */ 704 i86_monitor(mcpu_mwait, 0, 0); 705 if (*mcpu_mwait == MWAIT_HALTED) { 706 if (cpu_idle_enter(IDLE_STATE_C1, 0, 707 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { 708 if (*mcpu_mwait == MWAIT_HALTED) { 709 i86_mwait(0, 0); 710 } 711 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 712 } 713 } 714 715 /* 716 * We're no longer halted 717 */ 718 if (hset_update) { 719 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 720 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 721 } 722 } 723 724 /* 725 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 726 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 727 * need to be woken up so that they can steal the thread we placed on this CPU. 728 * This function is only used on MP systems. 729 */ 730 static void 731 cpu_wakeup_mwait(cpu_t *cp, int bound) 732 { 733 cpupart_t *cpu_part; 734 uint_t cpu_found; 735 processorid_t cpu_sid; 736 737 cpu_part = cp->cpu_part; 738 cpu_sid = cp->cpu_seqid; 739 740 /* 741 * Clear the halted bit for that CPU since it will be woken up 742 * in a moment. 743 */ 744 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { 745 /* 746 * Clear the halted bit for that CPU since it will be 747 * poked in a moment. 748 */ 749 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); 750 /* 751 * We may find the current CPU present in the halted cpuset 752 * if we're in the context of an interrupt that occurred 753 * before we had a chance to clear our bit in cpu_idle(). 754 * Waking ourself is obviously unnecessary, since if 755 * we're here, we're not halted. 756 * 757 * monitor/mwait wakeup via writing to our cache line is 758 * harmless and less expensive than always checking if we 759 * are waking ourself which is an uncommon case. 760 */ 761 MWAIT_WAKEUP(cp); /* write to monitored line */ 762 return; 763 } else { 764 /* 765 * This cpu isn't halted, but it's idle or undergoing a 766 * context switch. No need to awaken anyone else. 767 */ 768 if (cp->cpu_thread == cp->cpu_idle_thread || 769 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 770 return; 771 } 772 773 /* 774 * No need to wake up other CPUs if the thread we just enqueued 775 * is bound. 776 */ 777 if (bound || ncpus == 1) 778 return; 779 780 /* 781 * See if there's any other halted CPUs. If there are, then 782 * select one, and awaken it. 783 * It's possible that after we find a CPU, somebody else 784 * will awaken it before we get the chance. 785 * In that case, look again. 786 */ 787 do { 788 cpu_found = bitset_find(&cpu_part->cp_haltset); 789 if (cpu_found == (uint_t)-1) 790 return; 791 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, 792 cpu_found) < 0); 793 794 /* 795 * Do not check if cpu_found is ourself as monitor/mwait 796 * wakeup is cheap. 797 */ 798 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ 799 } 800 801 #endif 802 803 void (*cpu_pause_handler)(volatile char *) = NULL; 804 805 static int 806 mp_disable_intr(int cpun) 807 { 808 /* 809 * switch to the offline cpu 810 */ 811 affinity_set(cpun); 812 /* 813 * raise ipl to just below cross call 814 */ 815 splx(XC_SYS_PIL - 1); 816 /* 817 * set base spl to prevent the next swtch to idle from 818 * lowering back to ipl 0 819 */ 820 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); 821 set_base_spl(); 822 affinity_clear(); 823 return (DDI_SUCCESS); 824 } 825 826 static void 827 mp_enable_intr(int cpun) 828 { 829 /* 830 * switch to the online cpu 831 */ 832 affinity_set(cpun); 833 /* 834 * clear the interrupt active mask 835 */ 836 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); 837 set_base_spl(); 838 (void) spl0(); 839 affinity_clear(); 840 } 841 842 static void 843 mach_get_platform(int owner) 844 { 845 void **srv_opsp; 846 void **clt_opsp; 847 int i; 848 int total_ops; 849 850 /* fix up psm ops */ 851 srv_opsp = (void **)mach_set[0]; 852 clt_opsp = (void **)mach_set[owner]; 853 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 854 total_ops = sizeof (struct psm_ops_ver01) / 855 sizeof (void (*)(void)); 856 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 857 /* no psm_notify_func */ 858 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 859 sizeof (void (*)(void)); 860 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 861 /* no psm_timer funcs */ 862 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 863 sizeof (void (*)(void)); 864 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 865 /* no psm_preshutdown function */ 866 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 867 sizeof (void (*)(void)); 868 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 869 /* no psm_intr_ops function */ 870 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 871 sizeof (void (*)(void)); 872 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_5) 873 /* no psm_state function */ 874 total_ops = OFFSETOF(struct psm_ops, psm_state) / 875 sizeof (void (*)(void)); 876 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_6) 877 /* no psm_cpu_ops function */ 878 total_ops = OFFSETOF(struct psm_ops, psm_cpu_ops) / 879 sizeof (void (*)(void)); 880 else 881 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 882 883 /* 884 * Save the version of the PSM module, in case we need to 885 * behave differently based on version. 886 */ 887 mach_ver[0] = mach_ver[owner]; 888 889 for (i = 0; i < total_ops; i++) 890 if (clt_opsp[i] != NULL) 891 srv_opsp[i] = clt_opsp[i]; 892 } 893 894 static void 895 mach_construct_info() 896 { 897 struct psm_sw *swp; 898 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 899 int conflict_owner = 0; 900 901 if (psmsw->psw_forw == psmsw) 902 panic("No valid PSM modules found"); 903 mutex_enter(&psmsw_lock); 904 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 905 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 906 continue; 907 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 908 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 909 mach_cnt[swp->psw_infop->p_owner]++; 910 } 911 mutex_exit(&psmsw_lock); 912 913 mach_get_platform(PSM_OWN_SYS_DEFAULT); 914 915 /* check to see are there any conflicts */ 916 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 917 conflict_owner = PSM_OWN_EXCLUSIVE; 918 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 919 conflict_owner = PSM_OWN_OVERRIDE; 920 if (conflict_owner) { 921 /* remove all psm modules except uppc */ 922 cmn_err(CE_WARN, 923 "Conflicts detected on the following PSM modules:"); 924 mutex_enter(&psmsw_lock); 925 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 926 if (swp->psw_infop->p_owner == conflict_owner) 927 cmn_err(CE_WARN, "%s ", 928 swp->psw_infop->p_mach_idstring); 929 } 930 mutex_exit(&psmsw_lock); 931 cmn_err(CE_WARN, 932 "Setting the system back to SINGLE processor mode!"); 933 cmn_err(CE_WARN, 934 "Please edit /etc/mach to remove the invalid PSM module."); 935 return; 936 } 937 938 if (mach_set[PSM_OWN_EXCLUSIVE]) 939 mach_get_platform(PSM_OWN_EXCLUSIVE); 940 941 if (mach_set[PSM_OWN_OVERRIDE]) 942 mach_get_platform(PSM_OWN_OVERRIDE); 943 } 944 945 static void 946 mach_init() 947 { 948 struct psm_ops *pops; 949 950 mach_construct_info(); 951 952 pops = mach_set[0]; 953 954 /* register the interrupt and clock initialization rotuines */ 955 picinitf = mach_picinit; 956 clkinitf = mach_clkinit; 957 psm_get_clockirq = pops->psm_get_clockirq; 958 959 /* register the interrupt setup code */ 960 slvltovect = mach_softlvl_to_vect; 961 addspl = pops->psm_addspl; 962 delspl = pops->psm_delspl; 963 964 if (pops->psm_translate_irq) 965 psm_translate_irq = pops->psm_translate_irq; 966 if (pops->psm_intr_ops) 967 psm_intr_ops = pops->psm_intr_ops; 968 969 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 970 /* 971 * Time-of-day functionality now handled in TOD modules. 972 * (Warn about PSM modules that think that we're going to use 973 * their ops vectors.) 974 */ 975 if (pops->psm_tod_get) 976 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 977 (void *)pops->psm_tod_get); 978 979 if (pops->psm_tod_set) 980 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 981 (void *)pops->psm_tod_set); 982 #endif 983 984 if (pops->psm_notify_error) { 985 psm_notify_error = mach_notify_error; 986 notify_error = pops->psm_notify_error; 987 } 988 989 (*pops->psm_softinit)(); 990 991 /* 992 * Initialize the dispatcher's function hooks to enable CPU halting 993 * when idle. Set both the deep-idle and non-deep-idle hooks. 994 * 995 * Assume we can use power saving deep-idle loop cpu_idle_adaptive. 996 * Platform deep-idle driver will reset our idle loop to 997 * non_deep_idle_cpu if power saving deep-idle feature is not available. 998 * 999 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) 1000 * or idle_cpu_prefer_mwait is not set. 1001 * Allocate monitor/mwait buffer for cpu0. 1002 */ 1003 #ifndef __xpv 1004 non_deep_idle_disp_enq_thread = disp_enq_thread; 1005 #endif 1006 if (idle_cpu_use_hlt) { 1007 idle_cpu = cpu_idle_adaptive; 1008 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1009 #ifndef __xpv 1010 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) { 1011 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); 1012 /* 1013 * Protect ourself from insane mwait size. 1014 */ 1015 if (CPU->cpu_m.mcpu_mwait == NULL) { 1016 #ifdef DEBUG 1017 cmn_err(CE_NOTE, "Using hlt idle. Cannot " 1018 "handle cpu 0 mwait size."); 1019 #endif 1020 idle_cpu_prefer_mwait = 0; 1021 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1022 } else { 1023 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; 1024 } 1025 } else { 1026 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1027 } 1028 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; 1029 1030 /* 1031 * Disable power saving deep idle loop? 1032 */ 1033 if (idle_cpu_no_deep_c) { 1034 idle_cpu = non_deep_idle_cpu; 1035 } 1036 #endif 1037 } 1038 1039 mach_smpinit(); 1040 } 1041 1042 static void 1043 mach_smpinit(void) 1044 { 1045 struct psm_ops *pops; 1046 processorid_t cpu_id; 1047 int cnt; 1048 cpuset_t cpumask; 1049 1050 pops = mach_set[0]; 1051 CPUSET_ZERO(cpumask); 1052 1053 cpu_id = -1; 1054 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1055 /* 1056 * Only add boot_ncpus CPUs to mp_cpus. Other CPUs will be handled 1057 * by CPU DR driver at runtime. 1058 */ 1059 for (cnt = 0; cpu_id != -1 && cnt < boot_ncpus; cnt++) { 1060 CPUSET_ADD(cpumask, cpu_id); 1061 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1062 } 1063 1064 mp_cpus = cpumask; 1065 1066 /* MP related routines */ 1067 ap_mlsetup = pops->psm_post_cpu_start; 1068 send_dirintf = pops->psm_send_ipi; 1069 1070 /* optional MP related routines */ 1071 if (pops->psm_shutdown) 1072 psm_shutdownf = pops->psm_shutdown; 1073 if (pops->psm_preshutdown) 1074 psm_preshutdownf = pops->psm_preshutdown; 1075 if (pops->psm_notify_func) 1076 psm_notifyf = pops->psm_notify_func; 1077 if (pops->psm_set_idlecpu) 1078 psm_set_idle_cpuf = pops->psm_set_idlecpu; 1079 if (pops->psm_unset_idlecpu) 1080 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 1081 1082 psm_clkinit = pops->psm_clkinit; 1083 1084 if (pops->psm_timer_reprogram) 1085 psm_timer_reprogram = pops->psm_timer_reprogram; 1086 1087 if (pops->psm_timer_enable) 1088 psm_timer_enable = pops->psm_timer_enable; 1089 1090 if (pops->psm_timer_disable) 1091 psm_timer_disable = pops->psm_timer_disable; 1092 1093 if (pops->psm_post_cyclic_setup) 1094 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 1095 1096 if (pops->psm_state) 1097 psm_state = pops->psm_state; 1098 1099 /* 1100 * Set these vectors here so they can be used by Suspend/Resume 1101 * on UP machines. 1102 */ 1103 if (pops->psm_disable_intr) 1104 psm_disable_intr = pops->psm_disable_intr; 1105 if (pops->psm_enable_intr) 1106 psm_enable_intr = pops->psm_enable_intr; 1107 1108 /* check for multiple CPUs */ 1109 if (cnt < 2 && plat_dr_support_cpu() == B_FALSE) 1110 return; 1111 1112 /* check for MP platforms */ 1113 if (pops->psm_cpu_start == NULL) 1114 return; 1115 1116 /* 1117 * Set the dispatcher hook to enable cpu "wake up" 1118 * when a thread becomes runnable. 1119 */ 1120 if (idle_cpu_use_hlt) { 1121 disp_enq_thread = cpu_wakeup; 1122 #ifndef __xpv 1123 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) 1124 disp_enq_thread = cpu_wakeup_mwait; 1125 non_deep_idle_disp_enq_thread = disp_enq_thread; 1126 #endif 1127 } 1128 1129 psm_get_ipivect = pops->psm_get_ipivect; 1130 1131 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", 1132 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 1133 NULL, NULL, NULL, NULL); 1134 1135 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 1136 } 1137 1138 static void 1139 mach_picinit() 1140 { 1141 struct psm_ops *pops; 1142 1143 pops = mach_set[0]; 1144 1145 /* register the interrupt handlers */ 1146 setlvl = pops->psm_intr_enter; 1147 setlvlx = pops->psm_intr_exit; 1148 1149 /* initialize the interrupt hardware */ 1150 (*pops->psm_picinit)(); 1151 1152 /* set interrupt mask for current ipl */ 1153 setspl = pops->psm_setspl; 1154 cli(); 1155 setspl(CPU->cpu_pri); 1156 } 1157 1158 uint_t cpu_freq; /* MHz */ 1159 uint64_t cpu_freq_hz; /* measured (in hertz) */ 1160 1161 #define MEGA_HZ 1000000 1162 1163 #ifdef __xpv 1164 1165 int xpv_cpufreq_workaround = 1; 1166 int xpv_cpufreq_verbose = 0; 1167 1168 #else /* __xpv */ 1169 1170 static uint64_t 1171 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 1172 { 1173 uint64_t cpu_hz; 1174 1175 if ((pit_counter == 0) || (*processor_clks == 0) || 1176 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 1177 return (0); 1178 1179 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 1180 1181 return (cpu_hz); 1182 } 1183 1184 #endif /* __xpv */ 1185 1186 static uint64_t 1187 mach_getcpufreq(void) 1188 { 1189 #if defined(__xpv) 1190 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; 1191 uint64_t cpu_hz; 1192 1193 /* 1194 * During dom0 bringup, it was noted that on at least one older 1195 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul 1196 * value that is quite wrong (the 3.06GHz clock was reported 1197 * as 4.77GHz) 1198 * 1199 * The curious thing is, that if you stop the kernel at entry, 1200 * breakpoint here and inspect the value with kmdb, the value 1201 * is correct - but if you don't stop and simply enable the 1202 * printf statement (below), you can see the bad value printed 1203 * here. Almost as if something kmdb did caused the hypervisor to 1204 * figure it out correctly. And, note that the hypervisor 1205 * eventually -does- figure it out correctly ... if you look at 1206 * the field later in the life of dom0, it is correct. 1207 * 1208 * For now, on dom0, we employ a slightly cheesy workaround of 1209 * using the DOM0_PHYSINFO hypercall. 1210 */ 1211 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { 1212 cpu_hz = 1000 * xpv_cpu_khz(); 1213 } else { 1214 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; 1215 1216 if (vti->tsc_shift < 0) 1217 cpu_hz <<= -vti->tsc_shift; 1218 else 1219 cpu_hz >>= vti->tsc_shift; 1220 } 1221 1222 if (xpv_cpufreq_verbose) 1223 printf("mach_getcpufreq: system_mul 0x%x, shift %d, " 1224 "cpu_hz %" PRId64 "Hz\n", 1225 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); 1226 1227 return (cpu_hz); 1228 #else /* __xpv */ 1229 uint32_t pit_counter; 1230 uint64_t processor_clks; 1231 1232 if (x86_feature & X86_TSC) { 1233 /* 1234 * We have a TSC. freq_tsc() knows how to measure the number 1235 * of clock cycles sampled against the PIT. 1236 */ 1237 ulong_t flags = clear_int_flag(); 1238 processor_clks = freq_tsc(&pit_counter); 1239 restore_int_flag(flags); 1240 return (mach_calchz(pit_counter, &processor_clks)); 1241 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1242 #if defined(__amd64) 1243 panic("mach_getcpufreq: no TSC!"); 1244 #elif defined(__i386) 1245 /* 1246 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1247 * for which freq_notsc() knows how to measure the number of 1248 * elapsed clock cycles sampled against the PIT 1249 */ 1250 ulong_t flags = clear_int_flag(); 1251 processor_clks = freq_notsc(&pit_counter); 1252 restore_int_flag(flags); 1253 return (mach_calchz(pit_counter, &processor_clks)); 1254 #endif /* __i386 */ 1255 } 1256 1257 /* We do not know how to calculate cpu frequency for this cpu. */ 1258 return (0); 1259 #endif /* __xpv */ 1260 } 1261 1262 /* 1263 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1264 * to this array, instead improve the accuracy of the algorithm that determines 1265 * the clock speed of the processor or extend the implementation to support the 1266 * vendor as appropriate. This is here only to support adjusting the speed on 1267 * older slower processors that mach_fixcpufreq() would not be able to account 1268 * for otherwise. 1269 */ 1270 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1271 1272 /* 1273 * On fast processors the clock frequency that is measured may be off by 1274 * a few MHz from the value printed on the part. This is a combination of 1275 * the factors that for such fast parts being off by this much is within 1276 * the tolerances for manufacture and because of the difficulties in the 1277 * measurement that can lead to small error. This function uses some 1278 * heuristics in order to tweak the value that was measured to match what 1279 * is most likely printed on the part. 1280 * 1281 * Some examples: 1282 * AMD Athlon 1000 mhz measured as 998 mhz 1283 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1284 * Intel Pentium IV 1500 mhz measured as 1495mhz 1285 * 1286 * If in the future this function is no longer sufficient to correct 1287 * for the error in the measurement, then the algorithm used to perform 1288 * the measurement will have to be improved in order to increase accuracy 1289 * rather than adding horrible and questionable kludges here. 1290 * 1291 * This is called after the cyclics subsystem because of the potential 1292 * that the heuristics within may give a worse estimate of the clock 1293 * frequency than the value that was measured. 1294 */ 1295 static void 1296 mach_fixcpufreq(void) 1297 { 1298 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1299 1300 freq = (uint32_t)cpu_freq; 1301 1302 /* 1303 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1304 * measured speed taking into account that the 667 MHz parts were 1305 * the first to round-up. 1306 */ 1307 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1308 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1309 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1310 1311 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1312 mul = (freq + 25) / 50; 1313 near50 = mul * 50; 1314 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1315 1316 /* Find the closer of the two */ 1317 if (delta66 < delta50) { 1318 fixed = near66; 1319 delta = delta66; 1320 } else { 1321 fixed = near50; 1322 delta = delta50; 1323 } 1324 1325 if (fixed > INT_MAX) 1326 return; 1327 1328 /* 1329 * Some older parts have a core clock frequency that is not an 1330 * integral multiple of 50 or 66 MHz. Check if one of the old 1331 * clock frequencies is closer to the measured value than any 1332 * of the integral multiples of 50 an 66, and if so set fixed 1333 * and delta appropriately to represent the closest value. 1334 */ 1335 i = sizeof (x86_cpu_freq) / sizeof (int); 1336 while (i > 0) { 1337 i--; 1338 1339 if (x86_cpu_freq[i] <= freq) { 1340 mul = freq - x86_cpu_freq[i]; 1341 1342 if (mul < delta) { 1343 fixed = x86_cpu_freq[i]; 1344 delta = mul; 1345 } 1346 1347 break; 1348 } 1349 1350 mul = x86_cpu_freq[i] - freq; 1351 1352 if (mul < delta) { 1353 fixed = x86_cpu_freq[i]; 1354 delta = mul; 1355 } 1356 } 1357 1358 /* 1359 * Set a reasonable maximum for how much to correct the measured 1360 * result by. This check is here to prevent the adjustment made 1361 * by this function from being more harm than good. It is entirely 1362 * possible that in the future parts will be made that are not 1363 * integral multiples of 66 or 50 in clock frequency or that 1364 * someone may overclock a part to some odd frequency. If the 1365 * measured value is farther from the corrected value than 1366 * allowed, then assume the corrected value is in error and use 1367 * the measured value. 1368 */ 1369 if (6 < delta) 1370 return; 1371 1372 cpu_freq = (int)fixed; 1373 } 1374 1375 1376 static int 1377 machhztomhz(uint64_t cpu_freq_hz) 1378 { 1379 uint64_t cpu_mhz; 1380 1381 /* Round to nearest MHZ */ 1382 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1383 1384 if (cpu_mhz > INT_MAX) 1385 return (0); 1386 1387 return ((int)cpu_mhz); 1388 1389 } 1390 1391 1392 static int 1393 mach_clkinit(int preferred_mode, int *set_mode) 1394 { 1395 struct psm_ops *pops; 1396 int resolution; 1397 1398 pops = mach_set[0]; 1399 1400 cpu_freq_hz = mach_getcpufreq(); 1401 1402 cpu_freq = machhztomhz(cpu_freq_hz); 1403 1404 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 1405 tsc_gethrtime_enable = 0; 1406 1407 #ifndef __xpv 1408 if (tsc_gethrtime_enable) { 1409 tsc_hrtimeinit(cpu_freq_hz); 1410 } else 1411 #endif 1412 { 1413 if (pops->psm_hrtimeinit) 1414 (*pops->psm_hrtimeinit)(); 1415 gethrtimef = pops->psm_gethrtime; 1416 gethrtimeunscaledf = gethrtimef; 1417 /* scalehrtimef will remain dummy */ 1418 } 1419 1420 mach_fixcpufreq(); 1421 1422 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1423 if (preferred_mode == TIMER_ONESHOT) { 1424 1425 resolution = (*pops->psm_clkinit)(0); 1426 if (resolution != 0) { 1427 *set_mode = TIMER_ONESHOT; 1428 return (resolution); 1429 } 1430 } 1431 1432 /* 1433 * either periodic mode was requested or could not set to 1434 * one-shot mode 1435 */ 1436 resolution = (*pops->psm_clkinit)(hz); 1437 /* 1438 * psm should be able to do periodic, so we do not check 1439 * for return value of psm_clkinit here. 1440 */ 1441 *set_mode = TIMER_PERIODIC; 1442 return (resolution); 1443 } else { 1444 /* 1445 * PSMI interface prior to PSMI_3 does not define a return 1446 * value for psm_clkinit, so the return value is ignored. 1447 */ 1448 (void) (*pops->psm_clkinit)(hz); 1449 *set_mode = TIMER_PERIODIC; 1450 return (nsec_per_tick); 1451 } 1452 } 1453 1454 1455 /*ARGSUSED*/ 1456 static int 1457 mach_softlvl_to_vect(int ipl) 1458 { 1459 setsoftint = av_set_softint_pending; 1460 kdisetsoftint = kdi_av_set_softint_pending; 1461 1462 return (PSM_SV_SOFTWARE); 1463 } 1464 1465 #ifdef DEBUG 1466 /* 1467 * This is here to allow us to simulate cpus that refuse to start. 1468 */ 1469 cpuset_t cpufailset; 1470 #endif 1471 1472 int 1473 mach_cpu_start(struct cpu *cp, void *ctx) 1474 { 1475 struct psm_ops *pops = mach_set[0]; 1476 processorid_t id = cp->cpu_id; 1477 1478 #ifdef DEBUG 1479 if (CPU_IN_SET(cpufailset, id)) 1480 return (0); 1481 #endif 1482 return ((*pops->psm_cpu_start)(id, ctx)); 1483 } 1484 1485 int 1486 mach_cpuid_start(processorid_t id, void *ctx) 1487 { 1488 struct psm_ops *pops = mach_set[0]; 1489 1490 #ifdef DEBUG 1491 if (CPU_IN_SET(cpufailset, id)) 1492 return (0); 1493 #endif 1494 return ((*pops->psm_cpu_start)(id, ctx)); 1495 } 1496 1497 int 1498 mach_cpu_stop(cpu_t *cp, void *ctx) 1499 { 1500 struct psm_ops *pops = mach_set[0]; 1501 psm_cpu_request_t request; 1502 1503 if (pops->psm_cpu_ops == NULL) { 1504 return (ENOTSUP); 1505 } 1506 1507 ASSERT(cp->cpu_id != -1); 1508 request.pcr_cmd = PSM_CPU_STOP; 1509 request.req.cpu_stop.cpuid = cp->cpu_id; 1510 request.req.cpu_stop.ctx = ctx; 1511 1512 return ((*pops->psm_cpu_ops)(&request)); 1513 } 1514 1515 int 1516 mach_cpu_add(mach_cpu_add_arg_t *argp, processorid_t *cpuidp) 1517 { 1518 int rc; 1519 struct psm_ops *pops = mach_set[0]; 1520 psm_cpu_request_t request; 1521 1522 if (pops->psm_cpu_ops == NULL) { 1523 return (ENOTSUP); 1524 } 1525 1526 request.pcr_cmd = PSM_CPU_ADD; 1527 request.req.cpu_add.argp = argp; 1528 request.req.cpu_add.cpuid = -1; 1529 rc = (*pops->psm_cpu_ops)(&request); 1530 if (rc == 0) { 1531 ASSERT(request.req.cpu_add.cpuid != -1); 1532 *cpuidp = request.req.cpu_add.cpuid; 1533 } 1534 1535 return (rc); 1536 } 1537 1538 int 1539 mach_cpu_remove(processorid_t cpuid) 1540 { 1541 struct psm_ops *pops = mach_set[0]; 1542 psm_cpu_request_t request; 1543 1544 if (pops->psm_cpu_ops == NULL) { 1545 return (ENOTSUP); 1546 } 1547 1548 request.pcr_cmd = PSM_CPU_REMOVE; 1549 request.req.cpu_remove.cpuid = cpuid; 1550 1551 return ((*pops->psm_cpu_ops)(&request)); 1552 } 1553 1554 /* 1555 * Default handler to create device node for CPU. 1556 * One reference count will be held on created device node. 1557 */ 1558 static int 1559 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp) 1560 { 1561 int rv, circ; 1562 dev_info_t *dip; 1563 static kmutex_t cpu_node_lock; 1564 static dev_info_t *cpu_nex_devi = NULL; 1565 1566 ASSERT(cp != NULL); 1567 ASSERT(dipp != NULL); 1568 *dipp = NULL; 1569 1570 if (cpu_nex_devi == NULL) { 1571 mutex_enter(&cpu_node_lock); 1572 /* First check whether cpus exists. */ 1573 cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0); 1574 /* Create cpus if it doesn't exist. */ 1575 if (cpu_nex_devi == NULL) { 1576 ndi_devi_enter(ddi_root_node(), &circ); 1577 rv = ndi_devi_alloc(ddi_root_node(), "cpus", 1578 (pnode_t)DEVI_SID_NODEID, &dip); 1579 if (rv != NDI_SUCCESS) { 1580 mutex_exit(&cpu_node_lock); 1581 cmn_err(CE_CONT, 1582 "?failed to create cpu nexus device.\n"); 1583 return (PSM_FAILURE); 1584 } 1585 ASSERT(dip != NULL); 1586 (void) ndi_devi_online(dip, 0); 1587 ndi_devi_exit(ddi_root_node(), circ); 1588 cpu_nex_devi = dip; 1589 } 1590 mutex_exit(&cpu_node_lock); 1591 } 1592 1593 /* 1594 * create a child node for cpu identified as 'cpu_id' 1595 */ 1596 ndi_devi_enter(cpu_nex_devi, &circ); 1597 dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1); 1598 if (dip == NULL) { 1599 cmn_err(CE_CONT, 1600 "?failed to create device node for cpu%d.\n", cp->cpu_id); 1601 rv = PSM_FAILURE; 1602 } else { 1603 *dipp = dip; 1604 (void) ndi_hold_devi(dip); 1605 rv = PSM_SUCCESS; 1606 } 1607 ndi_devi_exit(cpu_nex_devi, circ); 1608 1609 return (rv); 1610 } 1611 1612 /* 1613 * Create cpu device node in device tree and online it. 1614 * Return created dip with reference count held if requested. 1615 */ 1616 int 1617 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp) 1618 { 1619 int rv; 1620 dev_info_t *dip = NULL; 1621 1622 ASSERT(psm_cpu_create_devinfo != NULL); 1623 rv = psm_cpu_create_devinfo(cp, &dip); 1624 if (rv == PSM_SUCCESS) { 1625 cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi); 1626 /* Recursively attach driver for parent nexus device. */ 1627 if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) == 1628 DDI_SUCCESS) { 1629 /* Configure cpu itself and descendants. */ 1630 (void) ndi_devi_online(dip, 1631 NDI_ONLINE_ATTACH | NDI_CONFIG); 1632 } 1633 if (dipp != NULL) { 1634 *dipp = dip; 1635 } else { 1636 (void) ndi_rele_devi(dip); 1637 } 1638 } 1639 1640 return (rv); 1641 } 1642 1643 /* 1644 * The dipp contains one of following values on return: 1645 * - NULL if no device node found 1646 * - pointer to device node if found 1647 */ 1648 int 1649 mach_cpu_get_device_node(struct cpu *cp, dev_info_t **dipp) 1650 { 1651 *dipp = NULL; 1652 if (psm_cpu_get_devinfo != NULL) { 1653 if (psm_cpu_get_devinfo(cp, dipp) == PSM_SUCCESS) { 1654 return (PSM_SUCCESS); 1655 } 1656 } 1657 1658 return (PSM_FAILURE); 1659 } 1660 1661 /*ARGSUSED*/ 1662 static int 1663 mach_translate_irq(dev_info_t *dip, int irqno) 1664 { 1665 return (irqno); /* default to NO translation */ 1666 } 1667 1668 static void 1669 mach_notify_error(int level, char *errmsg) 1670 { 1671 /* 1672 * SL_FATAL is pass in once panicstr is set, deliver it 1673 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1674 * codes for the psmi handler 1675 */ 1676 if (level & SL_FATAL) 1677 (*notify_error)(CE_PANIC, errmsg); 1678 else if (level & SL_WARN) 1679 (*notify_error)(CE_WARN, errmsg); 1680 else if (level & SL_NOTE) 1681 (*notify_error)(CE_NOTE, errmsg); 1682 else if (level & SL_CONSOLE) 1683 (*notify_error)(CE_CONT, errmsg); 1684 } 1685 1686 /* 1687 * It provides the default basic intr_ops interface for the new DDI 1688 * interrupt framework if the PSM doesn't have one. 1689 * 1690 * Input: 1691 * dip - pointer to the dev_info structure of the requested device 1692 * hdlp - pointer to the internal interrupt handle structure for the 1693 * requested interrupt 1694 * intr_op - opcode for this call 1695 * result - pointer to the integer that will hold the result to be 1696 * passed back if return value is PSM_SUCCESS 1697 * 1698 * Output: 1699 * return value is either PSM_SUCCESS or PSM_FAILURE 1700 */ 1701 static int 1702 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1703 psm_intr_op_t intr_op, int *result) 1704 { 1705 struct intrspec *ispec; 1706 1707 switch (intr_op) { 1708 case PSM_INTR_OP_CHECK_MSI: 1709 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1710 DDI_INTR_TYPE_MSIX); 1711 break; 1712 case PSM_INTR_OP_ALLOC_VECTORS: 1713 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1714 *result = 1; 1715 else 1716 *result = 0; 1717 break; 1718 case PSM_INTR_OP_FREE_VECTORS: 1719 break; 1720 case PSM_INTR_OP_NAVAIL_VECTORS: 1721 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1722 *result = 1; 1723 else 1724 *result = 0; 1725 break; 1726 case PSM_INTR_OP_XLATE_VECTOR: 1727 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1728 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1729 break; 1730 case PSM_INTR_OP_GET_CAP: 1731 *result = 0; 1732 break; 1733 case PSM_INTR_OP_GET_PENDING: 1734 case PSM_INTR_OP_CLEAR_MASK: 1735 case PSM_INTR_OP_SET_MASK: 1736 case PSM_INTR_OP_GET_SHARED: 1737 case PSM_INTR_OP_SET_PRI: 1738 case PSM_INTR_OP_SET_CAP: 1739 case PSM_INTR_OP_SET_CPU: 1740 case PSM_INTR_OP_GET_INTR: 1741 default: 1742 return (PSM_FAILURE); 1743 } 1744 return (PSM_SUCCESS); 1745 } 1746 /* 1747 * Return 1 if CMT load balancing policies should be 1748 * implemented across instances of the specified hardware 1749 * sharing relationship. 1750 */ 1751 int 1752 pg_cmt_load_bal_hw(pghw_type_t hw) 1753 { 1754 if (hw == PGHW_IPIPE || 1755 hw == PGHW_FPU || 1756 hw == PGHW_PROCNODE || 1757 hw == PGHW_CHIP) 1758 return (1); 1759 else 1760 return (0); 1761 } 1762 /* 1763 * Return 1 if thread affinity polices should be implemented 1764 * for instances of the specifed hardware sharing relationship. 1765 */ 1766 int 1767 pg_cmt_affinity_hw(pghw_type_t hw) 1768 { 1769 if (hw == PGHW_CACHE) 1770 return (1); 1771 else 1772 return (0); 1773 } 1774 1775 /* 1776 * Return number of counter events requested to measure hardware capacity and 1777 * utilization and setup CPC requests for specified CPU as needed 1778 * 1779 * May return 0 when platform or processor specific code knows that no CPC 1780 * events should be programmed on this CPU or -1 when platform or processor 1781 * specific code doesn't know which counter events are best to use and common 1782 * code should decide for itself 1783 */ 1784 int 1785 /* LINTED E_FUNC_ARG_UNUSED */ 1786 cu_plat_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs) 1787 { 1788 const char *impl_name; 1789 1790 /* 1791 * Return error if pcbe_ops not set 1792 */ 1793 if (pcbe_ops == NULL) 1794 return (-1); 1795 1796 /* 1797 * Return that no CPC events should be programmed on hyperthreaded 1798 * Pentium 4 and return error for all other x86 processors to tell 1799 * common code to decide what counter events to program on those CPUs 1800 * for measuring hardware capacity and utilization 1801 */ 1802 impl_name = pcbe_ops->pcbe_impl_name(); 1803 if (impl_name != NULL && strcmp(impl_name, PCBE_IMPL_NAME_P4HT) == 0) 1804 return (0); 1805 else 1806 return (-1); 1807 } 1808