1 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* 27 * Copyright (c) 2009, Intel Corporation. 28 * All rights reserved. 29 */ 30 31 #define PSMI_1_6 32 #include <sys/smp_impldefs.h> 33 #include <sys/psm.h> 34 #include <sys/psm_modctl.h> 35 #include <sys/pit.h> 36 #include <sys/cmn_err.h> 37 #include <sys/strlog.h> 38 #include <sys/clock.h> 39 #include <sys/debug.h> 40 #include <sys/rtc.h> 41 #include <sys/x86_archext.h> 42 #include <sys/cpupart.h> 43 #include <sys/cpuvar.h> 44 #include <sys/cpu_event.h> 45 #include <sys/cmt.h> 46 #include <sys/cpu.h> 47 #include <sys/disp.h> 48 #include <sys/archsystm.h> 49 #include <sys/machsystm.h> 50 #include <sys/sysmacros.h> 51 #include <sys/memlist.h> 52 #include <sys/param.h> 53 #include <sys/promif.h> 54 #include <sys/cpu_pm.h> 55 #if defined(__xpv) 56 #include <sys/hypervisor.h> 57 #endif 58 #include <sys/mach_intr.h> 59 #include <vm/hat_i86.h> 60 #include <sys/kdi_machimpl.h> 61 #include <sys/sdt.h> 62 #include <sys/hpet.h> 63 #include <sys/sunddi.h> 64 #include <sys/sunndi.h> 65 #include <sys/cpc_pcbe.h> 66 67 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 68 69 /* 70 * Local function prototypes 71 */ 72 static int mp_disable_intr(processorid_t cpun); 73 static void mp_enable_intr(processorid_t cpun); 74 static void mach_init(); 75 static void mach_picinit(); 76 static int machhztomhz(uint64_t cpu_freq_hz); 77 static uint64_t mach_getcpufreq(void); 78 static void mach_fixcpufreq(void); 79 static int mach_clkinit(int, int *); 80 static void mach_smpinit(void); 81 static int mach_softlvl_to_vect(int ipl); 82 static void mach_get_platform(int owner); 83 static void mach_construct_info(); 84 static int mach_translate_irq(dev_info_t *dip, int irqno); 85 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 86 psm_intr_op_t, int *); 87 static void mach_notify_error(int level, char *errmsg); 88 static hrtime_t dummy_hrtime(void); 89 static void dummy_scalehrtime(hrtime_t *); 90 static uint64_t dummy_unscalehrtime(hrtime_t); 91 void cpu_idle(void); 92 static void cpu_wakeup(cpu_t *, int); 93 #ifndef __xpv 94 void cpu_idle_mwait(void); 95 static void cpu_wakeup_mwait(cpu_t *, int); 96 #endif 97 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp); 98 99 /* 100 * External reference functions 101 */ 102 extern void return_instr(); 103 extern uint64_t freq_tsc(uint32_t *); 104 #if defined(__i386) 105 extern uint64_t freq_notsc(uint32_t *); 106 #endif 107 extern void pc_gethrestime(timestruc_t *); 108 extern int cpuid_get_coreid(cpu_t *); 109 extern int cpuid_get_chipid(cpu_t *); 110 111 /* 112 * PSM functions initialization 113 */ 114 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 115 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 116 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 117 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 118 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 119 void (*psminitf)() = mach_init; 120 void (*picinitf)() = return_instr; 121 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 122 int (*ap_mlsetup)() = (int (*)(void))return_instr; 123 void (*send_dirintf)() = return_instr; 124 void (*setspl)(int) = (void (*)(int))return_instr; 125 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 126 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 127 void (*kdisetsoftint)(int, struct av_softinfo *)= 128 (void (*)(int, struct av_softinfo *))return_instr; 129 void (*setsoftint)(int, struct av_softinfo *)= 130 (void (*)(int, struct av_softinfo *))return_instr; 131 int (*slvltovect)(int) = (int (*)(int))return_instr; 132 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 133 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 134 int (*psm_disable_intr)(int) = mp_disable_intr; 135 void (*psm_enable_intr)(int) = mp_enable_intr; 136 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 137 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 138 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 139 uint64_t (*unscalehrtimef)(hrtime_t) = dummy_unscalehrtime; 140 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 141 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 142 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 143 int (*psm_get_clockirq)(int) = NULL; 144 int (*psm_get_ipivect)(int, int) = NULL; 145 146 int (*psm_clkinit)(int) = NULL; 147 void (*psm_timer_reprogram)(hrtime_t) = NULL; 148 void (*psm_timer_enable)(void) = NULL; 149 void (*psm_timer_disable)(void) = NULL; 150 void (*psm_post_cyclic_setup)(void *arg) = NULL; 151 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 152 int *) = mach_intr_ops; 153 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) 154 return_instr; 155 156 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 157 void (*hrtime_tick)(void) = return_instr; 158 159 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo; 160 161 /* 162 * True if the generic TSC code is our source of hrtime, rather than whatever 163 * the PSM can provide. 164 */ 165 #ifdef __xpv 166 int tsc_gethrtime_enable = 0; 167 #else 168 int tsc_gethrtime_enable = 1; 169 #endif 170 int tsc_gethrtime_initted = 0; 171 172 /* 173 * True if the hrtime implementation is "hires"; namely, better than microdata. 174 */ 175 int gethrtime_hires = 0; 176 177 /* 178 * Local Static Data 179 */ 180 static struct psm_ops mach_ops; 181 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 182 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 183 184 /* 185 * virtualization support for psm 186 */ 187 void *psm_vt_ops = NULL; 188 /* 189 * If non-zero, idle cpus will become "halted" when there's 190 * no work to do. 191 */ 192 int idle_cpu_use_hlt = 1; 193 194 #ifndef __xpv 195 /* 196 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 197 */ 198 int idle_cpu_prefer_mwait = 1; 199 /* 200 * Set to 0 to avoid MONITOR+CLFLUSH assertion. 201 */ 202 int idle_cpu_assert_cflush_monitor = 1; 203 204 /* 205 * If non-zero, idle cpus will not use power saving Deep C-States idle loop. 206 */ 207 int idle_cpu_no_deep_c = 0; 208 /* 209 * Non-power saving idle loop and wakeup pointers. 210 * Allows user to toggle Deep Idle power saving feature on/off. 211 */ 212 void (*non_deep_idle_cpu)() = cpu_idle; 213 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); 214 215 /* 216 * Object for the kernel to access the HPET. 217 */ 218 hpet_t hpet; 219 220 #endif /* ifndef __xpv */ 221 222 /*ARGSUSED*/ 223 int 224 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 225 { 226 switch (hw) { 227 case PGHW_IPIPE: 228 if (x86_feature & (X86_HTT)) { 229 /* 230 * Hyper-threading is SMT 231 */ 232 return (1); 233 } else { 234 return (0); 235 } 236 case PGHW_PROCNODE: 237 if (cpuid_get_procnodes_per_pkg(cp) > 1) 238 return (1); 239 else 240 return (0); 241 case PGHW_CHIP: 242 if (x86_feature & (X86_CMP|X86_HTT)) 243 return (1); 244 else 245 return (0); 246 case PGHW_CACHE: 247 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 248 return (1); 249 else 250 return (0); 251 case PGHW_POW_ACTIVE: 252 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) 253 return (1); 254 else 255 return (0); 256 case PGHW_POW_IDLE: 257 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) 258 return (1); 259 else 260 return (0); 261 default: 262 return (0); 263 } 264 } 265 266 /* 267 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 268 * If pghw_type_t is an unsupported hardware type, then return -1 269 */ 270 int 271 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 272 { 273 id_t pgp_a, pgp_b; 274 275 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 276 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 277 278 if (pgp_a == -1 || pgp_b == -1) 279 return (-1); 280 281 return (pgp_a == pgp_b); 282 } 283 284 /* 285 * Return a physical instance identifier for known hardware sharing 286 * relationships 287 */ 288 id_t 289 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 290 { 291 switch (hw) { 292 case PGHW_IPIPE: 293 return (cpuid_get_coreid(cpu)); 294 case PGHW_CACHE: 295 return (cpuid_get_last_lvl_cacheid(cpu)); 296 case PGHW_PROCNODE: 297 return (cpuid_get_procnodeid(cpu)); 298 case PGHW_CHIP: 299 return (cpuid_get_chipid(cpu)); 300 case PGHW_POW_ACTIVE: 301 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); 302 case PGHW_POW_IDLE: 303 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); 304 default: 305 return (-1); 306 } 307 } 308 309 /* 310 * Express preference for optimizing for sharing relationship 311 * hw1 vs hw2 312 */ 313 pghw_type_t 314 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) 315 { 316 int i, rank1, rank2; 317 318 static pghw_type_t hw_hier[] = { 319 PGHW_IPIPE, 320 PGHW_CACHE, 321 PGHW_PROCNODE, 322 PGHW_CHIP, 323 PGHW_POW_IDLE, 324 PGHW_POW_ACTIVE, 325 PGHW_NUM_COMPONENTS 326 }; 327 328 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 329 if (hw_hier[i] == hw1) 330 rank1 = i; 331 if (hw_hier[i] == hw2) 332 rank2 = i; 333 } 334 335 if (rank1 > rank2) 336 return (hw1); 337 else 338 return (hw2); 339 } 340 341 /* 342 * Override the default CMT dispatcher policy for the specified 343 * hardware sharing relationship 344 */ 345 pg_cmt_policy_t 346 pg_plat_cmt_policy(pghw_type_t hw) 347 { 348 /* 349 * For shared caches, also load balance across them to 350 * maximize aggregate cache capacity 351 */ 352 switch (hw) { 353 case PGHW_CACHE: 354 return (CMT_BALANCE|CMT_AFFINITY); 355 default: 356 return (CMT_NO_POLICY); 357 } 358 } 359 360 id_t 361 pg_plat_get_core_id(cpu_t *cpu) 362 { 363 return ((id_t)cpuid_get_coreid(cpu)); 364 } 365 366 void 367 cmp_set_nosteal_interval(void) 368 { 369 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 370 nosteal_nsec = 100000UL; 371 } 372 373 /* 374 * Routine to ensure initial callers to hrtime gets 0 as return 375 */ 376 static hrtime_t 377 dummy_hrtime(void) 378 { 379 return (0); 380 } 381 382 /* ARGSUSED */ 383 static void 384 dummy_scalehrtime(hrtime_t *ticks) 385 {} 386 387 static uint64_t 388 dummy_unscalehrtime(hrtime_t nsecs) 389 { 390 return ((uint64_t)nsecs); 391 } 392 393 /* 394 * Supports Deep C-State power saving idle loop. 395 */ 396 void 397 cpu_idle_adaptive(void) 398 { 399 (*CPU->cpu_m.mcpu_idle_cpu)(); 400 } 401 402 /* 403 * Function called by CPU idle notification framework to check whether CPU 404 * has been awakened. It will be called with interrupt disabled. 405 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 406 * notification framework. 407 */ 408 /*ARGSUSED*/ 409 static void 410 cpu_idle_check_wakeup(void *arg) 411 { 412 /* 413 * Toggle interrupt flag to detect pending interrupts. 414 * If interrupt happened, do_interrupt() will notify CPU idle 415 * notification framework so no need to call cpu_idle_exit() here. 416 */ 417 sti(); 418 SMT_PAUSE(); 419 cli(); 420 } 421 422 /* 423 * Idle the present CPU until wakened via an interrupt 424 */ 425 void 426 cpu_idle(void) 427 { 428 cpu_t *cpup = CPU; 429 processorid_t cpu_sid = cpup->cpu_seqid; 430 cpupart_t *cp = cpup->cpu_part; 431 int hset_update = 1; 432 433 /* 434 * If this CPU is online, and there's multiple CPUs 435 * in the system, then we should notate our halting 436 * by adding ourselves to the partition's halted CPU 437 * bitmap. This allows other CPUs to find/awaken us when 438 * work becomes available. 439 */ 440 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 441 hset_update = 0; 442 443 /* 444 * Add ourselves to the partition's halted CPUs bitmap 445 * and set our HALTED flag, if necessary. 446 * 447 * When a thread becomes runnable, it is placed on the queue 448 * and then the halted CPU bitmap is checked to determine who 449 * (if anyone) should be awakened. We therefore need to first 450 * add ourselves to the bitmap, and and then check if there 451 * is any work available. The order is important to prevent a race 452 * that can lead to work languishing on a run queue somewhere while 453 * this CPU remains halted. 454 * 455 * Either the producing CPU will see we're halted and will awaken us, 456 * or this CPU will see the work available in disp_anywork(). 457 * 458 * Note that memory barriers after updating the HALTED flag 459 * are not necessary since an atomic operation (updating the bitset) 460 * immediately follows. On x86 the atomic operation acts as a 461 * memory barrier for the update of cpu_disp_flags. 462 */ 463 if (hset_update) { 464 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 465 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 466 } 467 468 /* 469 * Check to make sure there's really nothing to do. 470 * Work destined for this CPU may become available after 471 * this check. We'll be notified through the clearing of our 472 * bit in the halted CPU bitmap, and a poke. 473 */ 474 if (disp_anywork()) { 475 if (hset_update) { 476 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 477 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 478 } 479 return; 480 } 481 482 /* 483 * We're on our way to being halted. 484 * 485 * Disable interrupts now, so that we'll awaken immediately 486 * after halting if someone tries to poke us between now and 487 * the time we actually halt. 488 * 489 * We check for the presence of our bit after disabling interrupts. 490 * If it's cleared, we'll return. If the bit is cleared after 491 * we check then the poke will pop us out of the halted state. 492 * 493 * This means that the ordering of the poke and the clearing 494 * of the bit by cpu_wakeup is important. 495 * cpu_wakeup() must clear, then poke. 496 * cpu_idle() must disable interrupts, then check for the bit. 497 */ 498 cli(); 499 500 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { 501 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 502 sti(); 503 return; 504 } 505 506 /* 507 * The check for anything locally runnable is here for performance 508 * and isn't needed for correctness. disp_nrunnable ought to be 509 * in our cache still, so it's inexpensive to check, and if there 510 * is anything runnable we won't have to wait for the poke. 511 */ 512 if (cpup->cpu_disp->disp_nrunnable != 0) { 513 if (hset_update) { 514 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 515 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 516 } 517 sti(); 518 return; 519 } 520 521 if (cpu_idle_enter(IDLE_STATE_C1, 0, 522 cpu_idle_check_wakeup, NULL) == 0) { 523 mach_cpu_idle(); 524 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 525 } 526 527 /* 528 * We're no longer halted 529 */ 530 if (hset_update) { 531 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 532 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 533 } 534 } 535 536 537 /* 538 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 539 * Otherwise, see if other CPUs in the cpu partition are halted and need to 540 * be woken up so that they can steal the thread we placed on this CPU. 541 * This function is only used on MP systems. 542 */ 543 static void 544 cpu_wakeup(cpu_t *cpu, int bound) 545 { 546 uint_t cpu_found; 547 processorid_t cpu_sid; 548 cpupart_t *cp; 549 550 cp = cpu->cpu_part; 551 cpu_sid = cpu->cpu_seqid; 552 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 553 /* 554 * Clear the halted bit for that CPU since it will be 555 * poked in a moment. 556 */ 557 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 558 /* 559 * We may find the current CPU present in the halted cpuset 560 * if we're in the context of an interrupt that occurred 561 * before we had a chance to clear our bit in cpu_idle(). 562 * Poking ourself is obviously unnecessary, since if 563 * we're here, we're not halted. 564 */ 565 if (cpu != CPU) 566 poke_cpu(cpu->cpu_id); 567 return; 568 } else { 569 /* 570 * This cpu isn't halted, but it's idle or undergoing a 571 * context switch. No need to awaken anyone else. 572 */ 573 if (cpu->cpu_thread == cpu->cpu_idle_thread || 574 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 575 return; 576 } 577 578 /* 579 * No need to wake up other CPUs if this is for a bound thread. 580 */ 581 if (bound) 582 return; 583 584 /* 585 * The CPU specified for wakeup isn't currently halted, so check 586 * to see if there are any other halted CPUs in the partition, 587 * and if there are then awaken one. 588 */ 589 do { 590 cpu_found = bitset_find(&cp->cp_haltset); 591 if (cpu_found == (uint_t)-1) 592 return; 593 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 594 595 if (cpu_found != CPU->cpu_seqid) { 596 poke_cpu(cpu_seq[cpu_found]->cpu_id); 597 } 598 } 599 600 #ifndef __xpv 601 /* 602 * Function called by CPU idle notification framework to check whether CPU 603 * has been awakened. It will be called with interrupt disabled. 604 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 605 * notification framework. 606 */ 607 static void 608 cpu_idle_mwait_check_wakeup(void *arg) 609 { 610 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; 611 612 ASSERT(arg != NULL); 613 if (*mcpu_mwait != MWAIT_HALTED) { 614 /* 615 * CPU has been awakened, notify CPU idle notification system. 616 */ 617 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 618 } else { 619 /* 620 * Toggle interrupt flag to detect pending interrupts. 621 * If interrupt happened, do_interrupt() will notify CPU idle 622 * notification framework so no need to call cpu_idle_exit() 623 * here. 624 */ 625 sti(); 626 SMT_PAUSE(); 627 cli(); 628 } 629 } 630 631 /* 632 * Idle the present CPU until awakened via touching its monitored line 633 */ 634 void 635 cpu_idle_mwait(void) 636 { 637 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 638 cpu_t *cpup = CPU; 639 processorid_t cpu_sid = cpup->cpu_seqid; 640 cpupart_t *cp = cpup->cpu_part; 641 int hset_update = 1; 642 643 /* 644 * Set our mcpu_mwait here, so we can tell if anyone tries to 645 * wake us between now and when we call mwait. No other cpu will 646 * attempt to set our mcpu_mwait until we add ourself to the halted 647 * CPU bitmap. 648 */ 649 *mcpu_mwait = MWAIT_HALTED; 650 651 /* 652 * If this CPU is online, and there's multiple CPUs 653 * in the system, then we should note our halting 654 * by adding ourselves to the partition's halted CPU 655 * bitmap. This allows other CPUs to find/awaken us when 656 * work becomes available. 657 */ 658 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 659 hset_update = 0; 660 661 /* 662 * Add ourselves to the partition's halted CPUs bitmap 663 * and set our HALTED flag, if necessary. 664 * 665 * When a thread becomes runnable, it is placed on the queue 666 * and then the halted CPU bitmap is checked to determine who 667 * (if anyone) should be awakened. We therefore need to first 668 * add ourselves to the bitmap, and and then check if there 669 * is any work available. 670 * 671 * Note that memory barriers after updating the HALTED flag 672 * are not necessary since an atomic operation (updating the bitmap) 673 * immediately follows. On x86 the atomic operation acts as a 674 * memory barrier for the update of cpu_disp_flags. 675 */ 676 if (hset_update) { 677 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 678 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 679 } 680 681 /* 682 * Check to make sure there's really nothing to do. 683 * Work destined for this CPU may become available after 684 * this check. We'll be notified through the clearing of our 685 * bit in the halted CPU bitmap, and a write to our mcpu_mwait. 686 * 687 * disp_anywork() checks disp_nrunnable, so we do not have to later. 688 */ 689 if (disp_anywork()) { 690 if (hset_update) { 691 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 692 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 693 } 694 return; 695 } 696 697 /* 698 * We're on our way to being halted. 699 * To avoid a lost wakeup, arm the monitor before checking if another 700 * cpu wrote to mcpu_mwait to wake us up. 701 */ 702 i86_monitor(mcpu_mwait, 0, 0); 703 if (*mcpu_mwait == MWAIT_HALTED) { 704 if (cpu_idle_enter(IDLE_STATE_C1, 0, 705 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { 706 if (*mcpu_mwait == MWAIT_HALTED) { 707 i86_mwait(0, 0); 708 } 709 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 710 } 711 } 712 713 /* 714 * We're no longer halted 715 */ 716 if (hset_update) { 717 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 718 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 719 } 720 } 721 722 /* 723 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 724 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 725 * need to be woken up so that they can steal the thread we placed on this CPU. 726 * This function is only used on MP systems. 727 */ 728 static void 729 cpu_wakeup_mwait(cpu_t *cp, int bound) 730 { 731 cpupart_t *cpu_part; 732 uint_t cpu_found; 733 processorid_t cpu_sid; 734 735 cpu_part = cp->cpu_part; 736 cpu_sid = cp->cpu_seqid; 737 738 /* 739 * Clear the halted bit for that CPU since it will be woken up 740 * in a moment. 741 */ 742 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { 743 /* 744 * Clear the halted bit for that CPU since it will be 745 * poked in a moment. 746 */ 747 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); 748 /* 749 * We may find the current CPU present in the halted cpuset 750 * if we're in the context of an interrupt that occurred 751 * before we had a chance to clear our bit in cpu_idle(). 752 * Waking ourself is obviously unnecessary, since if 753 * we're here, we're not halted. 754 * 755 * monitor/mwait wakeup via writing to our cache line is 756 * harmless and less expensive than always checking if we 757 * are waking ourself which is an uncommon case. 758 */ 759 MWAIT_WAKEUP(cp); /* write to monitored line */ 760 return; 761 } else { 762 /* 763 * This cpu isn't halted, but it's idle or undergoing a 764 * context switch. No need to awaken anyone else. 765 */ 766 if (cp->cpu_thread == cp->cpu_idle_thread || 767 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 768 return; 769 } 770 771 /* 772 * No need to wake up other CPUs if the thread we just enqueued 773 * is bound. 774 */ 775 if (bound || ncpus == 1) 776 return; 777 778 /* 779 * See if there's any other halted CPUs. If there are, then 780 * select one, and awaken it. 781 * It's possible that after we find a CPU, somebody else 782 * will awaken it before we get the chance. 783 * In that case, look again. 784 */ 785 do { 786 cpu_found = bitset_find(&cpu_part->cp_haltset); 787 if (cpu_found == (uint_t)-1) 788 return; 789 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, 790 cpu_found) < 0); 791 792 /* 793 * Do not check if cpu_found is ourself as monitor/mwait 794 * wakeup is cheap. 795 */ 796 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ 797 } 798 799 #endif 800 801 void (*cpu_pause_handler)(volatile char *) = NULL; 802 803 static int 804 mp_disable_intr(int cpun) 805 { 806 /* 807 * switch to the offline cpu 808 */ 809 affinity_set(cpun); 810 /* 811 * raise ipl to just below cross call 812 */ 813 splx(XC_SYS_PIL - 1); 814 /* 815 * set base spl to prevent the next swtch to idle from 816 * lowering back to ipl 0 817 */ 818 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); 819 set_base_spl(); 820 affinity_clear(); 821 return (DDI_SUCCESS); 822 } 823 824 static void 825 mp_enable_intr(int cpun) 826 { 827 /* 828 * switch to the online cpu 829 */ 830 affinity_set(cpun); 831 /* 832 * clear the interrupt active mask 833 */ 834 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); 835 set_base_spl(); 836 (void) spl0(); 837 affinity_clear(); 838 } 839 840 static void 841 mach_get_platform(int owner) 842 { 843 void **srv_opsp; 844 void **clt_opsp; 845 int i; 846 int total_ops; 847 848 /* fix up psm ops */ 849 srv_opsp = (void **)mach_set[0]; 850 clt_opsp = (void **)mach_set[owner]; 851 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 852 total_ops = sizeof (struct psm_ops_ver01) / 853 sizeof (void (*)(void)); 854 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 855 /* no psm_notify_func */ 856 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 857 sizeof (void (*)(void)); 858 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 859 /* no psm_timer funcs */ 860 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 861 sizeof (void (*)(void)); 862 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 863 /* no psm_preshutdown function */ 864 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 865 sizeof (void (*)(void)); 866 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 867 /* no psm_preshutdown function */ 868 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 869 sizeof (void (*)(void)); 870 else 871 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 872 873 /* 874 * Save the version of the PSM module, in case we need to 875 * behave differently based on version. 876 */ 877 mach_ver[0] = mach_ver[owner]; 878 879 for (i = 0; i < total_ops; i++) 880 if (clt_opsp[i] != NULL) 881 srv_opsp[i] = clt_opsp[i]; 882 } 883 884 static void 885 mach_construct_info() 886 { 887 struct psm_sw *swp; 888 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 889 int conflict_owner = 0; 890 891 if (psmsw->psw_forw == psmsw) 892 panic("No valid PSM modules found"); 893 mutex_enter(&psmsw_lock); 894 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 895 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 896 continue; 897 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 898 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 899 mach_cnt[swp->psw_infop->p_owner]++; 900 } 901 mutex_exit(&psmsw_lock); 902 903 mach_get_platform(PSM_OWN_SYS_DEFAULT); 904 905 /* check to see are there any conflicts */ 906 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 907 conflict_owner = PSM_OWN_EXCLUSIVE; 908 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 909 conflict_owner = PSM_OWN_OVERRIDE; 910 if (conflict_owner) { 911 /* remove all psm modules except uppc */ 912 cmn_err(CE_WARN, 913 "Conflicts detected on the following PSM modules:"); 914 mutex_enter(&psmsw_lock); 915 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 916 if (swp->psw_infop->p_owner == conflict_owner) 917 cmn_err(CE_WARN, "%s ", 918 swp->psw_infop->p_mach_idstring); 919 } 920 mutex_exit(&psmsw_lock); 921 cmn_err(CE_WARN, 922 "Setting the system back to SINGLE processor mode!"); 923 cmn_err(CE_WARN, 924 "Please edit /etc/mach to remove the invalid PSM module."); 925 return; 926 } 927 928 if (mach_set[PSM_OWN_EXCLUSIVE]) 929 mach_get_platform(PSM_OWN_EXCLUSIVE); 930 931 if (mach_set[PSM_OWN_OVERRIDE]) 932 mach_get_platform(PSM_OWN_OVERRIDE); 933 } 934 935 static void 936 mach_init() 937 { 938 struct psm_ops *pops; 939 940 mach_construct_info(); 941 942 pops = mach_set[0]; 943 944 /* register the interrupt and clock initialization rotuines */ 945 picinitf = mach_picinit; 946 clkinitf = mach_clkinit; 947 psm_get_clockirq = pops->psm_get_clockirq; 948 949 /* register the interrupt setup code */ 950 slvltovect = mach_softlvl_to_vect; 951 addspl = pops->psm_addspl; 952 delspl = pops->psm_delspl; 953 954 if (pops->psm_translate_irq) 955 psm_translate_irq = pops->psm_translate_irq; 956 if (pops->psm_intr_ops) 957 psm_intr_ops = pops->psm_intr_ops; 958 959 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 960 /* 961 * Time-of-day functionality now handled in TOD modules. 962 * (Warn about PSM modules that think that we're going to use 963 * their ops vectors.) 964 */ 965 if (pops->psm_tod_get) 966 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 967 (void *)pops->psm_tod_get); 968 969 if (pops->psm_tod_set) 970 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 971 (void *)pops->psm_tod_set); 972 #endif 973 974 if (pops->psm_notify_error) { 975 psm_notify_error = mach_notify_error; 976 notify_error = pops->psm_notify_error; 977 } 978 979 (*pops->psm_softinit)(); 980 981 /* 982 * Initialize the dispatcher's function hooks to enable CPU halting 983 * when idle. Set both the deep-idle and non-deep-idle hooks. 984 * 985 * Assume we can use power saving deep-idle loop cpu_idle_adaptive. 986 * Platform deep-idle driver will reset our idle loop to 987 * non_deep_idle_cpu if power saving deep-idle feature is not available. 988 * 989 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) 990 * or idle_cpu_prefer_mwait is not set. 991 * Allocate monitor/mwait buffer for cpu0. 992 */ 993 #ifndef __xpv 994 non_deep_idle_disp_enq_thread = disp_enq_thread; 995 #endif 996 if (idle_cpu_use_hlt) { 997 idle_cpu = cpu_idle_adaptive; 998 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 999 #ifndef __xpv 1000 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) { 1001 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); 1002 /* 1003 * Protect ourself from insane mwait size. 1004 */ 1005 if (CPU->cpu_m.mcpu_mwait == NULL) { 1006 #ifdef DEBUG 1007 cmn_err(CE_NOTE, "Using hlt idle. Cannot " 1008 "handle cpu 0 mwait size."); 1009 #endif 1010 idle_cpu_prefer_mwait = 0; 1011 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1012 } else { 1013 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; 1014 } 1015 } else { 1016 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1017 } 1018 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; 1019 1020 /* 1021 * Disable power saving deep idle loop? 1022 */ 1023 if (idle_cpu_no_deep_c) { 1024 idle_cpu = non_deep_idle_cpu; 1025 } 1026 #endif 1027 } 1028 1029 mach_smpinit(); 1030 } 1031 1032 static void 1033 mach_smpinit(void) 1034 { 1035 struct psm_ops *pops; 1036 processorid_t cpu_id; 1037 int cnt; 1038 cpuset_t cpumask; 1039 1040 pops = mach_set[0]; 1041 CPUSET_ZERO(cpumask); 1042 1043 cpu_id = -1; 1044 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1045 for (cnt = 0; cpu_id != -1; cnt++) { 1046 CPUSET_ADD(cpumask, cpu_id); 1047 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1048 } 1049 1050 mp_cpus = cpumask; 1051 1052 /* MP related routines */ 1053 ap_mlsetup = pops->psm_post_cpu_start; 1054 send_dirintf = pops->psm_send_ipi; 1055 1056 /* optional MP related routines */ 1057 if (pops->psm_shutdown) 1058 psm_shutdownf = pops->psm_shutdown; 1059 if (pops->psm_preshutdown) 1060 psm_preshutdownf = pops->psm_preshutdown; 1061 if (pops->psm_notify_func) 1062 psm_notifyf = pops->psm_notify_func; 1063 if (pops->psm_set_idlecpu) 1064 psm_set_idle_cpuf = pops->psm_set_idlecpu; 1065 if (pops->psm_unset_idlecpu) 1066 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 1067 1068 psm_clkinit = pops->psm_clkinit; 1069 1070 if (pops->psm_timer_reprogram) 1071 psm_timer_reprogram = pops->psm_timer_reprogram; 1072 1073 if (pops->psm_timer_enable) 1074 psm_timer_enable = pops->psm_timer_enable; 1075 1076 if (pops->psm_timer_disable) 1077 psm_timer_disable = pops->psm_timer_disable; 1078 1079 if (pops->psm_post_cyclic_setup) 1080 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 1081 1082 if (pops->psm_state) 1083 psm_state = pops->psm_state; 1084 1085 /* 1086 * Set these vectors here so they can be used by Suspend/Resume 1087 * on UP machines. 1088 */ 1089 if (pops->psm_disable_intr) 1090 psm_disable_intr = pops->psm_disable_intr; 1091 if (pops->psm_enable_intr) 1092 psm_enable_intr = pops->psm_enable_intr; 1093 1094 /* check for multiple CPUs */ 1095 if (cnt < 2) 1096 return; 1097 1098 /* check for MP platforms */ 1099 if (pops->psm_cpu_start == NULL) 1100 return; 1101 1102 /* 1103 * Set the dispatcher hook to enable cpu "wake up" 1104 * when a thread becomes runnable. 1105 */ 1106 if (idle_cpu_use_hlt) { 1107 disp_enq_thread = cpu_wakeup; 1108 #ifndef __xpv 1109 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) 1110 disp_enq_thread = cpu_wakeup_mwait; 1111 non_deep_idle_disp_enq_thread = disp_enq_thread; 1112 #endif 1113 } 1114 1115 psm_get_ipivect = pops->psm_get_ipivect; 1116 1117 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", 1118 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 1119 NULL, NULL, NULL, NULL); 1120 1121 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 1122 } 1123 1124 static void 1125 mach_picinit() 1126 { 1127 struct psm_ops *pops; 1128 1129 pops = mach_set[0]; 1130 1131 /* register the interrupt handlers */ 1132 setlvl = pops->psm_intr_enter; 1133 setlvlx = pops->psm_intr_exit; 1134 1135 /* initialize the interrupt hardware */ 1136 (*pops->psm_picinit)(); 1137 1138 /* set interrupt mask for current ipl */ 1139 setspl = pops->psm_setspl; 1140 cli(); 1141 setspl(CPU->cpu_pri); 1142 } 1143 1144 uint_t cpu_freq; /* MHz */ 1145 uint64_t cpu_freq_hz; /* measured (in hertz) */ 1146 1147 #define MEGA_HZ 1000000 1148 1149 #ifdef __xpv 1150 1151 int xpv_cpufreq_workaround = 1; 1152 int xpv_cpufreq_verbose = 0; 1153 1154 #else /* __xpv */ 1155 1156 static uint64_t 1157 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 1158 { 1159 uint64_t cpu_hz; 1160 1161 if ((pit_counter == 0) || (*processor_clks == 0) || 1162 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 1163 return (0); 1164 1165 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 1166 1167 return (cpu_hz); 1168 } 1169 1170 #endif /* __xpv */ 1171 1172 static uint64_t 1173 mach_getcpufreq(void) 1174 { 1175 #if defined(__xpv) 1176 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; 1177 uint64_t cpu_hz; 1178 1179 /* 1180 * During dom0 bringup, it was noted that on at least one older 1181 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul 1182 * value that is quite wrong (the 3.06GHz clock was reported 1183 * as 4.77GHz) 1184 * 1185 * The curious thing is, that if you stop the kernel at entry, 1186 * breakpoint here and inspect the value with kmdb, the value 1187 * is correct - but if you don't stop and simply enable the 1188 * printf statement (below), you can see the bad value printed 1189 * here. Almost as if something kmdb did caused the hypervisor to 1190 * figure it out correctly. And, note that the hypervisor 1191 * eventually -does- figure it out correctly ... if you look at 1192 * the field later in the life of dom0, it is correct. 1193 * 1194 * For now, on dom0, we employ a slightly cheesy workaround of 1195 * using the DOM0_PHYSINFO hypercall. 1196 */ 1197 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { 1198 cpu_hz = 1000 * xpv_cpu_khz(); 1199 } else { 1200 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; 1201 1202 if (vti->tsc_shift < 0) 1203 cpu_hz <<= -vti->tsc_shift; 1204 else 1205 cpu_hz >>= vti->tsc_shift; 1206 } 1207 1208 if (xpv_cpufreq_verbose) 1209 printf("mach_getcpufreq: system_mul 0x%x, shift %d, " 1210 "cpu_hz %" PRId64 "Hz\n", 1211 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); 1212 1213 return (cpu_hz); 1214 #else /* __xpv */ 1215 uint32_t pit_counter; 1216 uint64_t processor_clks; 1217 1218 if (x86_feature & X86_TSC) { 1219 /* 1220 * We have a TSC. freq_tsc() knows how to measure the number 1221 * of clock cycles sampled against the PIT. 1222 */ 1223 ulong_t flags = clear_int_flag(); 1224 processor_clks = freq_tsc(&pit_counter); 1225 restore_int_flag(flags); 1226 return (mach_calchz(pit_counter, &processor_clks)); 1227 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1228 #if defined(__amd64) 1229 panic("mach_getcpufreq: no TSC!"); 1230 #elif defined(__i386) 1231 /* 1232 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1233 * for which freq_notsc() knows how to measure the number of 1234 * elapsed clock cycles sampled against the PIT 1235 */ 1236 ulong_t flags = clear_int_flag(); 1237 processor_clks = freq_notsc(&pit_counter); 1238 restore_int_flag(flags); 1239 return (mach_calchz(pit_counter, &processor_clks)); 1240 #endif /* __i386 */ 1241 } 1242 1243 /* We do not know how to calculate cpu frequency for this cpu. */ 1244 return (0); 1245 #endif /* __xpv */ 1246 } 1247 1248 /* 1249 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1250 * to this array, instead improve the accuracy of the algorithm that determines 1251 * the clock speed of the processor or extend the implementation to support the 1252 * vendor as appropriate. This is here only to support adjusting the speed on 1253 * older slower processors that mach_fixcpufreq() would not be able to account 1254 * for otherwise. 1255 */ 1256 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1257 1258 /* 1259 * On fast processors the clock frequency that is measured may be off by 1260 * a few MHz from the value printed on the part. This is a combination of 1261 * the factors that for such fast parts being off by this much is within 1262 * the tolerances for manufacture and because of the difficulties in the 1263 * measurement that can lead to small error. This function uses some 1264 * heuristics in order to tweak the value that was measured to match what 1265 * is most likely printed on the part. 1266 * 1267 * Some examples: 1268 * AMD Athlon 1000 mhz measured as 998 mhz 1269 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1270 * Intel Pentium IV 1500 mhz measured as 1495mhz 1271 * 1272 * If in the future this function is no longer sufficient to correct 1273 * for the error in the measurement, then the algorithm used to perform 1274 * the measurement will have to be improved in order to increase accuracy 1275 * rather than adding horrible and questionable kludges here. 1276 * 1277 * This is called after the cyclics subsystem because of the potential 1278 * that the heuristics within may give a worse estimate of the clock 1279 * frequency than the value that was measured. 1280 */ 1281 static void 1282 mach_fixcpufreq(void) 1283 { 1284 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1285 1286 freq = (uint32_t)cpu_freq; 1287 1288 /* 1289 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1290 * measured speed taking into account that the 667 MHz parts were 1291 * the first to round-up. 1292 */ 1293 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1294 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1295 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1296 1297 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1298 mul = (freq + 25) / 50; 1299 near50 = mul * 50; 1300 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1301 1302 /* Find the closer of the two */ 1303 if (delta66 < delta50) { 1304 fixed = near66; 1305 delta = delta66; 1306 } else { 1307 fixed = near50; 1308 delta = delta50; 1309 } 1310 1311 if (fixed > INT_MAX) 1312 return; 1313 1314 /* 1315 * Some older parts have a core clock frequency that is not an 1316 * integral multiple of 50 or 66 MHz. Check if one of the old 1317 * clock frequencies is closer to the measured value than any 1318 * of the integral multiples of 50 an 66, and if so set fixed 1319 * and delta appropriately to represent the closest value. 1320 */ 1321 i = sizeof (x86_cpu_freq) / sizeof (int); 1322 while (i > 0) { 1323 i--; 1324 1325 if (x86_cpu_freq[i] <= freq) { 1326 mul = freq - x86_cpu_freq[i]; 1327 1328 if (mul < delta) { 1329 fixed = x86_cpu_freq[i]; 1330 delta = mul; 1331 } 1332 1333 break; 1334 } 1335 1336 mul = x86_cpu_freq[i] - freq; 1337 1338 if (mul < delta) { 1339 fixed = x86_cpu_freq[i]; 1340 delta = mul; 1341 } 1342 } 1343 1344 /* 1345 * Set a reasonable maximum for how much to correct the measured 1346 * result by. This check is here to prevent the adjustment made 1347 * by this function from being more harm than good. It is entirely 1348 * possible that in the future parts will be made that are not 1349 * integral multiples of 66 or 50 in clock frequency or that 1350 * someone may overclock a part to some odd frequency. If the 1351 * measured value is farther from the corrected value than 1352 * allowed, then assume the corrected value is in error and use 1353 * the measured value. 1354 */ 1355 if (6 < delta) 1356 return; 1357 1358 cpu_freq = (int)fixed; 1359 } 1360 1361 1362 static int 1363 machhztomhz(uint64_t cpu_freq_hz) 1364 { 1365 uint64_t cpu_mhz; 1366 1367 /* Round to nearest MHZ */ 1368 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1369 1370 if (cpu_mhz > INT_MAX) 1371 return (0); 1372 1373 return ((int)cpu_mhz); 1374 1375 } 1376 1377 1378 static int 1379 mach_clkinit(int preferred_mode, int *set_mode) 1380 { 1381 struct psm_ops *pops; 1382 int resolution; 1383 1384 pops = mach_set[0]; 1385 1386 cpu_freq_hz = mach_getcpufreq(); 1387 1388 cpu_freq = machhztomhz(cpu_freq_hz); 1389 1390 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 1391 tsc_gethrtime_enable = 0; 1392 1393 #ifndef __xpv 1394 if (tsc_gethrtime_enable) { 1395 tsc_hrtimeinit(cpu_freq_hz); 1396 } else 1397 #endif 1398 { 1399 if (pops->psm_hrtimeinit) 1400 (*pops->psm_hrtimeinit)(); 1401 gethrtimef = pops->psm_gethrtime; 1402 gethrtimeunscaledf = gethrtimef; 1403 /* scalehrtimef will remain dummy */ 1404 } 1405 1406 mach_fixcpufreq(); 1407 1408 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1409 if (preferred_mode == TIMER_ONESHOT) { 1410 1411 resolution = (*pops->psm_clkinit)(0); 1412 if (resolution != 0) { 1413 *set_mode = TIMER_ONESHOT; 1414 return (resolution); 1415 } 1416 } 1417 1418 /* 1419 * either periodic mode was requested or could not set to 1420 * one-shot mode 1421 */ 1422 resolution = (*pops->psm_clkinit)(hz); 1423 /* 1424 * psm should be able to do periodic, so we do not check 1425 * for return value of psm_clkinit here. 1426 */ 1427 *set_mode = TIMER_PERIODIC; 1428 return (resolution); 1429 } else { 1430 /* 1431 * PSMI interface prior to PSMI_3 does not define a return 1432 * value for psm_clkinit, so the return value is ignored. 1433 */ 1434 (void) (*pops->psm_clkinit)(hz); 1435 *set_mode = TIMER_PERIODIC; 1436 return (nsec_per_tick); 1437 } 1438 } 1439 1440 1441 /*ARGSUSED*/ 1442 static int 1443 mach_softlvl_to_vect(int ipl) 1444 { 1445 setsoftint = av_set_softint_pending; 1446 kdisetsoftint = kdi_av_set_softint_pending; 1447 1448 return (PSM_SV_SOFTWARE); 1449 } 1450 1451 #ifdef DEBUG 1452 /* 1453 * This is here to allow us to simulate cpus that refuse to start. 1454 */ 1455 cpuset_t cpufailset; 1456 #endif 1457 1458 int 1459 mach_cpu_start(struct cpu *cp, void *ctx) 1460 { 1461 struct psm_ops *pops = mach_set[0]; 1462 processorid_t id = cp->cpu_id; 1463 1464 #ifdef DEBUG 1465 if (CPU_IN_SET(cpufailset, id)) 1466 return (0); 1467 #endif 1468 return ((*pops->psm_cpu_start)(id, ctx)); 1469 } 1470 1471 int 1472 mach_cpuid_start(processorid_t id, void *ctx) 1473 { 1474 struct psm_ops *pops = mach_set[0]; 1475 1476 #ifdef DEBUG 1477 if (CPU_IN_SET(cpufailset, id)) 1478 return (0); 1479 #endif 1480 return ((*pops->psm_cpu_start)(id, ctx)); 1481 } 1482 1483 /* 1484 * Default handler to create device node for CPU. 1485 * One reference count will be held on created device node. 1486 */ 1487 static int 1488 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp) 1489 { 1490 int rv, circ; 1491 dev_info_t *dip; 1492 static kmutex_t cpu_node_lock; 1493 static dev_info_t *cpu_nex_devi = NULL; 1494 1495 ASSERT(cp != NULL); 1496 ASSERT(dipp != NULL); 1497 *dipp = NULL; 1498 1499 if (cpu_nex_devi == NULL) { 1500 mutex_enter(&cpu_node_lock); 1501 /* First check whether cpus exists. */ 1502 cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0); 1503 /* Create cpus if it doesn't exist. */ 1504 if (cpu_nex_devi == NULL) { 1505 ndi_devi_enter(ddi_root_node(), &circ); 1506 rv = ndi_devi_alloc(ddi_root_node(), "cpus", 1507 (pnode_t)DEVI_SID_NODEID, &dip); 1508 if (rv != NDI_SUCCESS) { 1509 mutex_exit(&cpu_node_lock); 1510 cmn_err(CE_CONT, 1511 "?failed to create cpu nexus device.\n"); 1512 return (PSM_FAILURE); 1513 } 1514 ASSERT(dip != NULL); 1515 (void) ndi_devi_online(dip, 0); 1516 ndi_devi_exit(ddi_root_node(), circ); 1517 cpu_nex_devi = dip; 1518 } 1519 mutex_exit(&cpu_node_lock); 1520 } 1521 1522 /* 1523 * create a child node for cpu identified as 'cpu_id' 1524 */ 1525 ndi_devi_enter(cpu_nex_devi, &circ); 1526 dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1); 1527 if (dip == NULL) { 1528 cmn_err(CE_CONT, 1529 "?failed to create device node for cpu%d.\n", cp->cpu_id); 1530 rv = PSM_FAILURE; 1531 } else { 1532 *dipp = dip; 1533 (void) ndi_hold_devi(dip); 1534 rv = PSM_SUCCESS; 1535 } 1536 ndi_devi_exit(cpu_nex_devi, circ); 1537 1538 return (rv); 1539 } 1540 1541 /* 1542 * Create cpu device node in device tree and online it. 1543 * Return created dip with reference count held if requested. 1544 */ 1545 int 1546 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp) 1547 { 1548 int rv; 1549 dev_info_t *dip = NULL; 1550 1551 ASSERT(psm_cpu_create_devinfo != NULL); 1552 rv = psm_cpu_create_devinfo(cp, &dip); 1553 if (rv == PSM_SUCCESS) { 1554 cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi); 1555 /* Recursively attach driver for parent nexus device. */ 1556 if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) == 1557 DDI_SUCCESS) { 1558 /* Configure cpu itself and descendants. */ 1559 (void) ndi_devi_online(dip, 1560 NDI_ONLINE_ATTACH | NDI_CONFIG); 1561 } 1562 if (dipp != NULL) { 1563 *dipp = dip; 1564 } else { 1565 (void) ndi_rele_devi(dip); 1566 } 1567 } 1568 1569 return (rv); 1570 } 1571 1572 /*ARGSUSED*/ 1573 static int 1574 mach_translate_irq(dev_info_t *dip, int irqno) 1575 { 1576 return (irqno); /* default to NO translation */ 1577 } 1578 1579 static void 1580 mach_notify_error(int level, char *errmsg) 1581 { 1582 /* 1583 * SL_FATAL is pass in once panicstr is set, deliver it 1584 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1585 * codes for the psmi handler 1586 */ 1587 if (level & SL_FATAL) 1588 (*notify_error)(CE_PANIC, errmsg); 1589 else if (level & SL_WARN) 1590 (*notify_error)(CE_WARN, errmsg); 1591 else if (level & SL_NOTE) 1592 (*notify_error)(CE_NOTE, errmsg); 1593 else if (level & SL_CONSOLE) 1594 (*notify_error)(CE_CONT, errmsg); 1595 } 1596 1597 /* 1598 * It provides the default basic intr_ops interface for the new DDI 1599 * interrupt framework if the PSM doesn't have one. 1600 * 1601 * Input: 1602 * dip - pointer to the dev_info structure of the requested device 1603 * hdlp - pointer to the internal interrupt handle structure for the 1604 * requested interrupt 1605 * intr_op - opcode for this call 1606 * result - pointer to the integer that will hold the result to be 1607 * passed back if return value is PSM_SUCCESS 1608 * 1609 * Output: 1610 * return value is either PSM_SUCCESS or PSM_FAILURE 1611 */ 1612 static int 1613 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1614 psm_intr_op_t intr_op, int *result) 1615 { 1616 struct intrspec *ispec; 1617 1618 switch (intr_op) { 1619 case PSM_INTR_OP_CHECK_MSI: 1620 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1621 DDI_INTR_TYPE_MSIX); 1622 break; 1623 case PSM_INTR_OP_ALLOC_VECTORS: 1624 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1625 *result = 1; 1626 else 1627 *result = 0; 1628 break; 1629 case PSM_INTR_OP_FREE_VECTORS: 1630 break; 1631 case PSM_INTR_OP_NAVAIL_VECTORS: 1632 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1633 *result = 1; 1634 else 1635 *result = 0; 1636 break; 1637 case PSM_INTR_OP_XLATE_VECTOR: 1638 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1639 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1640 break; 1641 case PSM_INTR_OP_GET_CAP: 1642 *result = 0; 1643 break; 1644 case PSM_INTR_OP_GET_PENDING: 1645 case PSM_INTR_OP_CLEAR_MASK: 1646 case PSM_INTR_OP_SET_MASK: 1647 case PSM_INTR_OP_GET_SHARED: 1648 case PSM_INTR_OP_SET_PRI: 1649 case PSM_INTR_OP_SET_CAP: 1650 case PSM_INTR_OP_SET_CPU: 1651 case PSM_INTR_OP_GET_INTR: 1652 default: 1653 return (PSM_FAILURE); 1654 } 1655 return (PSM_SUCCESS); 1656 } 1657 /* 1658 * Return 1 if CMT load balancing policies should be 1659 * implemented across instances of the specified hardware 1660 * sharing relationship. 1661 */ 1662 int 1663 pg_cmt_load_bal_hw(pghw_type_t hw) 1664 { 1665 if (hw == PGHW_IPIPE || 1666 hw == PGHW_FPU || 1667 hw == PGHW_PROCNODE || 1668 hw == PGHW_CHIP) 1669 return (1); 1670 else 1671 return (0); 1672 } 1673 /* 1674 * Return 1 if thread affinity polices should be implemented 1675 * for instances of the specifed hardware sharing relationship. 1676 */ 1677 int 1678 pg_cmt_affinity_hw(pghw_type_t hw) 1679 { 1680 if (hw == PGHW_CACHE) 1681 return (1); 1682 else 1683 return (0); 1684 } 1685 1686 /* 1687 * Return number of counter events requested to measure hardware capacity and 1688 * utilization and setup CPC requests for specified CPU as needed 1689 * 1690 * May return 0 when platform or processor specific code knows that no CPC 1691 * events should be programmed on this CPU or -1 when platform or processor 1692 * specific code doesn't know which counter events are best to use and common 1693 * code should decide for itself 1694 */ 1695 int 1696 /* LINTED E_FUNC_ARG_UNUSED */ 1697 cu_plat_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs) 1698 { 1699 const char *impl_name; 1700 1701 /* 1702 * Return error if pcbe_ops not set 1703 */ 1704 if (pcbe_ops == NULL) 1705 return (-1); 1706 1707 /* 1708 * Return that no CPC events should be programmed on hyperthreaded 1709 * Pentium 4 and return error for all other x86 processors to tell 1710 * common code to decide what counter events to program on those CPUs 1711 * for measuring hardware capacity and utilization 1712 */ 1713 impl_name = pcbe_ops->pcbe_impl_name(); 1714 if (impl_name != NULL && strcmp(impl_name, PCBE_IMPL_NAME_P4HT) == 0) 1715 return (0); 1716 else 1717 return (-1); 1718 } 1719