1 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* 27 * Copyright (c) 2009-2010, Intel Corporation. 28 * All rights reserved. 29 */ 30 31 #define PSMI_1_7 32 #include <sys/smp_impldefs.h> 33 #include <sys/psm.h> 34 #include <sys/psm_modctl.h> 35 #include <sys/pit.h> 36 #include <sys/cmn_err.h> 37 #include <sys/strlog.h> 38 #include <sys/clock.h> 39 #include <sys/debug.h> 40 #include <sys/rtc.h> 41 #include <sys/x86_archext.h> 42 #include <sys/cpupart.h> 43 #include <sys/cpuvar.h> 44 #include <sys/cpu_event.h> 45 #include <sys/cmt.h> 46 #include <sys/cpu.h> 47 #include <sys/disp.h> 48 #include <sys/archsystm.h> 49 #include <sys/machsystm.h> 50 #include <sys/sysmacros.h> 51 #include <sys/memlist.h> 52 #include <sys/param.h> 53 #include <sys/promif.h> 54 #include <sys/cpu_pm.h> 55 #if defined(__xpv) 56 #include <sys/hypervisor.h> 57 #endif 58 #include <sys/mach_intr.h> 59 #include <vm/hat_i86.h> 60 #include <sys/kdi_machimpl.h> 61 #include <sys/sdt.h> 62 #include <sys/hpet.h> 63 #include <sys/sunddi.h> 64 #include <sys/sunndi.h> 65 #include <sys/cpc_pcbe.h> 66 67 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 68 69 /* 70 * Local function prototypes 71 */ 72 static int mp_disable_intr(processorid_t cpun); 73 static void mp_enable_intr(processorid_t cpun); 74 static void mach_init(); 75 static void mach_picinit(); 76 static int machhztomhz(uint64_t cpu_freq_hz); 77 static uint64_t mach_getcpufreq(void); 78 static void mach_fixcpufreq(void); 79 static int mach_clkinit(int, int *); 80 static void mach_smpinit(void); 81 static int mach_softlvl_to_vect(int ipl); 82 static void mach_get_platform(int owner); 83 static void mach_construct_info(); 84 static int mach_translate_irq(dev_info_t *dip, int irqno); 85 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 86 psm_intr_op_t, int *); 87 static void mach_notify_error(int level, char *errmsg); 88 static hrtime_t dummy_hrtime(void); 89 static void dummy_scalehrtime(hrtime_t *); 90 static uint64_t dummy_unscalehrtime(hrtime_t); 91 void cpu_idle(void); 92 static void cpu_wakeup(cpu_t *, int); 93 #ifndef __xpv 94 void cpu_idle_mwait(void); 95 static void cpu_wakeup_mwait(cpu_t *, int); 96 #endif 97 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp); 98 99 /* 100 * External reference functions 101 */ 102 extern void return_instr(); 103 extern uint64_t freq_tsc(uint32_t *); 104 #if defined(__i386) 105 extern uint64_t freq_notsc(uint32_t *); 106 #endif 107 extern void pc_gethrestime(timestruc_t *); 108 extern int cpuid_get_coreid(cpu_t *); 109 extern int cpuid_get_chipid(cpu_t *); 110 111 /* 112 * PSM functions initialization 113 */ 114 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 115 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 116 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 117 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 118 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 119 void (*psminitf)() = mach_init; 120 void (*picinitf)() = return_instr; 121 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 122 int (*ap_mlsetup)() = (int (*)(void))return_instr; 123 void (*send_dirintf)() = return_instr; 124 void (*setspl)(int) = (void (*)(int))return_instr; 125 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 126 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 127 void (*kdisetsoftint)(int, struct av_softinfo *)= 128 (void (*)(int, struct av_softinfo *))return_instr; 129 void (*setsoftint)(int, struct av_softinfo *)= 130 (void (*)(int, struct av_softinfo *))return_instr; 131 int (*slvltovect)(int) = (int (*)(int))return_instr; 132 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 133 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 134 int (*psm_disable_intr)(int) = mp_disable_intr; 135 void (*psm_enable_intr)(int) = mp_enable_intr; 136 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 137 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 138 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 139 uint64_t (*unscalehrtimef)(hrtime_t) = dummy_unscalehrtime; 140 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 141 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 142 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 143 int (*psm_get_clockirq)(int) = NULL; 144 int (*psm_get_ipivect)(int, int) = NULL; 145 146 int (*psm_clkinit)(int) = NULL; 147 void (*psm_timer_reprogram)(hrtime_t) = NULL; 148 void (*psm_timer_enable)(void) = NULL; 149 void (*psm_timer_disable)(void) = NULL; 150 void (*psm_post_cyclic_setup)(void *arg) = NULL; 151 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 152 int *) = mach_intr_ops; 153 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) 154 return_instr; 155 156 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 157 void (*hrtime_tick)(void) = return_instr; 158 159 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo; 160 int (*psm_cpu_get_devinfo)(cpu_t *, dev_info_t **) = NULL; 161 162 /* 163 * True if the generic TSC code is our source of hrtime, rather than whatever 164 * the PSM can provide. 165 */ 166 #ifdef __xpv 167 int tsc_gethrtime_enable = 0; 168 #else 169 int tsc_gethrtime_enable = 1; 170 #endif 171 int tsc_gethrtime_initted = 0; 172 173 /* 174 * True if the hrtime implementation is "hires"; namely, better than microdata. 175 */ 176 int gethrtime_hires = 0; 177 178 /* 179 * Local Static Data 180 */ 181 static struct psm_ops mach_ops; 182 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 183 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 184 185 /* 186 * virtualization support for psm 187 */ 188 void *psm_vt_ops = NULL; 189 /* 190 * If non-zero, idle cpus will become "halted" when there's 191 * no work to do. 192 */ 193 int idle_cpu_use_hlt = 1; 194 195 #ifndef __xpv 196 /* 197 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 198 */ 199 int idle_cpu_prefer_mwait = 1; 200 /* 201 * Set to 0 to avoid MONITOR+CLFLUSH assertion. 202 */ 203 int idle_cpu_assert_cflush_monitor = 1; 204 205 /* 206 * If non-zero, idle cpus will not use power saving Deep C-States idle loop. 207 */ 208 int idle_cpu_no_deep_c = 0; 209 /* 210 * Non-power saving idle loop and wakeup pointers. 211 * Allows user to toggle Deep Idle power saving feature on/off. 212 */ 213 void (*non_deep_idle_cpu)() = cpu_idle; 214 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); 215 216 /* 217 * Object for the kernel to access the HPET. 218 */ 219 hpet_t hpet; 220 221 #endif /* ifndef __xpv */ 222 223 /*ARGSUSED*/ 224 int 225 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 226 { 227 switch (hw) { 228 case PGHW_IPIPE: 229 if (x86_feature & (X86_HTT)) { 230 /* 231 * Hyper-threading is SMT 232 */ 233 return (1); 234 } else { 235 return (0); 236 } 237 case PGHW_PROCNODE: 238 if (cpuid_get_procnodes_per_pkg(cp) > 1) 239 return (1); 240 else 241 return (0); 242 case PGHW_CHIP: 243 if (x86_feature & (X86_CMP|X86_HTT)) 244 return (1); 245 else 246 return (0); 247 case PGHW_CACHE: 248 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 249 return (1); 250 else 251 return (0); 252 case PGHW_POW_ACTIVE: 253 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) 254 return (1); 255 else 256 return (0); 257 case PGHW_POW_IDLE: 258 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) 259 return (1); 260 else 261 return (0); 262 default: 263 return (0); 264 } 265 } 266 267 /* 268 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 269 * If pghw_type_t is an unsupported hardware type, then return -1 270 */ 271 int 272 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 273 { 274 id_t pgp_a, pgp_b; 275 276 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 277 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 278 279 if (pgp_a == -1 || pgp_b == -1) 280 return (-1); 281 282 return (pgp_a == pgp_b); 283 } 284 285 /* 286 * Return a physical instance identifier for known hardware sharing 287 * relationships 288 */ 289 id_t 290 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 291 { 292 switch (hw) { 293 case PGHW_IPIPE: 294 return (cpuid_get_coreid(cpu)); 295 case PGHW_CACHE: 296 return (cpuid_get_last_lvl_cacheid(cpu)); 297 case PGHW_PROCNODE: 298 return (cpuid_get_procnodeid(cpu)); 299 case PGHW_CHIP: 300 return (cpuid_get_chipid(cpu)); 301 case PGHW_POW_ACTIVE: 302 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); 303 case PGHW_POW_IDLE: 304 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); 305 default: 306 return (-1); 307 } 308 } 309 310 /* 311 * Express preference for optimizing for sharing relationship 312 * hw1 vs hw2 313 */ 314 pghw_type_t 315 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) 316 { 317 int i, rank1, rank2; 318 319 static pghw_type_t hw_hier[] = { 320 PGHW_IPIPE, 321 PGHW_CACHE, 322 PGHW_PROCNODE, 323 PGHW_CHIP, 324 PGHW_POW_IDLE, 325 PGHW_POW_ACTIVE, 326 PGHW_NUM_COMPONENTS 327 }; 328 329 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 330 if (hw_hier[i] == hw1) 331 rank1 = i; 332 if (hw_hier[i] == hw2) 333 rank2 = i; 334 } 335 336 if (rank1 > rank2) 337 return (hw1); 338 else 339 return (hw2); 340 } 341 342 /* 343 * Override the default CMT dispatcher policy for the specified 344 * hardware sharing relationship 345 */ 346 pg_cmt_policy_t 347 pg_plat_cmt_policy(pghw_type_t hw) 348 { 349 /* 350 * For shared caches, also load balance across them to 351 * maximize aggregate cache capacity 352 */ 353 switch (hw) { 354 case PGHW_CACHE: 355 return (CMT_BALANCE|CMT_AFFINITY); 356 default: 357 return (CMT_NO_POLICY); 358 } 359 } 360 361 id_t 362 pg_plat_get_core_id(cpu_t *cpu) 363 { 364 return ((id_t)cpuid_get_coreid(cpu)); 365 } 366 367 void 368 cmp_set_nosteal_interval(void) 369 { 370 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 371 nosteal_nsec = 100000UL; 372 } 373 374 /* 375 * Routine to ensure initial callers to hrtime gets 0 as return 376 */ 377 static hrtime_t 378 dummy_hrtime(void) 379 { 380 return (0); 381 } 382 383 /* ARGSUSED */ 384 static void 385 dummy_scalehrtime(hrtime_t *ticks) 386 {} 387 388 static uint64_t 389 dummy_unscalehrtime(hrtime_t nsecs) 390 { 391 return ((uint64_t)nsecs); 392 } 393 394 /* 395 * Supports Deep C-State power saving idle loop. 396 */ 397 void 398 cpu_idle_adaptive(void) 399 { 400 (*CPU->cpu_m.mcpu_idle_cpu)(); 401 } 402 403 /* 404 * Function called by CPU idle notification framework to check whether CPU 405 * has been awakened. It will be called with interrupt disabled. 406 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 407 * notification framework. 408 */ 409 /*ARGSUSED*/ 410 static void 411 cpu_idle_check_wakeup(void *arg) 412 { 413 /* 414 * Toggle interrupt flag to detect pending interrupts. 415 * If interrupt happened, do_interrupt() will notify CPU idle 416 * notification framework so no need to call cpu_idle_exit() here. 417 */ 418 sti(); 419 SMT_PAUSE(); 420 cli(); 421 } 422 423 /* 424 * Idle the present CPU until wakened via an interrupt 425 */ 426 void 427 cpu_idle(void) 428 { 429 cpu_t *cpup = CPU; 430 processorid_t cpu_sid = cpup->cpu_seqid; 431 cpupart_t *cp = cpup->cpu_part; 432 int hset_update = 1; 433 434 /* 435 * If this CPU is online, and there's multiple CPUs 436 * in the system, then we should notate our halting 437 * by adding ourselves to the partition's halted CPU 438 * bitmap. This allows other CPUs to find/awaken us when 439 * work becomes available. 440 */ 441 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 442 hset_update = 0; 443 444 /* 445 * Add ourselves to the partition's halted CPUs bitmap 446 * and set our HALTED flag, if necessary. 447 * 448 * When a thread becomes runnable, it is placed on the queue 449 * and then the halted CPU bitmap is checked to determine who 450 * (if anyone) should be awakened. We therefore need to first 451 * add ourselves to the bitmap, and and then check if there 452 * is any work available. The order is important to prevent a race 453 * that can lead to work languishing on a run queue somewhere while 454 * this CPU remains halted. 455 * 456 * Either the producing CPU will see we're halted and will awaken us, 457 * or this CPU will see the work available in disp_anywork(). 458 * 459 * Note that memory barriers after updating the HALTED flag 460 * are not necessary since an atomic operation (updating the bitset) 461 * immediately follows. On x86 the atomic operation acts as a 462 * memory barrier for the update of cpu_disp_flags. 463 */ 464 if (hset_update) { 465 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 466 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 467 } 468 469 /* 470 * Check to make sure there's really nothing to do. 471 * Work destined for this CPU may become available after 472 * this check. We'll be notified through the clearing of our 473 * bit in the halted CPU bitmap, and a poke. 474 */ 475 if (disp_anywork()) { 476 if (hset_update) { 477 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 478 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 479 } 480 return; 481 } 482 483 /* 484 * We're on our way to being halted. 485 * 486 * Disable interrupts now, so that we'll awaken immediately 487 * after halting if someone tries to poke us between now and 488 * the time we actually halt. 489 * 490 * We check for the presence of our bit after disabling interrupts. 491 * If it's cleared, we'll return. If the bit is cleared after 492 * we check then the poke will pop us out of the halted state. 493 * 494 * This means that the ordering of the poke and the clearing 495 * of the bit by cpu_wakeup is important. 496 * cpu_wakeup() must clear, then poke. 497 * cpu_idle() must disable interrupts, then check for the bit. 498 */ 499 cli(); 500 501 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { 502 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 503 sti(); 504 return; 505 } 506 507 /* 508 * The check for anything locally runnable is here for performance 509 * and isn't needed for correctness. disp_nrunnable ought to be 510 * in our cache still, so it's inexpensive to check, and if there 511 * is anything runnable we won't have to wait for the poke. 512 */ 513 if (cpup->cpu_disp->disp_nrunnable != 0) { 514 if (hset_update) { 515 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 516 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 517 } 518 sti(); 519 return; 520 } 521 522 if (cpu_idle_enter(IDLE_STATE_C1, 0, 523 cpu_idle_check_wakeup, NULL) == 0) { 524 mach_cpu_idle(); 525 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 526 } 527 528 /* 529 * We're no longer halted 530 */ 531 if (hset_update) { 532 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 533 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 534 } 535 } 536 537 538 /* 539 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 540 * Otherwise, see if other CPUs in the cpu partition are halted and need to 541 * be woken up so that they can steal the thread we placed on this CPU. 542 * This function is only used on MP systems. 543 */ 544 static void 545 cpu_wakeup(cpu_t *cpu, int bound) 546 { 547 uint_t cpu_found; 548 processorid_t cpu_sid; 549 cpupart_t *cp; 550 551 cp = cpu->cpu_part; 552 cpu_sid = cpu->cpu_seqid; 553 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 554 /* 555 * Clear the halted bit for that CPU since it will be 556 * poked in a moment. 557 */ 558 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 559 /* 560 * We may find the current CPU present in the halted cpuset 561 * if we're in the context of an interrupt that occurred 562 * before we had a chance to clear our bit in cpu_idle(). 563 * Poking ourself is obviously unnecessary, since if 564 * we're here, we're not halted. 565 */ 566 if (cpu != CPU) 567 poke_cpu(cpu->cpu_id); 568 return; 569 } else { 570 /* 571 * This cpu isn't halted, but it's idle or undergoing a 572 * context switch. No need to awaken anyone else. 573 */ 574 if (cpu->cpu_thread == cpu->cpu_idle_thread || 575 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 576 return; 577 } 578 579 /* 580 * No need to wake up other CPUs if this is for a bound thread. 581 */ 582 if (bound) 583 return; 584 585 /* 586 * The CPU specified for wakeup isn't currently halted, so check 587 * to see if there are any other halted CPUs in the partition, 588 * and if there are then awaken one. 589 */ 590 do { 591 cpu_found = bitset_find(&cp->cp_haltset); 592 if (cpu_found == (uint_t)-1) 593 return; 594 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 595 596 if (cpu_found != CPU->cpu_seqid) { 597 poke_cpu(cpu_seq[cpu_found]->cpu_id); 598 } 599 } 600 601 #ifndef __xpv 602 /* 603 * Function called by CPU idle notification framework to check whether CPU 604 * has been awakened. It will be called with interrupt disabled. 605 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 606 * notification framework. 607 */ 608 static void 609 cpu_idle_mwait_check_wakeup(void *arg) 610 { 611 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; 612 613 ASSERT(arg != NULL); 614 if (*mcpu_mwait != MWAIT_HALTED) { 615 /* 616 * CPU has been awakened, notify CPU idle notification system. 617 */ 618 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 619 } else { 620 /* 621 * Toggle interrupt flag to detect pending interrupts. 622 * If interrupt happened, do_interrupt() will notify CPU idle 623 * notification framework so no need to call cpu_idle_exit() 624 * here. 625 */ 626 sti(); 627 SMT_PAUSE(); 628 cli(); 629 } 630 } 631 632 /* 633 * Idle the present CPU until awakened via touching its monitored line 634 */ 635 void 636 cpu_idle_mwait(void) 637 { 638 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 639 cpu_t *cpup = CPU; 640 processorid_t cpu_sid = cpup->cpu_seqid; 641 cpupart_t *cp = cpup->cpu_part; 642 int hset_update = 1; 643 644 /* 645 * Set our mcpu_mwait here, so we can tell if anyone tries to 646 * wake us between now and when we call mwait. No other cpu will 647 * attempt to set our mcpu_mwait until we add ourself to the halted 648 * CPU bitmap. 649 */ 650 *mcpu_mwait = MWAIT_HALTED; 651 652 /* 653 * If this CPU is online, and there's multiple CPUs 654 * in the system, then we should note our halting 655 * by adding ourselves to the partition's halted CPU 656 * bitmap. This allows other CPUs to find/awaken us when 657 * work becomes available. 658 */ 659 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 660 hset_update = 0; 661 662 /* 663 * Add ourselves to the partition's halted CPUs bitmap 664 * and set our HALTED flag, if necessary. 665 * 666 * When a thread becomes runnable, it is placed on the queue 667 * and then the halted CPU bitmap is checked to determine who 668 * (if anyone) should be awakened. We therefore need to first 669 * add ourselves to the bitmap, and and then check if there 670 * is any work available. 671 * 672 * Note that memory barriers after updating the HALTED flag 673 * are not necessary since an atomic operation (updating the bitmap) 674 * immediately follows. On x86 the atomic operation acts as a 675 * memory barrier for the update of cpu_disp_flags. 676 */ 677 if (hset_update) { 678 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 679 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 680 } 681 682 /* 683 * Check to make sure there's really nothing to do. 684 * Work destined for this CPU may become available after 685 * this check. We'll be notified through the clearing of our 686 * bit in the halted CPU bitmap, and a write to our mcpu_mwait. 687 * 688 * disp_anywork() checks disp_nrunnable, so we do not have to later. 689 */ 690 if (disp_anywork()) { 691 if (hset_update) { 692 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 693 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 694 } 695 return; 696 } 697 698 /* 699 * We're on our way to being halted. 700 * To avoid a lost wakeup, arm the monitor before checking if another 701 * cpu wrote to mcpu_mwait to wake us up. 702 */ 703 i86_monitor(mcpu_mwait, 0, 0); 704 if (*mcpu_mwait == MWAIT_HALTED) { 705 if (cpu_idle_enter(IDLE_STATE_C1, 0, 706 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { 707 if (*mcpu_mwait == MWAIT_HALTED) { 708 i86_mwait(0, 0); 709 } 710 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 711 } 712 } 713 714 /* 715 * We're no longer halted 716 */ 717 if (hset_update) { 718 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 719 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 720 } 721 } 722 723 /* 724 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 725 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 726 * need to be woken up so that they can steal the thread we placed on this CPU. 727 * This function is only used on MP systems. 728 */ 729 static void 730 cpu_wakeup_mwait(cpu_t *cp, int bound) 731 { 732 cpupart_t *cpu_part; 733 uint_t cpu_found; 734 processorid_t cpu_sid; 735 736 cpu_part = cp->cpu_part; 737 cpu_sid = cp->cpu_seqid; 738 739 /* 740 * Clear the halted bit for that CPU since it will be woken up 741 * in a moment. 742 */ 743 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { 744 /* 745 * Clear the halted bit for that CPU since it will be 746 * poked in a moment. 747 */ 748 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); 749 /* 750 * We may find the current CPU present in the halted cpuset 751 * if we're in the context of an interrupt that occurred 752 * before we had a chance to clear our bit in cpu_idle(). 753 * Waking ourself is obviously unnecessary, since if 754 * we're here, we're not halted. 755 * 756 * monitor/mwait wakeup via writing to our cache line is 757 * harmless and less expensive than always checking if we 758 * are waking ourself which is an uncommon case. 759 */ 760 MWAIT_WAKEUP(cp); /* write to monitored line */ 761 return; 762 } else { 763 /* 764 * This cpu isn't halted, but it's idle or undergoing a 765 * context switch. No need to awaken anyone else. 766 */ 767 if (cp->cpu_thread == cp->cpu_idle_thread || 768 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 769 return; 770 } 771 772 /* 773 * No need to wake up other CPUs if the thread we just enqueued 774 * is bound. 775 */ 776 if (bound || ncpus == 1) 777 return; 778 779 /* 780 * See if there's any other halted CPUs. If there are, then 781 * select one, and awaken it. 782 * It's possible that after we find a CPU, somebody else 783 * will awaken it before we get the chance. 784 * In that case, look again. 785 */ 786 do { 787 cpu_found = bitset_find(&cpu_part->cp_haltset); 788 if (cpu_found == (uint_t)-1) 789 return; 790 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, 791 cpu_found) < 0); 792 793 /* 794 * Do not check if cpu_found is ourself as monitor/mwait 795 * wakeup is cheap. 796 */ 797 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ 798 } 799 800 #endif 801 802 void (*cpu_pause_handler)(volatile char *) = NULL; 803 804 static int 805 mp_disable_intr(int cpun) 806 { 807 /* 808 * switch to the offline cpu 809 */ 810 affinity_set(cpun); 811 /* 812 * raise ipl to just below cross call 813 */ 814 splx(XC_SYS_PIL - 1); 815 /* 816 * set base spl to prevent the next swtch to idle from 817 * lowering back to ipl 0 818 */ 819 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); 820 set_base_spl(); 821 affinity_clear(); 822 return (DDI_SUCCESS); 823 } 824 825 static void 826 mp_enable_intr(int cpun) 827 { 828 /* 829 * switch to the online cpu 830 */ 831 affinity_set(cpun); 832 /* 833 * clear the interrupt active mask 834 */ 835 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); 836 set_base_spl(); 837 (void) spl0(); 838 affinity_clear(); 839 } 840 841 static void 842 mach_get_platform(int owner) 843 { 844 void **srv_opsp; 845 void **clt_opsp; 846 int i; 847 int total_ops; 848 849 /* fix up psm ops */ 850 srv_opsp = (void **)mach_set[0]; 851 clt_opsp = (void **)mach_set[owner]; 852 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 853 total_ops = sizeof (struct psm_ops_ver01) / 854 sizeof (void (*)(void)); 855 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 856 /* no psm_notify_func */ 857 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 858 sizeof (void (*)(void)); 859 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 860 /* no psm_timer funcs */ 861 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 862 sizeof (void (*)(void)); 863 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 864 /* no psm_preshutdown function */ 865 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 866 sizeof (void (*)(void)); 867 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 868 /* no psm_intr_ops function */ 869 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 870 sizeof (void (*)(void)); 871 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_5) 872 /* no psm_state function */ 873 total_ops = OFFSETOF(struct psm_ops, psm_state) / 874 sizeof (void (*)(void)); 875 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_6) 876 /* no psm_cpu_ops function */ 877 total_ops = OFFSETOF(struct psm_ops, psm_cpu_ops) / 878 sizeof (void (*)(void)); 879 else 880 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 881 882 /* 883 * Save the version of the PSM module, in case we need to 884 * behave differently based on version. 885 */ 886 mach_ver[0] = mach_ver[owner]; 887 888 for (i = 0; i < total_ops; i++) 889 if (clt_opsp[i] != NULL) 890 srv_opsp[i] = clt_opsp[i]; 891 } 892 893 static void 894 mach_construct_info() 895 { 896 struct psm_sw *swp; 897 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 898 int conflict_owner = 0; 899 900 if (psmsw->psw_forw == psmsw) 901 panic("No valid PSM modules found"); 902 mutex_enter(&psmsw_lock); 903 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 904 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 905 continue; 906 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 907 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 908 mach_cnt[swp->psw_infop->p_owner]++; 909 } 910 mutex_exit(&psmsw_lock); 911 912 mach_get_platform(PSM_OWN_SYS_DEFAULT); 913 914 /* check to see are there any conflicts */ 915 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 916 conflict_owner = PSM_OWN_EXCLUSIVE; 917 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 918 conflict_owner = PSM_OWN_OVERRIDE; 919 if (conflict_owner) { 920 /* remove all psm modules except uppc */ 921 cmn_err(CE_WARN, 922 "Conflicts detected on the following PSM modules:"); 923 mutex_enter(&psmsw_lock); 924 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 925 if (swp->psw_infop->p_owner == conflict_owner) 926 cmn_err(CE_WARN, "%s ", 927 swp->psw_infop->p_mach_idstring); 928 } 929 mutex_exit(&psmsw_lock); 930 cmn_err(CE_WARN, 931 "Setting the system back to SINGLE processor mode!"); 932 cmn_err(CE_WARN, 933 "Please edit /etc/mach to remove the invalid PSM module."); 934 return; 935 } 936 937 if (mach_set[PSM_OWN_EXCLUSIVE]) 938 mach_get_platform(PSM_OWN_EXCLUSIVE); 939 940 if (mach_set[PSM_OWN_OVERRIDE]) 941 mach_get_platform(PSM_OWN_OVERRIDE); 942 } 943 944 static void 945 mach_init() 946 { 947 struct psm_ops *pops; 948 949 mach_construct_info(); 950 951 pops = mach_set[0]; 952 953 /* register the interrupt and clock initialization rotuines */ 954 picinitf = mach_picinit; 955 clkinitf = mach_clkinit; 956 psm_get_clockirq = pops->psm_get_clockirq; 957 958 /* register the interrupt setup code */ 959 slvltovect = mach_softlvl_to_vect; 960 addspl = pops->psm_addspl; 961 delspl = pops->psm_delspl; 962 963 if (pops->psm_translate_irq) 964 psm_translate_irq = pops->psm_translate_irq; 965 if (pops->psm_intr_ops) 966 psm_intr_ops = pops->psm_intr_ops; 967 968 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 969 /* 970 * Time-of-day functionality now handled in TOD modules. 971 * (Warn about PSM modules that think that we're going to use 972 * their ops vectors.) 973 */ 974 if (pops->psm_tod_get) 975 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 976 (void *)pops->psm_tod_get); 977 978 if (pops->psm_tod_set) 979 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 980 (void *)pops->psm_tod_set); 981 #endif 982 983 if (pops->psm_notify_error) { 984 psm_notify_error = mach_notify_error; 985 notify_error = pops->psm_notify_error; 986 } 987 988 (*pops->psm_softinit)(); 989 990 /* 991 * Initialize the dispatcher's function hooks to enable CPU halting 992 * when idle. Set both the deep-idle and non-deep-idle hooks. 993 * 994 * Assume we can use power saving deep-idle loop cpu_idle_adaptive. 995 * Platform deep-idle driver will reset our idle loop to 996 * non_deep_idle_cpu if power saving deep-idle feature is not available. 997 * 998 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) 999 * or idle_cpu_prefer_mwait is not set. 1000 * Allocate monitor/mwait buffer for cpu0. 1001 */ 1002 #ifndef __xpv 1003 non_deep_idle_disp_enq_thread = disp_enq_thread; 1004 #endif 1005 if (idle_cpu_use_hlt) { 1006 idle_cpu = cpu_idle_adaptive; 1007 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1008 #ifndef __xpv 1009 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) { 1010 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); 1011 /* 1012 * Protect ourself from insane mwait size. 1013 */ 1014 if (CPU->cpu_m.mcpu_mwait == NULL) { 1015 #ifdef DEBUG 1016 cmn_err(CE_NOTE, "Using hlt idle. Cannot " 1017 "handle cpu 0 mwait size."); 1018 #endif 1019 idle_cpu_prefer_mwait = 0; 1020 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1021 } else { 1022 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; 1023 } 1024 } else { 1025 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1026 } 1027 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; 1028 1029 /* 1030 * Disable power saving deep idle loop? 1031 */ 1032 if (idle_cpu_no_deep_c) { 1033 idle_cpu = non_deep_idle_cpu; 1034 } 1035 #endif 1036 } 1037 1038 mach_smpinit(); 1039 } 1040 1041 static void 1042 mach_smpinit(void) 1043 { 1044 struct psm_ops *pops; 1045 processorid_t cpu_id; 1046 int cnt; 1047 cpuset_t cpumask; 1048 1049 pops = mach_set[0]; 1050 CPUSET_ZERO(cpumask); 1051 1052 cpu_id = -1; 1053 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1054 /* 1055 * Only add boot_ncpus CPUs to mp_cpus. Other CPUs will be handled 1056 * by CPU DR driver at runtime. 1057 */ 1058 for (cnt = 0; cpu_id != -1 && cnt < boot_ncpus; cnt++) { 1059 CPUSET_ADD(cpumask, cpu_id); 1060 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1061 } 1062 1063 mp_cpus = cpumask; 1064 1065 /* MP related routines */ 1066 ap_mlsetup = pops->psm_post_cpu_start; 1067 send_dirintf = pops->psm_send_ipi; 1068 1069 /* optional MP related routines */ 1070 if (pops->psm_shutdown) 1071 psm_shutdownf = pops->psm_shutdown; 1072 if (pops->psm_preshutdown) 1073 psm_preshutdownf = pops->psm_preshutdown; 1074 if (pops->psm_notify_func) 1075 psm_notifyf = pops->psm_notify_func; 1076 if (pops->psm_set_idlecpu) 1077 psm_set_idle_cpuf = pops->psm_set_idlecpu; 1078 if (pops->psm_unset_idlecpu) 1079 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 1080 1081 psm_clkinit = pops->psm_clkinit; 1082 1083 if (pops->psm_timer_reprogram) 1084 psm_timer_reprogram = pops->psm_timer_reprogram; 1085 1086 if (pops->psm_timer_enable) 1087 psm_timer_enable = pops->psm_timer_enable; 1088 1089 if (pops->psm_timer_disable) 1090 psm_timer_disable = pops->psm_timer_disable; 1091 1092 if (pops->psm_post_cyclic_setup) 1093 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 1094 1095 if (pops->psm_state) 1096 psm_state = pops->psm_state; 1097 1098 /* 1099 * Set these vectors here so they can be used by Suspend/Resume 1100 * on UP machines. 1101 */ 1102 if (pops->psm_disable_intr) 1103 psm_disable_intr = pops->psm_disable_intr; 1104 if (pops->psm_enable_intr) 1105 psm_enable_intr = pops->psm_enable_intr; 1106 1107 /* check for multiple CPUs */ 1108 if (cnt < 2 && plat_dr_support_cpu() == B_FALSE) 1109 return; 1110 1111 /* check for MP platforms */ 1112 if (pops->psm_cpu_start == NULL) 1113 return; 1114 1115 /* 1116 * Set the dispatcher hook to enable cpu "wake up" 1117 * when a thread becomes runnable. 1118 */ 1119 if (idle_cpu_use_hlt) { 1120 disp_enq_thread = cpu_wakeup; 1121 #ifndef __xpv 1122 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) 1123 disp_enq_thread = cpu_wakeup_mwait; 1124 non_deep_idle_disp_enq_thread = disp_enq_thread; 1125 #endif 1126 } 1127 1128 psm_get_ipivect = pops->psm_get_ipivect; 1129 1130 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", 1131 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 1132 NULL, NULL, NULL, NULL); 1133 1134 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 1135 } 1136 1137 static void 1138 mach_picinit() 1139 { 1140 struct psm_ops *pops; 1141 1142 pops = mach_set[0]; 1143 1144 /* register the interrupt handlers */ 1145 setlvl = pops->psm_intr_enter; 1146 setlvlx = pops->psm_intr_exit; 1147 1148 /* initialize the interrupt hardware */ 1149 (*pops->psm_picinit)(); 1150 1151 /* set interrupt mask for current ipl */ 1152 setspl = pops->psm_setspl; 1153 cli(); 1154 setspl(CPU->cpu_pri); 1155 } 1156 1157 uint_t cpu_freq; /* MHz */ 1158 uint64_t cpu_freq_hz; /* measured (in hertz) */ 1159 1160 #define MEGA_HZ 1000000 1161 1162 #ifdef __xpv 1163 1164 int xpv_cpufreq_workaround = 1; 1165 int xpv_cpufreq_verbose = 0; 1166 1167 #else /* __xpv */ 1168 1169 static uint64_t 1170 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 1171 { 1172 uint64_t cpu_hz; 1173 1174 if ((pit_counter == 0) || (*processor_clks == 0) || 1175 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 1176 return (0); 1177 1178 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 1179 1180 return (cpu_hz); 1181 } 1182 1183 #endif /* __xpv */ 1184 1185 static uint64_t 1186 mach_getcpufreq(void) 1187 { 1188 #if defined(__xpv) 1189 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; 1190 uint64_t cpu_hz; 1191 1192 /* 1193 * During dom0 bringup, it was noted that on at least one older 1194 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul 1195 * value that is quite wrong (the 3.06GHz clock was reported 1196 * as 4.77GHz) 1197 * 1198 * The curious thing is, that if you stop the kernel at entry, 1199 * breakpoint here and inspect the value with kmdb, the value 1200 * is correct - but if you don't stop and simply enable the 1201 * printf statement (below), you can see the bad value printed 1202 * here. Almost as if something kmdb did caused the hypervisor to 1203 * figure it out correctly. And, note that the hypervisor 1204 * eventually -does- figure it out correctly ... if you look at 1205 * the field later in the life of dom0, it is correct. 1206 * 1207 * For now, on dom0, we employ a slightly cheesy workaround of 1208 * using the DOM0_PHYSINFO hypercall. 1209 */ 1210 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { 1211 cpu_hz = 1000 * xpv_cpu_khz(); 1212 } else { 1213 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; 1214 1215 if (vti->tsc_shift < 0) 1216 cpu_hz <<= -vti->tsc_shift; 1217 else 1218 cpu_hz >>= vti->tsc_shift; 1219 } 1220 1221 if (xpv_cpufreq_verbose) 1222 printf("mach_getcpufreq: system_mul 0x%x, shift %d, " 1223 "cpu_hz %" PRId64 "Hz\n", 1224 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); 1225 1226 return (cpu_hz); 1227 #else /* __xpv */ 1228 uint32_t pit_counter; 1229 uint64_t processor_clks; 1230 1231 if (x86_feature & X86_TSC) { 1232 /* 1233 * We have a TSC. freq_tsc() knows how to measure the number 1234 * of clock cycles sampled against the PIT. 1235 */ 1236 ulong_t flags = clear_int_flag(); 1237 processor_clks = freq_tsc(&pit_counter); 1238 restore_int_flag(flags); 1239 return (mach_calchz(pit_counter, &processor_clks)); 1240 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1241 #if defined(__amd64) 1242 panic("mach_getcpufreq: no TSC!"); 1243 #elif defined(__i386) 1244 /* 1245 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1246 * for which freq_notsc() knows how to measure the number of 1247 * elapsed clock cycles sampled against the PIT 1248 */ 1249 ulong_t flags = clear_int_flag(); 1250 processor_clks = freq_notsc(&pit_counter); 1251 restore_int_flag(flags); 1252 return (mach_calchz(pit_counter, &processor_clks)); 1253 #endif /* __i386 */ 1254 } 1255 1256 /* We do not know how to calculate cpu frequency for this cpu. */ 1257 return (0); 1258 #endif /* __xpv */ 1259 } 1260 1261 /* 1262 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1263 * to this array, instead improve the accuracy of the algorithm that determines 1264 * the clock speed of the processor or extend the implementation to support the 1265 * vendor as appropriate. This is here only to support adjusting the speed on 1266 * older slower processors that mach_fixcpufreq() would not be able to account 1267 * for otherwise. 1268 */ 1269 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1270 1271 /* 1272 * On fast processors the clock frequency that is measured may be off by 1273 * a few MHz from the value printed on the part. This is a combination of 1274 * the factors that for such fast parts being off by this much is within 1275 * the tolerances for manufacture and because of the difficulties in the 1276 * measurement that can lead to small error. This function uses some 1277 * heuristics in order to tweak the value that was measured to match what 1278 * is most likely printed on the part. 1279 * 1280 * Some examples: 1281 * AMD Athlon 1000 mhz measured as 998 mhz 1282 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1283 * Intel Pentium IV 1500 mhz measured as 1495mhz 1284 * 1285 * If in the future this function is no longer sufficient to correct 1286 * for the error in the measurement, then the algorithm used to perform 1287 * the measurement will have to be improved in order to increase accuracy 1288 * rather than adding horrible and questionable kludges here. 1289 * 1290 * This is called after the cyclics subsystem because of the potential 1291 * that the heuristics within may give a worse estimate of the clock 1292 * frequency than the value that was measured. 1293 */ 1294 static void 1295 mach_fixcpufreq(void) 1296 { 1297 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1298 1299 freq = (uint32_t)cpu_freq; 1300 1301 /* 1302 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1303 * measured speed taking into account that the 667 MHz parts were 1304 * the first to round-up. 1305 */ 1306 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1307 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1308 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1309 1310 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1311 mul = (freq + 25) / 50; 1312 near50 = mul * 50; 1313 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1314 1315 /* Find the closer of the two */ 1316 if (delta66 < delta50) { 1317 fixed = near66; 1318 delta = delta66; 1319 } else { 1320 fixed = near50; 1321 delta = delta50; 1322 } 1323 1324 if (fixed > INT_MAX) 1325 return; 1326 1327 /* 1328 * Some older parts have a core clock frequency that is not an 1329 * integral multiple of 50 or 66 MHz. Check if one of the old 1330 * clock frequencies is closer to the measured value than any 1331 * of the integral multiples of 50 an 66, and if so set fixed 1332 * and delta appropriately to represent the closest value. 1333 */ 1334 i = sizeof (x86_cpu_freq) / sizeof (int); 1335 while (i > 0) { 1336 i--; 1337 1338 if (x86_cpu_freq[i] <= freq) { 1339 mul = freq - x86_cpu_freq[i]; 1340 1341 if (mul < delta) { 1342 fixed = x86_cpu_freq[i]; 1343 delta = mul; 1344 } 1345 1346 break; 1347 } 1348 1349 mul = x86_cpu_freq[i] - freq; 1350 1351 if (mul < delta) { 1352 fixed = x86_cpu_freq[i]; 1353 delta = mul; 1354 } 1355 } 1356 1357 /* 1358 * Set a reasonable maximum for how much to correct the measured 1359 * result by. This check is here to prevent the adjustment made 1360 * by this function from being more harm than good. It is entirely 1361 * possible that in the future parts will be made that are not 1362 * integral multiples of 66 or 50 in clock frequency or that 1363 * someone may overclock a part to some odd frequency. If the 1364 * measured value is farther from the corrected value than 1365 * allowed, then assume the corrected value is in error and use 1366 * the measured value. 1367 */ 1368 if (6 < delta) 1369 return; 1370 1371 cpu_freq = (int)fixed; 1372 } 1373 1374 1375 static int 1376 machhztomhz(uint64_t cpu_freq_hz) 1377 { 1378 uint64_t cpu_mhz; 1379 1380 /* Round to nearest MHZ */ 1381 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1382 1383 if (cpu_mhz > INT_MAX) 1384 return (0); 1385 1386 return ((int)cpu_mhz); 1387 1388 } 1389 1390 1391 static int 1392 mach_clkinit(int preferred_mode, int *set_mode) 1393 { 1394 struct psm_ops *pops; 1395 int resolution; 1396 1397 pops = mach_set[0]; 1398 1399 cpu_freq_hz = mach_getcpufreq(); 1400 1401 cpu_freq = machhztomhz(cpu_freq_hz); 1402 1403 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 1404 tsc_gethrtime_enable = 0; 1405 1406 #ifndef __xpv 1407 if (tsc_gethrtime_enable) { 1408 tsc_hrtimeinit(cpu_freq_hz); 1409 } else 1410 #endif 1411 { 1412 if (pops->psm_hrtimeinit) 1413 (*pops->psm_hrtimeinit)(); 1414 gethrtimef = pops->psm_gethrtime; 1415 gethrtimeunscaledf = gethrtimef; 1416 /* scalehrtimef will remain dummy */ 1417 } 1418 1419 mach_fixcpufreq(); 1420 1421 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1422 if (preferred_mode == TIMER_ONESHOT) { 1423 1424 resolution = (*pops->psm_clkinit)(0); 1425 if (resolution != 0) { 1426 *set_mode = TIMER_ONESHOT; 1427 return (resolution); 1428 } 1429 } 1430 1431 /* 1432 * either periodic mode was requested or could not set to 1433 * one-shot mode 1434 */ 1435 resolution = (*pops->psm_clkinit)(hz); 1436 /* 1437 * psm should be able to do periodic, so we do not check 1438 * for return value of psm_clkinit here. 1439 */ 1440 *set_mode = TIMER_PERIODIC; 1441 return (resolution); 1442 } else { 1443 /* 1444 * PSMI interface prior to PSMI_3 does not define a return 1445 * value for psm_clkinit, so the return value is ignored. 1446 */ 1447 (void) (*pops->psm_clkinit)(hz); 1448 *set_mode = TIMER_PERIODIC; 1449 return (nsec_per_tick); 1450 } 1451 } 1452 1453 1454 /*ARGSUSED*/ 1455 static int 1456 mach_softlvl_to_vect(int ipl) 1457 { 1458 setsoftint = av_set_softint_pending; 1459 kdisetsoftint = kdi_av_set_softint_pending; 1460 1461 return (PSM_SV_SOFTWARE); 1462 } 1463 1464 #ifdef DEBUG 1465 /* 1466 * This is here to allow us to simulate cpus that refuse to start. 1467 */ 1468 cpuset_t cpufailset; 1469 #endif 1470 1471 int 1472 mach_cpu_start(struct cpu *cp, void *ctx) 1473 { 1474 struct psm_ops *pops = mach_set[0]; 1475 processorid_t id = cp->cpu_id; 1476 1477 #ifdef DEBUG 1478 if (CPU_IN_SET(cpufailset, id)) 1479 return (0); 1480 #endif 1481 return ((*pops->psm_cpu_start)(id, ctx)); 1482 } 1483 1484 int 1485 mach_cpuid_start(processorid_t id, void *ctx) 1486 { 1487 struct psm_ops *pops = mach_set[0]; 1488 1489 #ifdef DEBUG 1490 if (CPU_IN_SET(cpufailset, id)) 1491 return (0); 1492 #endif 1493 return ((*pops->psm_cpu_start)(id, ctx)); 1494 } 1495 1496 int 1497 mach_cpu_stop(cpu_t *cp, void *ctx) 1498 { 1499 struct psm_ops *pops = mach_set[0]; 1500 psm_cpu_request_t request; 1501 1502 if (pops->psm_cpu_ops == NULL) { 1503 return (ENOTSUP); 1504 } 1505 1506 ASSERT(cp->cpu_id != -1); 1507 request.pcr_cmd = PSM_CPU_STOP; 1508 request.req.cpu_stop.cpuid = cp->cpu_id; 1509 request.req.cpu_stop.ctx = ctx; 1510 1511 return ((*pops->psm_cpu_ops)(&request)); 1512 } 1513 1514 int 1515 mach_cpu_add(mach_cpu_add_arg_t *argp, processorid_t *cpuidp) 1516 { 1517 int rc; 1518 struct psm_ops *pops = mach_set[0]; 1519 psm_cpu_request_t request; 1520 1521 if (pops->psm_cpu_ops == NULL) { 1522 return (ENOTSUP); 1523 } 1524 1525 request.pcr_cmd = PSM_CPU_ADD; 1526 request.req.cpu_add.argp = argp; 1527 request.req.cpu_add.cpuid = -1; 1528 rc = (*pops->psm_cpu_ops)(&request); 1529 if (rc == 0) { 1530 ASSERT(request.req.cpu_add.cpuid != -1); 1531 *cpuidp = request.req.cpu_add.cpuid; 1532 } 1533 1534 return (rc); 1535 } 1536 1537 int 1538 mach_cpu_remove(processorid_t cpuid) 1539 { 1540 struct psm_ops *pops = mach_set[0]; 1541 psm_cpu_request_t request; 1542 1543 if (pops->psm_cpu_ops == NULL) { 1544 return (ENOTSUP); 1545 } 1546 1547 request.pcr_cmd = PSM_CPU_REMOVE; 1548 request.req.cpu_remove.cpuid = cpuid; 1549 1550 return ((*pops->psm_cpu_ops)(&request)); 1551 } 1552 1553 /* 1554 * Default handler to create device node for CPU. 1555 * One reference count will be held on created device node. 1556 */ 1557 static int 1558 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp) 1559 { 1560 int rv, circ; 1561 dev_info_t *dip; 1562 static kmutex_t cpu_node_lock; 1563 static dev_info_t *cpu_nex_devi = NULL; 1564 1565 ASSERT(cp != NULL); 1566 ASSERT(dipp != NULL); 1567 *dipp = NULL; 1568 1569 if (cpu_nex_devi == NULL) { 1570 mutex_enter(&cpu_node_lock); 1571 /* First check whether cpus exists. */ 1572 cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0); 1573 /* Create cpus if it doesn't exist. */ 1574 if (cpu_nex_devi == NULL) { 1575 ndi_devi_enter(ddi_root_node(), &circ); 1576 rv = ndi_devi_alloc(ddi_root_node(), "cpus", 1577 (pnode_t)DEVI_SID_NODEID, &dip); 1578 if (rv != NDI_SUCCESS) { 1579 mutex_exit(&cpu_node_lock); 1580 cmn_err(CE_CONT, 1581 "?failed to create cpu nexus device.\n"); 1582 return (PSM_FAILURE); 1583 } 1584 ASSERT(dip != NULL); 1585 (void) ndi_devi_online(dip, 0); 1586 ndi_devi_exit(ddi_root_node(), circ); 1587 cpu_nex_devi = dip; 1588 } 1589 mutex_exit(&cpu_node_lock); 1590 } 1591 1592 /* 1593 * create a child node for cpu identified as 'cpu_id' 1594 */ 1595 ndi_devi_enter(cpu_nex_devi, &circ); 1596 dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1); 1597 if (dip == NULL) { 1598 cmn_err(CE_CONT, 1599 "?failed to create device node for cpu%d.\n", cp->cpu_id); 1600 rv = PSM_FAILURE; 1601 } else { 1602 *dipp = dip; 1603 (void) ndi_hold_devi(dip); 1604 rv = PSM_SUCCESS; 1605 } 1606 ndi_devi_exit(cpu_nex_devi, circ); 1607 1608 return (rv); 1609 } 1610 1611 /* 1612 * Create cpu device node in device tree and online it. 1613 * Return created dip with reference count held if requested. 1614 */ 1615 int 1616 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp) 1617 { 1618 int rv; 1619 dev_info_t *dip = NULL; 1620 1621 ASSERT(psm_cpu_create_devinfo != NULL); 1622 rv = psm_cpu_create_devinfo(cp, &dip); 1623 if (rv == PSM_SUCCESS) { 1624 cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi); 1625 /* Recursively attach driver for parent nexus device. */ 1626 if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) == 1627 DDI_SUCCESS) { 1628 /* Configure cpu itself and descendants. */ 1629 (void) ndi_devi_online(dip, 1630 NDI_ONLINE_ATTACH | NDI_CONFIG); 1631 } 1632 if (dipp != NULL) { 1633 *dipp = dip; 1634 } else { 1635 (void) ndi_rele_devi(dip); 1636 } 1637 } 1638 1639 return (rv); 1640 } 1641 1642 /* 1643 * The dipp contains one of following values on return: 1644 * - NULL if no device node found 1645 * - pointer to device node if found 1646 */ 1647 int 1648 mach_cpu_get_device_node(struct cpu *cp, dev_info_t **dipp) 1649 { 1650 *dipp = NULL; 1651 if (psm_cpu_get_devinfo != NULL) { 1652 if (psm_cpu_get_devinfo(cp, dipp) == PSM_SUCCESS) { 1653 return (PSM_SUCCESS); 1654 } 1655 } 1656 1657 return (PSM_FAILURE); 1658 } 1659 1660 /*ARGSUSED*/ 1661 static int 1662 mach_translate_irq(dev_info_t *dip, int irqno) 1663 { 1664 return (irqno); /* default to NO translation */ 1665 } 1666 1667 static void 1668 mach_notify_error(int level, char *errmsg) 1669 { 1670 /* 1671 * SL_FATAL is pass in once panicstr is set, deliver it 1672 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1673 * codes for the psmi handler 1674 */ 1675 if (level & SL_FATAL) 1676 (*notify_error)(CE_PANIC, errmsg); 1677 else if (level & SL_WARN) 1678 (*notify_error)(CE_WARN, errmsg); 1679 else if (level & SL_NOTE) 1680 (*notify_error)(CE_NOTE, errmsg); 1681 else if (level & SL_CONSOLE) 1682 (*notify_error)(CE_CONT, errmsg); 1683 } 1684 1685 /* 1686 * It provides the default basic intr_ops interface for the new DDI 1687 * interrupt framework if the PSM doesn't have one. 1688 * 1689 * Input: 1690 * dip - pointer to the dev_info structure of the requested device 1691 * hdlp - pointer to the internal interrupt handle structure for the 1692 * requested interrupt 1693 * intr_op - opcode for this call 1694 * result - pointer to the integer that will hold the result to be 1695 * passed back if return value is PSM_SUCCESS 1696 * 1697 * Output: 1698 * return value is either PSM_SUCCESS or PSM_FAILURE 1699 */ 1700 static int 1701 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1702 psm_intr_op_t intr_op, int *result) 1703 { 1704 struct intrspec *ispec; 1705 1706 switch (intr_op) { 1707 case PSM_INTR_OP_CHECK_MSI: 1708 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1709 DDI_INTR_TYPE_MSIX); 1710 break; 1711 case PSM_INTR_OP_ALLOC_VECTORS: 1712 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1713 *result = 1; 1714 else 1715 *result = 0; 1716 break; 1717 case PSM_INTR_OP_FREE_VECTORS: 1718 break; 1719 case PSM_INTR_OP_NAVAIL_VECTORS: 1720 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1721 *result = 1; 1722 else 1723 *result = 0; 1724 break; 1725 case PSM_INTR_OP_XLATE_VECTOR: 1726 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1727 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1728 break; 1729 case PSM_INTR_OP_GET_CAP: 1730 *result = 0; 1731 break; 1732 case PSM_INTR_OP_GET_PENDING: 1733 case PSM_INTR_OP_CLEAR_MASK: 1734 case PSM_INTR_OP_SET_MASK: 1735 case PSM_INTR_OP_GET_SHARED: 1736 case PSM_INTR_OP_SET_PRI: 1737 case PSM_INTR_OP_SET_CAP: 1738 case PSM_INTR_OP_SET_CPU: 1739 case PSM_INTR_OP_GET_INTR: 1740 default: 1741 return (PSM_FAILURE); 1742 } 1743 return (PSM_SUCCESS); 1744 } 1745 /* 1746 * Return 1 if CMT load balancing policies should be 1747 * implemented across instances of the specified hardware 1748 * sharing relationship. 1749 */ 1750 int 1751 pg_cmt_load_bal_hw(pghw_type_t hw) 1752 { 1753 if (hw == PGHW_IPIPE || 1754 hw == PGHW_FPU || 1755 hw == PGHW_PROCNODE || 1756 hw == PGHW_CHIP) 1757 return (1); 1758 else 1759 return (0); 1760 } 1761 /* 1762 * Return 1 if thread affinity polices should be implemented 1763 * for instances of the specifed hardware sharing relationship. 1764 */ 1765 int 1766 pg_cmt_affinity_hw(pghw_type_t hw) 1767 { 1768 if (hw == PGHW_CACHE) 1769 return (1); 1770 else 1771 return (0); 1772 } 1773 1774 /* 1775 * Return number of counter events requested to measure hardware capacity and 1776 * utilization and setup CPC requests for specified CPU as needed 1777 * 1778 * May return 0 when platform or processor specific code knows that no CPC 1779 * events should be programmed on this CPU or -1 when platform or processor 1780 * specific code doesn't know which counter events are best to use and common 1781 * code should decide for itself 1782 */ 1783 int 1784 /* LINTED E_FUNC_ARG_UNUSED */ 1785 cu_plat_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs) 1786 { 1787 const char *impl_name; 1788 1789 /* 1790 * Return error if pcbe_ops not set 1791 */ 1792 if (pcbe_ops == NULL) 1793 return (-1); 1794 1795 /* 1796 * Return that no CPC events should be programmed on hyperthreaded 1797 * Pentium 4 and return error for all other x86 processors to tell 1798 * common code to decide what counter events to program on those CPUs 1799 * for measuring hardware capacity and utilization 1800 */ 1801 impl_name = pcbe_ops->pcbe_impl_name(); 1802 if (impl_name != NULL && strcmp(impl_name, PCBE_IMPL_NAME_P4HT) == 0) 1803 return (0); 1804 else 1805 return (-1); 1806 } 1807