1 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 /* 26 * Copyright (c) 2009-2010, Intel Corporation. 27 * All rights reserved. 28 * Copyright 2018 Joyent, Inc. 29 */ 30 31 #define PSMI_1_7 32 #include <sys/smp_impldefs.h> 33 #include <sys/psm.h> 34 #include <sys/psm_modctl.h> 35 #include <sys/pit.h> 36 #include <sys/cmn_err.h> 37 #include <sys/strlog.h> 38 #include <sys/clock.h> 39 #include <sys/debug.h> 40 #include <sys/rtc.h> 41 #include <sys/x86_archext.h> 42 #include <sys/cpupart.h> 43 #include <sys/cpuvar.h> 44 #include <sys/cpu_event.h> 45 #include <sys/cmt.h> 46 #include <sys/cpu.h> 47 #include <sys/disp.h> 48 #include <sys/archsystm.h> 49 #include <sys/machsystm.h> 50 #include <sys/sysmacros.h> 51 #include <sys/memlist.h> 52 #include <sys/param.h> 53 #include <sys/promif.h> 54 #include <sys/cpu_pm.h> 55 #if defined(__xpv) 56 #include <sys/hypervisor.h> 57 #endif 58 #include <sys/mach_intr.h> 59 #include <vm/hat_i86.h> 60 #include <sys/kdi_machimpl.h> 61 #include <sys/sdt.h> 62 #include <sys/hpet.h> 63 #include <sys/sunddi.h> 64 #include <sys/sunndi.h> 65 #include <sys/cpc_pcbe.h> 66 67 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 68 69 /* 70 * Local function prototypes 71 */ 72 static int mp_disable_intr(processorid_t cpun); 73 static void mp_enable_intr(processorid_t cpun); 74 static void mach_init(); 75 static void mach_picinit(); 76 static int machhztomhz(uint64_t cpu_freq_hz); 77 static uint64_t mach_getcpufreq(void); 78 static void mach_fixcpufreq(void); 79 static int mach_clkinit(int, int *); 80 static void mach_smpinit(void); 81 static int mach_softlvl_to_vect(int ipl); 82 static void mach_get_platform(int owner); 83 static void mach_construct_info(); 84 static int mach_translate_irq(dev_info_t *dip, int irqno); 85 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 86 psm_intr_op_t, int *); 87 static void mach_notify_error(int level, char *errmsg); 88 static hrtime_t dummy_hrtime(void); 89 static void dummy_scalehrtime(hrtime_t *); 90 static uint64_t dummy_unscalehrtime(hrtime_t); 91 void cpu_idle(void); 92 static void cpu_wakeup(cpu_t *, int); 93 #ifndef __xpv 94 void cpu_idle_mwait(void); 95 static void cpu_wakeup_mwait(cpu_t *, int); 96 #endif 97 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp); 98 99 /* 100 * External reference functions 101 */ 102 extern void return_instr(); 103 extern uint64_t freq_tsc(uint32_t *); 104 #if defined(__i386) 105 extern uint64_t freq_notsc(uint32_t *); 106 #endif 107 extern void pc_gethrestime(timestruc_t *); 108 extern int cpuid_get_coreid(cpu_t *); 109 extern int cpuid_get_chipid(cpu_t *); 110 111 /* 112 * PSM functions initialization 113 */ 114 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 115 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 116 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 117 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 118 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 119 void (*psminitf)() = mach_init; 120 void (*picinitf)() = return_instr; 121 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 122 int (*ap_mlsetup)() = (int (*)(void))return_instr; 123 void (*send_dirintf)() = return_instr; 124 void (*setspl)(int) = (void (*)(int))return_instr; 125 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 126 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 127 int (*get_pending_spl)(void) = (int (*)(void))return_instr; 128 int (*addintr)(void *, int, avfunc, char *, int, caddr_t, caddr_t, 129 uint64_t *, dev_info_t *) = NULL; 130 void (*remintr)(void *, int, avfunc, int) = NULL; 131 void (*kdisetsoftint)(int, struct av_softinfo *)= 132 (void (*)(int, struct av_softinfo *))return_instr; 133 void (*setsoftint)(int, struct av_softinfo *)= 134 (void (*)(int, struct av_softinfo *))return_instr; 135 int (*slvltovect)(int) = (int (*)(int))return_instr; 136 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 137 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 138 int (*psm_disable_intr)(int) = mp_disable_intr; 139 void (*psm_enable_intr)(int) = mp_enable_intr; 140 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 141 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 142 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 143 uint64_t (*unscalehrtimef)(hrtime_t) = dummy_unscalehrtime; 144 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 145 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 146 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 147 int (*psm_get_clockirq)(int) = NULL; 148 int (*psm_get_ipivect)(int, int) = NULL; 149 uchar_t (*psm_get_ioapicid)(uchar_t) = NULL; 150 uint32_t (*psm_get_localapicid)(uint32_t) = NULL; 151 uchar_t (*psm_xlate_vector_by_irq)(uchar_t) = NULL; 152 int (*psm_get_pir_ipivect)(void) = NULL; 153 void (*psm_send_pir_ipi)(processorid_t) = NULL; 154 155 int (*psm_clkinit)(int) = NULL; 156 void (*psm_timer_reprogram)(hrtime_t) = NULL; 157 void (*psm_timer_enable)(void) = NULL; 158 void (*psm_timer_disable)(void) = NULL; 159 void (*psm_post_cyclic_setup)(void *arg) = NULL; 160 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 161 int *) = mach_intr_ops; 162 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) 163 return_instr; 164 165 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 166 void (*hrtime_tick)(void) = return_instr; 167 168 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo; 169 int (*psm_cpu_get_devinfo)(cpu_t *, dev_info_t **) = NULL; 170 171 /* global IRM pool for APIX (PSM) module */ 172 ddi_irm_pool_t *apix_irm_pool_p = NULL; 173 174 /* 175 * True if the generic TSC code is our source of hrtime, rather than whatever 176 * the PSM can provide. 177 */ 178 #ifdef __xpv 179 int tsc_gethrtime_enable = 0; 180 #else 181 int tsc_gethrtime_enable = 1; 182 #endif 183 int tsc_gethrtime_initted = 0; 184 185 /* 186 * True if the hrtime implementation is "hires"; namely, better than microdata. 187 */ 188 int gethrtime_hires = 0; 189 190 /* 191 * Local Static Data 192 */ 193 static struct psm_ops mach_ops; 194 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 195 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 196 197 /* 198 * virtualization support for psm 199 */ 200 void *psm_vt_ops = NULL; 201 /* 202 * If non-zero, idle cpus will become "halted" when there's 203 * no work to do. 204 */ 205 int idle_cpu_use_hlt = 1; 206 207 #ifndef __xpv 208 /* 209 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 210 */ 211 int idle_cpu_prefer_mwait = 1; 212 /* 213 * Set to 0 to avoid MONITOR+CLFLUSH assertion. 214 */ 215 int idle_cpu_assert_cflush_monitor = 1; 216 217 /* 218 * If non-zero, idle cpus will not use power saving Deep C-States idle loop. 219 */ 220 int idle_cpu_no_deep_c = 0; 221 /* 222 * Non-power saving idle loop and wakeup pointers. 223 * Allows user to toggle Deep Idle power saving feature on/off. 224 */ 225 void (*non_deep_idle_cpu)() = cpu_idle; 226 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); 227 228 /* 229 * Object for the kernel to access the HPET. 230 */ 231 hpet_t hpet; 232 233 #endif /* ifndef __xpv */ 234 235 uint_t cp_haltset_fanout = 0; 236 237 /*ARGSUSED*/ 238 int 239 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 240 { 241 switch (hw) { 242 case PGHW_IPIPE: 243 if (is_x86_feature(x86_featureset, X86FSET_HTT)) { 244 /* 245 * Hyper-threading is SMT 246 */ 247 return (1); 248 } else { 249 return (0); 250 } 251 case PGHW_FPU: 252 if (cpuid_get_cores_per_compunit(cp) > 1) 253 return (1); 254 else 255 return (0); 256 case PGHW_PROCNODE: 257 if (cpuid_get_procnodes_per_pkg(cp) > 1) 258 return (1); 259 else 260 return (0); 261 case PGHW_CHIP: 262 if (is_x86_feature(x86_featureset, X86FSET_CMP) || 263 is_x86_feature(x86_featureset, X86FSET_HTT)) 264 return (1); 265 else 266 return (0); 267 case PGHW_CACHE: 268 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 269 return (1); 270 else 271 return (0); 272 case PGHW_POW_ACTIVE: 273 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) 274 return (1); 275 else 276 return (0); 277 case PGHW_POW_IDLE: 278 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) 279 return (1); 280 else 281 return (0); 282 default: 283 return (0); 284 } 285 } 286 287 /* 288 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 289 * If pghw_type_t is an unsupported hardware type, then return -1 290 */ 291 int 292 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 293 { 294 id_t pgp_a, pgp_b; 295 296 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 297 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 298 299 if (pgp_a == -1 || pgp_b == -1) 300 return (-1); 301 302 return (pgp_a == pgp_b); 303 } 304 305 /* 306 * Return a physical instance identifier for known hardware sharing 307 * relationships 308 */ 309 id_t 310 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 311 { 312 switch (hw) { 313 case PGHW_IPIPE: 314 return (cpuid_get_coreid(cpu)); 315 case PGHW_CACHE: 316 return (cpuid_get_last_lvl_cacheid(cpu)); 317 case PGHW_FPU: 318 return (cpuid_get_compunitid(cpu)); 319 case PGHW_PROCNODE: 320 return (cpuid_get_procnodeid(cpu)); 321 case PGHW_CHIP: 322 return (cpuid_get_chipid(cpu)); 323 case PGHW_POW_ACTIVE: 324 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); 325 case PGHW_POW_IDLE: 326 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); 327 default: 328 return (-1); 329 } 330 } 331 332 /* 333 * Express preference for optimizing for sharing relationship 334 * hw1 vs hw2 335 */ 336 pghw_type_t 337 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) 338 { 339 int i, rank1, rank2; 340 341 static pghw_type_t hw_hier[] = { 342 PGHW_IPIPE, 343 PGHW_CACHE, 344 PGHW_FPU, 345 PGHW_PROCNODE, 346 PGHW_CHIP, 347 PGHW_POW_IDLE, 348 PGHW_POW_ACTIVE, 349 PGHW_NUM_COMPONENTS 350 }; 351 352 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 353 if (hw_hier[i] == hw1) 354 rank1 = i; 355 if (hw_hier[i] == hw2) 356 rank2 = i; 357 } 358 359 if (rank1 > rank2) 360 return (hw1); 361 else 362 return (hw2); 363 } 364 365 /* 366 * Override the default CMT dispatcher policy for the specified 367 * hardware sharing relationship 368 */ 369 pg_cmt_policy_t 370 pg_plat_cmt_policy(pghw_type_t hw) 371 { 372 /* 373 * For shared caches, also load balance across them to 374 * maximize aggregate cache capacity 375 * 376 * On AMD family 0x15 CPUs, cores come in pairs called 377 * compute units, sharing the FPU and the I$ and L2 378 * caches. Use balancing and cache affinity. 379 */ 380 switch (hw) { 381 case PGHW_FPU: 382 case PGHW_CACHE: 383 return (CMT_BALANCE|CMT_AFFINITY); 384 default: 385 return (CMT_NO_POLICY); 386 } 387 } 388 389 id_t 390 pg_plat_get_core_id(cpu_t *cpu) 391 { 392 return ((id_t)cpuid_get_coreid(cpu)); 393 } 394 395 void 396 cmp_set_nosteal_interval(void) 397 { 398 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 399 nosteal_nsec = 100000UL; 400 } 401 402 /* 403 * Routine to ensure initial callers to hrtime gets 0 as return 404 */ 405 static hrtime_t 406 dummy_hrtime(void) 407 { 408 return (0); 409 } 410 411 /* ARGSUSED */ 412 static void 413 dummy_scalehrtime(hrtime_t *ticks) 414 {} 415 416 static uint64_t 417 dummy_unscalehrtime(hrtime_t nsecs) 418 { 419 return ((uint64_t)nsecs); 420 } 421 422 /* 423 * Supports Deep C-State power saving idle loop. 424 */ 425 void 426 cpu_idle_adaptive(void) 427 { 428 (*CPU->cpu_m.mcpu_idle_cpu)(); 429 } 430 431 /* 432 * Function called by CPU idle notification framework to check whether CPU 433 * has been awakened. It will be called with interrupt disabled. 434 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 435 * notification framework. 436 */ 437 /*ARGSUSED*/ 438 static void 439 cpu_idle_check_wakeup(void *arg) 440 { 441 /* 442 * Toggle interrupt flag to detect pending interrupts. 443 * If interrupt happened, do_interrupt() will notify CPU idle 444 * notification framework so no need to call cpu_idle_exit() here. 445 */ 446 sti(); 447 SMT_PAUSE(); 448 cli(); 449 } 450 451 /* 452 * Idle the present CPU until wakened via an interrupt 453 */ 454 void 455 cpu_idle(void) 456 { 457 cpu_t *cpup = CPU; 458 processorid_t cpu_sid = cpup->cpu_seqid; 459 cpupart_t *cp = cpup->cpu_part; 460 int hset_update = 1; 461 462 /* 463 * If this CPU is online, and there's multiple CPUs 464 * in the system, then we should notate our halting 465 * by adding ourselves to the partition's halted CPU 466 * bitmap. This allows other CPUs to find/awaken us when 467 * work becomes available. 468 */ 469 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 470 hset_update = 0; 471 472 /* 473 * Add ourselves to the partition's halted CPUs bitmap 474 * and set our HALTED flag, if necessary. 475 * 476 * When a thread becomes runnable, it is placed on the queue 477 * and then the halted CPU bitmap is checked to determine who 478 * (if anyone) should be awakened. We therefore need to first 479 * add ourselves to the bitmap, and and then check if there 480 * is any work available. The order is important to prevent a race 481 * that can lead to work languishing on a run queue somewhere while 482 * this CPU remains halted. 483 * 484 * Either the producing CPU will see we're halted and will awaken us, 485 * or this CPU will see the work available in disp_anywork(). 486 * 487 * Note that memory barriers after updating the HALTED flag 488 * are not necessary since an atomic operation (updating the bitset) 489 * immediately follows. On x86 the atomic operation acts as a 490 * memory barrier for the update of cpu_disp_flags. 491 */ 492 if (hset_update) { 493 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 494 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 495 } 496 497 /* 498 * Check to make sure there's really nothing to do. 499 * Work destined for this CPU may become available after 500 * this check. We'll be notified through the clearing of our 501 * bit in the halted CPU bitmap, and a poke. 502 */ 503 if (disp_anywork()) { 504 if (hset_update) { 505 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 506 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 507 } 508 return; 509 } 510 511 /* 512 * We're on our way to being halted. 513 * 514 * Disable interrupts now, so that we'll awaken immediately 515 * after halting if someone tries to poke us between now and 516 * the time we actually halt. 517 * 518 * We check for the presence of our bit after disabling interrupts. 519 * If it's cleared, we'll return. If the bit is cleared after 520 * we check then the poke will pop us out of the halted state. 521 * 522 * This means that the ordering of the poke and the clearing 523 * of the bit by cpu_wakeup is important. 524 * cpu_wakeup() must clear, then poke. 525 * cpu_idle() must disable interrupts, then check for the bit. 526 */ 527 cli(); 528 529 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { 530 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 531 sti(); 532 return; 533 } 534 535 /* 536 * The check for anything locally runnable is here for performance 537 * and isn't needed for correctness. disp_nrunnable ought to be 538 * in our cache still, so it's inexpensive to check, and if there 539 * is anything runnable we won't have to wait for the poke. 540 */ 541 if (cpup->cpu_disp->disp_nrunnable != 0) { 542 if (hset_update) { 543 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 544 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 545 } 546 sti(); 547 return; 548 } 549 550 if (cpu_idle_enter(IDLE_STATE_C1, 0, 551 cpu_idle_check_wakeup, NULL) == 0) { 552 mach_cpu_idle(); 553 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 554 } 555 556 /* 557 * We're no longer halted 558 */ 559 if (hset_update) { 560 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 561 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 562 } 563 } 564 565 566 /* 567 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 568 * Otherwise, see if other CPUs in the cpu partition are halted and need to 569 * be woken up so that they can steal the thread we placed on this CPU. 570 * This function is only used on MP systems. 571 */ 572 static void 573 cpu_wakeup(cpu_t *cpu, int bound) 574 { 575 uint_t cpu_found; 576 processorid_t cpu_sid; 577 cpupart_t *cp; 578 579 cp = cpu->cpu_part; 580 cpu_sid = cpu->cpu_seqid; 581 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { 582 /* 583 * Clear the halted bit for that CPU since it will be 584 * poked in a moment. 585 */ 586 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 587 /* 588 * We may find the current CPU present in the halted cpuset 589 * if we're in the context of an interrupt that occurred 590 * before we had a chance to clear our bit in cpu_idle(). 591 * Poking ourself is obviously unnecessary, since if 592 * we're here, we're not halted. 593 */ 594 if (cpu != CPU) 595 poke_cpu(cpu->cpu_id); 596 return; 597 } else { 598 /* 599 * This cpu isn't halted, but it's idle or undergoing a 600 * context switch. No need to awaken anyone else. 601 */ 602 if (cpu->cpu_thread == cpu->cpu_idle_thread || 603 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 604 return; 605 } 606 607 /* 608 * No need to wake up other CPUs if this is for a bound thread. 609 */ 610 if (bound) 611 return; 612 613 /* 614 * The CPU specified for wakeup isn't currently halted, so check 615 * to see if there are any other halted CPUs in the partition, 616 * and if there are then awaken one. 617 */ 618 do { 619 cpu_found = bitset_find(&cp->cp_haltset); 620 if (cpu_found == (uint_t)-1) 621 return; 622 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); 623 624 if (cpu_found != CPU->cpu_seqid) { 625 poke_cpu(cpu_seq[cpu_found]->cpu_id); 626 } 627 } 628 629 #ifndef __xpv 630 /* 631 * Function called by CPU idle notification framework to check whether CPU 632 * has been awakened. It will be called with interrupt disabled. 633 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle 634 * notification framework. 635 */ 636 static void 637 cpu_idle_mwait_check_wakeup(void *arg) 638 { 639 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; 640 641 ASSERT(arg != NULL); 642 if (*mcpu_mwait != MWAIT_HALTED) { 643 /* 644 * CPU has been awakened, notify CPU idle notification system. 645 */ 646 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 647 } else { 648 /* 649 * Toggle interrupt flag to detect pending interrupts. 650 * If interrupt happened, do_interrupt() will notify CPU idle 651 * notification framework so no need to call cpu_idle_exit() 652 * here. 653 */ 654 sti(); 655 SMT_PAUSE(); 656 cli(); 657 } 658 } 659 660 /* 661 * Idle the present CPU until awakened via touching its monitored line 662 */ 663 void 664 cpu_idle_mwait(void) 665 { 666 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 667 cpu_t *cpup = CPU; 668 processorid_t cpu_sid = cpup->cpu_seqid; 669 cpupart_t *cp = cpup->cpu_part; 670 int hset_update = 1; 671 672 /* 673 * Set our mcpu_mwait here, so we can tell if anyone tries to 674 * wake us between now and when we call mwait. No other cpu will 675 * attempt to set our mcpu_mwait until we add ourself to the halted 676 * CPU bitmap. 677 */ 678 *mcpu_mwait = MWAIT_HALTED; 679 680 /* 681 * If this CPU is online, and there's multiple CPUs 682 * in the system, then we should note our halting 683 * by adding ourselves to the partition's halted CPU 684 * bitmap. This allows other CPUs to find/awaken us when 685 * work becomes available. 686 */ 687 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 688 hset_update = 0; 689 690 /* 691 * Add ourselves to the partition's halted CPUs bitmap 692 * and set our HALTED flag, if necessary. 693 * 694 * When a thread becomes runnable, it is placed on the queue 695 * and then the halted CPU bitmap is checked to determine who 696 * (if anyone) should be awakened. We therefore need to first 697 * add ourselves to the bitmap, and and then check if there 698 * is any work available. 699 * 700 * Note that memory barriers after updating the HALTED flag 701 * are not necessary since an atomic operation (updating the bitmap) 702 * immediately follows. On x86 the atomic operation acts as a 703 * memory barrier for the update of cpu_disp_flags. 704 */ 705 if (hset_update) { 706 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 707 bitset_atomic_add(&cp->cp_haltset, cpu_sid); 708 } 709 710 /* 711 * Check to make sure there's really nothing to do. 712 * Work destined for this CPU may become available after 713 * this check. We'll be notified through the clearing of our 714 * bit in the halted CPU bitmap, and a write to our mcpu_mwait. 715 * 716 * disp_anywork() checks disp_nrunnable, so we do not have to later. 717 */ 718 if (disp_anywork()) { 719 if (hset_update) { 720 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 721 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 722 } 723 return; 724 } 725 726 /* 727 * We're on our way to being halted. 728 * To avoid a lost wakeup, arm the monitor before checking if another 729 * cpu wrote to mcpu_mwait to wake us up. 730 */ 731 i86_monitor(mcpu_mwait, 0, 0); 732 if (*mcpu_mwait == MWAIT_HALTED) { 733 if (cpu_idle_enter(IDLE_STATE_C1, 0, 734 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { 735 if (*mcpu_mwait == MWAIT_HALTED) { 736 i86_mwait(0, 0); 737 } 738 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); 739 } 740 } 741 742 /* 743 * We're no longer halted 744 */ 745 if (hset_update) { 746 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 747 bitset_atomic_del(&cp->cp_haltset, cpu_sid); 748 } 749 } 750 751 /* 752 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 753 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 754 * need to be woken up so that they can steal the thread we placed on this CPU. 755 * This function is only used on MP systems. 756 */ 757 static void 758 cpu_wakeup_mwait(cpu_t *cp, int bound) 759 { 760 cpupart_t *cpu_part; 761 uint_t cpu_found; 762 processorid_t cpu_sid; 763 764 cpu_part = cp->cpu_part; 765 cpu_sid = cp->cpu_seqid; 766 767 /* 768 * Clear the halted bit for that CPU since it will be woken up 769 * in a moment. 770 */ 771 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { 772 /* 773 * Clear the halted bit for that CPU since it will be 774 * poked in a moment. 775 */ 776 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); 777 /* 778 * We may find the current CPU present in the halted cpuset 779 * if we're in the context of an interrupt that occurred 780 * before we had a chance to clear our bit in cpu_idle(). 781 * Waking ourself is obviously unnecessary, since if 782 * we're here, we're not halted. 783 * 784 * monitor/mwait wakeup via writing to our cache line is 785 * harmless and less expensive than always checking if we 786 * are waking ourself which is an uncommon case. 787 */ 788 MWAIT_WAKEUP(cp); /* write to monitored line */ 789 return; 790 } else { 791 /* 792 * This cpu isn't halted, but it's idle or undergoing a 793 * context switch. No need to awaken anyone else. 794 */ 795 if (cp->cpu_thread == cp->cpu_idle_thread || 796 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 797 return; 798 } 799 800 /* 801 * No need to wake up other CPUs if the thread we just enqueued 802 * is bound. 803 */ 804 if (bound || ncpus == 1) 805 return; 806 807 /* 808 * See if there's any other halted CPUs. If there are, then 809 * select one, and awaken it. 810 * It's possible that after we find a CPU, somebody else 811 * will awaken it before we get the chance. 812 * In that case, look again. 813 */ 814 do { 815 cpu_found = bitset_find(&cpu_part->cp_haltset); 816 if (cpu_found == (uint_t)-1) 817 return; 818 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, 819 cpu_found) < 0); 820 821 /* 822 * Do not check if cpu_found is ourself as monitor/mwait 823 * wakeup is cheap. 824 */ 825 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ 826 } 827 828 #endif 829 830 void (*cpu_pause_handler)(volatile char *) = NULL; 831 832 static int 833 mp_disable_intr(int cpun) 834 { 835 /* 836 * switch to the offline cpu 837 */ 838 affinity_set(cpun); 839 /* 840 * raise ipl to just below cross call 841 */ 842 splx(XC_SYS_PIL - 1); 843 /* 844 * set base spl to prevent the next swtch to idle from 845 * lowering back to ipl 0 846 */ 847 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); 848 set_base_spl(); 849 affinity_clear(); 850 return (DDI_SUCCESS); 851 } 852 853 static void 854 mp_enable_intr(int cpun) 855 { 856 /* 857 * switch to the online cpu 858 */ 859 affinity_set(cpun); 860 /* 861 * clear the interrupt active mask 862 */ 863 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); 864 set_base_spl(); 865 (void) spl0(); 866 affinity_clear(); 867 } 868 869 static void 870 mach_get_platform(int owner) 871 { 872 void **srv_opsp; 873 void **clt_opsp; 874 int i; 875 int total_ops; 876 877 /* fix up psm ops */ 878 srv_opsp = (void **)mach_set[0]; 879 clt_opsp = (void **)mach_set[owner]; 880 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 881 total_ops = sizeof (struct psm_ops_ver01) / 882 sizeof (void (*)(void)); 883 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 884 /* no psm_notify_func */ 885 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 886 sizeof (void (*)(void)); 887 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 888 /* no psm_timer funcs */ 889 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 890 sizeof (void (*)(void)); 891 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 892 /* no psm_preshutdown function */ 893 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 894 sizeof (void (*)(void)); 895 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 896 /* no psm_intr_ops function */ 897 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 898 sizeof (void (*)(void)); 899 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_5) 900 /* no psm_state function */ 901 total_ops = OFFSETOF(struct psm_ops, psm_state) / 902 sizeof (void (*)(void)); 903 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_6) 904 /* no psm_cpu_ops function */ 905 total_ops = OFFSETOF(struct psm_ops, psm_cpu_ops) / 906 sizeof (void (*)(void)); 907 else 908 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 909 910 /* 911 * Save the version of the PSM module, in case we need to 912 * behave differently based on version. 913 */ 914 mach_ver[0] = mach_ver[owner]; 915 916 for (i = 0; i < total_ops; i++) 917 if (clt_opsp[i] != NULL) 918 srv_opsp[i] = clt_opsp[i]; 919 } 920 921 static void 922 mach_construct_info() 923 { 924 struct psm_sw *swp; 925 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 926 int conflict_owner = 0; 927 928 if (psmsw->psw_forw == psmsw) 929 panic("No valid PSM modules found"); 930 mutex_enter(&psmsw_lock); 931 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 932 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 933 continue; 934 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 935 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 936 mach_cnt[swp->psw_infop->p_owner]++; 937 } 938 mutex_exit(&psmsw_lock); 939 940 mach_get_platform(PSM_OWN_SYS_DEFAULT); 941 942 /* check to see are there any conflicts */ 943 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 944 conflict_owner = PSM_OWN_EXCLUSIVE; 945 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 946 conflict_owner = PSM_OWN_OVERRIDE; 947 if (conflict_owner) { 948 /* remove all psm modules except uppc */ 949 cmn_err(CE_WARN, 950 "Conflicts detected on the following PSM modules:"); 951 mutex_enter(&psmsw_lock); 952 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 953 if (swp->psw_infop->p_owner == conflict_owner) 954 cmn_err(CE_WARN, "%s ", 955 swp->psw_infop->p_mach_idstring); 956 } 957 mutex_exit(&psmsw_lock); 958 cmn_err(CE_WARN, 959 "Setting the system back to SINGLE processor mode!"); 960 cmn_err(CE_WARN, 961 "Please edit /etc/mach to remove the invalid PSM module."); 962 return; 963 } 964 965 if (mach_set[PSM_OWN_EXCLUSIVE]) 966 mach_get_platform(PSM_OWN_EXCLUSIVE); 967 968 if (mach_set[PSM_OWN_OVERRIDE]) 969 mach_get_platform(PSM_OWN_OVERRIDE); 970 } 971 972 static void 973 mach_init() 974 { 975 struct psm_ops *pops; 976 977 mach_construct_info(); 978 979 pops = mach_set[0]; 980 981 /* register the interrupt and clock initialization rotuines */ 982 picinitf = mach_picinit; 983 clkinitf = mach_clkinit; 984 psm_get_clockirq = pops->psm_get_clockirq; 985 986 /* register the interrupt setup code */ 987 slvltovect = mach_softlvl_to_vect; 988 addspl = pops->psm_addspl; 989 delspl = pops->psm_delspl; 990 991 if (pops->psm_translate_irq) 992 psm_translate_irq = pops->psm_translate_irq; 993 if (pops->psm_intr_ops) 994 psm_intr_ops = pops->psm_intr_ops; 995 996 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 997 /* 998 * Time-of-day functionality now handled in TOD modules. 999 * (Warn about PSM modules that think that we're going to use 1000 * their ops vectors.) 1001 */ 1002 if (pops->psm_tod_get) 1003 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 1004 (void *)pops->psm_tod_get); 1005 1006 if (pops->psm_tod_set) 1007 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 1008 (void *)pops->psm_tod_set); 1009 #endif 1010 1011 if (pops->psm_notify_error) { 1012 psm_notify_error = mach_notify_error; 1013 notify_error = pops->psm_notify_error; 1014 } 1015 1016 (*pops->psm_softinit)(); 1017 1018 /* 1019 * Initialize the dispatcher's function hooks to enable CPU halting 1020 * when idle. Set both the deep-idle and non-deep-idle hooks. 1021 * 1022 * Assume we can use power saving deep-idle loop cpu_idle_adaptive. 1023 * Platform deep-idle driver will reset our idle loop to 1024 * non_deep_idle_cpu if power saving deep-idle feature is not available. 1025 * 1026 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) 1027 * or idle_cpu_prefer_mwait is not set. 1028 * Allocate monitor/mwait buffer for cpu0. 1029 */ 1030 #ifndef __xpv 1031 non_deep_idle_disp_enq_thread = disp_enq_thread; 1032 #endif 1033 if (idle_cpu_use_hlt) { 1034 idle_cpu = cpu_idle_adaptive; 1035 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1036 #ifndef __xpv 1037 if (is_x86_feature(x86_featureset, X86FSET_MWAIT) && 1038 idle_cpu_prefer_mwait) { 1039 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); 1040 /* 1041 * Protect ourself from insane mwait size. 1042 */ 1043 if (CPU->cpu_m.mcpu_mwait == NULL) { 1044 #ifdef DEBUG 1045 cmn_err(CE_NOTE, "Using hlt idle. Cannot " 1046 "handle cpu 0 mwait size."); 1047 #endif 1048 idle_cpu_prefer_mwait = 0; 1049 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1050 } else { 1051 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; 1052 } 1053 } else { 1054 CPU->cpu_m.mcpu_idle_cpu = cpu_idle; 1055 } 1056 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; 1057 1058 /* 1059 * Disable power saving deep idle loop? 1060 */ 1061 if (idle_cpu_no_deep_c) { 1062 idle_cpu = non_deep_idle_cpu; 1063 } 1064 #endif 1065 } 1066 1067 mach_smpinit(); 1068 } 1069 1070 static void 1071 mach_smpinit(void) 1072 { 1073 struct psm_ops *pops; 1074 processorid_t cpu_id; 1075 int cnt; 1076 cpuset_t cpumask; 1077 1078 pops = mach_set[0]; 1079 CPUSET_ZERO(cpumask); 1080 1081 cpu_id = -1; 1082 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1083 /* 1084 * Only add boot_ncpus CPUs to mp_cpus. Other CPUs will be handled 1085 * by CPU DR driver at runtime. 1086 */ 1087 for (cnt = 0; cpu_id != -1 && cnt < boot_ncpus; cnt++) { 1088 CPUSET_ADD(cpumask, cpu_id); 1089 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 1090 } 1091 1092 mp_cpus = cpumask; 1093 1094 /* MP related routines */ 1095 ap_mlsetup = pops->psm_post_cpu_start; 1096 send_dirintf = pops->psm_send_ipi; 1097 1098 /* optional MP related routines */ 1099 if (pops->psm_shutdown) 1100 psm_shutdownf = pops->psm_shutdown; 1101 if (pops->psm_preshutdown) 1102 psm_preshutdownf = pops->psm_preshutdown; 1103 if (pops->psm_notify_func) 1104 psm_notifyf = pops->psm_notify_func; 1105 if (pops->psm_set_idlecpu) 1106 psm_set_idle_cpuf = pops->psm_set_idlecpu; 1107 if (pops->psm_unset_idlecpu) 1108 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 1109 1110 psm_clkinit = pops->psm_clkinit; 1111 1112 if (pops->psm_timer_reprogram) 1113 psm_timer_reprogram = pops->psm_timer_reprogram; 1114 1115 if (pops->psm_timer_enable) 1116 psm_timer_enable = pops->psm_timer_enable; 1117 1118 if (pops->psm_timer_disable) 1119 psm_timer_disable = pops->psm_timer_disable; 1120 1121 if (pops->psm_post_cyclic_setup) 1122 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 1123 1124 if (pops->psm_state) 1125 psm_state = pops->psm_state; 1126 1127 /* 1128 * Set these vectors here so they can be used by Suspend/Resume 1129 * on UP machines. 1130 */ 1131 if (pops->psm_disable_intr) 1132 psm_disable_intr = pops->psm_disable_intr; 1133 if (pops->psm_enable_intr) 1134 psm_enable_intr = pops->psm_enable_intr; 1135 1136 /* check for multiple CPUs */ 1137 if (cnt < 2 && plat_dr_support_cpu() == B_FALSE) 1138 return; 1139 1140 /* check for MP platforms */ 1141 if (pops->psm_cpu_start == NULL) 1142 return; 1143 1144 /* 1145 * Set the dispatcher hook to enable cpu "wake up" 1146 * when a thread becomes runnable. 1147 */ 1148 if (idle_cpu_use_hlt) { 1149 disp_enq_thread = cpu_wakeup; 1150 #ifndef __xpv 1151 if (is_x86_feature(x86_featureset, X86FSET_MWAIT) && 1152 idle_cpu_prefer_mwait) 1153 disp_enq_thread = cpu_wakeup_mwait; 1154 non_deep_idle_disp_enq_thread = disp_enq_thread; 1155 #endif 1156 } 1157 1158 psm_get_ipivect = pops->psm_get_ipivect; 1159 psm_get_pir_ipivect = pops->psm_get_pir_ipivect; 1160 psm_send_pir_ipi = pops->psm_send_pir_ipi; 1161 1162 1163 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", 1164 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 1165 NULL, NULL, NULL, NULL); 1166 1167 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 1168 } 1169 1170 static void 1171 mach_picinit() 1172 { 1173 struct psm_ops *pops; 1174 1175 pops = mach_set[0]; 1176 1177 /* register the interrupt handlers */ 1178 setlvl = pops->psm_intr_enter; 1179 setlvlx = pops->psm_intr_exit; 1180 1181 /* initialize the interrupt hardware */ 1182 (*pops->psm_picinit)(); 1183 1184 /* set interrupt mask for current ipl */ 1185 setspl = pops->psm_setspl; 1186 cli(); 1187 setspl(CPU->cpu_pri); 1188 } 1189 1190 uint_t cpu_freq; /* MHz */ 1191 uint64_t cpu_freq_hz; /* measured (in hertz) */ 1192 1193 #define MEGA_HZ 1000000 1194 1195 #ifdef __xpv 1196 1197 int xpv_cpufreq_workaround = 1; 1198 int xpv_cpufreq_verbose = 0; 1199 1200 #else /* __xpv */ 1201 1202 static uint64_t 1203 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 1204 { 1205 uint64_t cpu_hz; 1206 1207 if ((pit_counter == 0) || (*processor_clks == 0) || 1208 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 1209 return (0); 1210 1211 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 1212 1213 return (cpu_hz); 1214 } 1215 1216 #endif /* __xpv */ 1217 1218 static uint64_t 1219 mach_getcpufreq(void) 1220 { 1221 #if defined(__xpv) 1222 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; 1223 uint64_t cpu_hz; 1224 1225 /* 1226 * During dom0 bringup, it was noted that on at least one older 1227 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul 1228 * value that is quite wrong (the 3.06GHz clock was reported 1229 * as 4.77GHz) 1230 * 1231 * The curious thing is, that if you stop the kernel at entry, 1232 * breakpoint here and inspect the value with kmdb, the value 1233 * is correct - but if you don't stop and simply enable the 1234 * printf statement (below), you can see the bad value printed 1235 * here. Almost as if something kmdb did caused the hypervisor to 1236 * figure it out correctly. And, note that the hypervisor 1237 * eventually -does- figure it out correctly ... if you look at 1238 * the field later in the life of dom0, it is correct. 1239 * 1240 * For now, on dom0, we employ a slightly cheesy workaround of 1241 * using the DOM0_PHYSINFO hypercall. 1242 */ 1243 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { 1244 cpu_hz = 1000 * xpv_cpu_khz(); 1245 } else { 1246 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; 1247 1248 if (vti->tsc_shift < 0) 1249 cpu_hz <<= -vti->tsc_shift; 1250 else 1251 cpu_hz >>= vti->tsc_shift; 1252 } 1253 1254 if (xpv_cpufreq_verbose) 1255 printf("mach_getcpufreq: system_mul 0x%x, shift %d, " 1256 "cpu_hz %" PRId64 "Hz\n", 1257 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); 1258 1259 return (cpu_hz); 1260 #else /* __xpv */ 1261 uint32_t pit_counter; 1262 uint64_t processor_clks; 1263 1264 if (is_x86_feature(x86_featureset, X86FSET_TSC)) { 1265 /* 1266 * We have a TSC. freq_tsc() knows how to measure the number 1267 * of clock cycles sampled against the PIT. 1268 */ 1269 ulong_t flags = clear_int_flag(); 1270 processor_clks = freq_tsc(&pit_counter); 1271 restore_int_flag(flags); 1272 return (mach_calchz(pit_counter, &processor_clks)); 1273 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1274 #if defined(__amd64) 1275 panic("mach_getcpufreq: no TSC!"); 1276 #elif defined(__i386) 1277 /* 1278 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1279 * for which freq_notsc() knows how to measure the number of 1280 * elapsed clock cycles sampled against the PIT 1281 */ 1282 ulong_t flags = clear_int_flag(); 1283 processor_clks = freq_notsc(&pit_counter); 1284 restore_int_flag(flags); 1285 return (mach_calchz(pit_counter, &processor_clks)); 1286 #endif /* __i386 */ 1287 } 1288 1289 /* We do not know how to calculate cpu frequency for this cpu. */ 1290 return (0); 1291 #endif /* __xpv */ 1292 } 1293 1294 /* 1295 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1296 * to this array, instead improve the accuracy of the algorithm that determines 1297 * the clock speed of the processor or extend the implementation to support the 1298 * vendor as appropriate. This is here only to support adjusting the speed on 1299 * older slower processors that mach_fixcpufreq() would not be able to account 1300 * for otherwise. 1301 */ 1302 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1303 1304 /* 1305 * On fast processors the clock frequency that is measured may be off by 1306 * a few MHz from the value printed on the part. This is a combination of 1307 * the factors that for such fast parts being off by this much is within 1308 * the tolerances for manufacture and because of the difficulties in the 1309 * measurement that can lead to small error. This function uses some 1310 * heuristics in order to tweak the value that was measured to match what 1311 * is most likely printed on the part. 1312 * 1313 * Some examples: 1314 * AMD Athlon 1000 mhz measured as 998 mhz 1315 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1316 * Intel Pentium IV 1500 mhz measured as 1495mhz 1317 * 1318 * If in the future this function is no longer sufficient to correct 1319 * for the error in the measurement, then the algorithm used to perform 1320 * the measurement will have to be improved in order to increase accuracy 1321 * rather than adding horrible and questionable kludges here. 1322 * 1323 * This is called after the cyclics subsystem because of the potential 1324 * that the heuristics within may give a worse estimate of the clock 1325 * frequency than the value that was measured. 1326 */ 1327 static void 1328 mach_fixcpufreq(void) 1329 { 1330 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1331 1332 freq = (uint32_t)cpu_freq; 1333 1334 /* 1335 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1336 * measured speed taking into account that the 667 MHz parts were 1337 * the first to round-up. 1338 */ 1339 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1340 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1341 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1342 1343 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1344 mul = (freq + 25) / 50; 1345 near50 = mul * 50; 1346 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1347 1348 /* Find the closer of the two */ 1349 if (delta66 < delta50) { 1350 fixed = near66; 1351 delta = delta66; 1352 } else { 1353 fixed = near50; 1354 delta = delta50; 1355 } 1356 1357 if (fixed > INT_MAX) 1358 return; 1359 1360 /* 1361 * Some older parts have a core clock frequency that is not an 1362 * integral multiple of 50 or 66 MHz. Check if one of the old 1363 * clock frequencies is closer to the measured value than any 1364 * of the integral multiples of 50 an 66, and if so set fixed 1365 * and delta appropriately to represent the closest value. 1366 */ 1367 i = sizeof (x86_cpu_freq) / sizeof (int); 1368 while (i > 0) { 1369 i--; 1370 1371 if (x86_cpu_freq[i] <= freq) { 1372 mul = freq - x86_cpu_freq[i]; 1373 1374 if (mul < delta) { 1375 fixed = x86_cpu_freq[i]; 1376 delta = mul; 1377 } 1378 1379 break; 1380 } 1381 1382 mul = x86_cpu_freq[i] - freq; 1383 1384 if (mul < delta) { 1385 fixed = x86_cpu_freq[i]; 1386 delta = mul; 1387 } 1388 } 1389 1390 /* 1391 * Set a reasonable maximum for how much to correct the measured 1392 * result by. This check is here to prevent the adjustment made 1393 * by this function from being more harm than good. It is entirely 1394 * possible that in the future parts will be made that are not 1395 * integral multiples of 66 or 50 in clock frequency or that 1396 * someone may overclock a part to some odd frequency. If the 1397 * measured value is farther from the corrected value than 1398 * allowed, then assume the corrected value is in error and use 1399 * the measured value. 1400 */ 1401 if (6 < delta) 1402 return; 1403 1404 cpu_freq = (int)fixed; 1405 } 1406 1407 1408 static int 1409 machhztomhz(uint64_t cpu_freq_hz) 1410 { 1411 uint64_t cpu_mhz; 1412 1413 /* Round to nearest MHZ */ 1414 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1415 1416 if (cpu_mhz > INT_MAX) 1417 return (0); 1418 1419 return ((int)cpu_mhz); 1420 1421 } 1422 1423 1424 static int 1425 mach_clkinit(int preferred_mode, int *set_mode) 1426 { 1427 struct psm_ops *pops; 1428 int resolution; 1429 1430 pops = mach_set[0]; 1431 1432 cpu_freq_hz = mach_getcpufreq(); 1433 1434 cpu_freq = machhztomhz(cpu_freq_hz); 1435 1436 if (!is_x86_feature(x86_featureset, X86FSET_TSC) || (cpu_freq == 0)) 1437 tsc_gethrtime_enable = 0; 1438 1439 #ifndef __xpv 1440 if (tsc_gethrtime_enable) { 1441 tsc_hrtimeinit(cpu_freq_hz); 1442 } else 1443 #endif 1444 { 1445 if (pops->psm_hrtimeinit) 1446 (*pops->psm_hrtimeinit)(); 1447 gethrtimef = pops->psm_gethrtime; 1448 gethrtimeunscaledf = gethrtimef; 1449 /* scalehrtimef will remain dummy */ 1450 } 1451 1452 mach_fixcpufreq(); 1453 1454 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1455 if (preferred_mode == TIMER_ONESHOT) { 1456 1457 resolution = (*pops->psm_clkinit)(0); 1458 if (resolution != 0) { 1459 *set_mode = TIMER_ONESHOT; 1460 return (resolution); 1461 } 1462 } 1463 1464 /* 1465 * either periodic mode was requested or could not set to 1466 * one-shot mode 1467 */ 1468 resolution = (*pops->psm_clkinit)(hz); 1469 /* 1470 * psm should be able to do periodic, so we do not check 1471 * for return value of psm_clkinit here. 1472 */ 1473 *set_mode = TIMER_PERIODIC; 1474 return (resolution); 1475 } else { 1476 /* 1477 * PSMI interface prior to PSMI_3 does not define a return 1478 * value for psm_clkinit, so the return value is ignored. 1479 */ 1480 (void) (*pops->psm_clkinit)(hz); 1481 *set_mode = TIMER_PERIODIC; 1482 return (nsec_per_tick); 1483 } 1484 } 1485 1486 1487 /*ARGSUSED*/ 1488 static int 1489 mach_softlvl_to_vect(int ipl) 1490 { 1491 setsoftint = av_set_softint_pending; 1492 kdisetsoftint = kdi_av_set_softint_pending; 1493 1494 return (PSM_SV_SOFTWARE); 1495 } 1496 1497 #ifdef DEBUG 1498 /* 1499 * This is here to allow us to simulate cpus that refuse to start. 1500 */ 1501 cpuset_t cpufailset; 1502 #endif 1503 1504 int 1505 mach_cpu_start(struct cpu *cp, void *ctx) 1506 { 1507 struct psm_ops *pops = mach_set[0]; 1508 processorid_t id = cp->cpu_id; 1509 1510 #ifdef DEBUG 1511 if (CPU_IN_SET(cpufailset, id)) 1512 return (0); 1513 #endif 1514 return ((*pops->psm_cpu_start)(id, ctx)); 1515 } 1516 1517 int 1518 mach_cpuid_start(processorid_t id, void *ctx) 1519 { 1520 struct psm_ops *pops = mach_set[0]; 1521 1522 #ifdef DEBUG 1523 if (CPU_IN_SET(cpufailset, id)) 1524 return (0); 1525 #endif 1526 return ((*pops->psm_cpu_start)(id, ctx)); 1527 } 1528 1529 int 1530 mach_cpu_stop(cpu_t *cp, void *ctx) 1531 { 1532 struct psm_ops *pops = mach_set[0]; 1533 psm_cpu_request_t request; 1534 1535 if (pops->psm_cpu_ops == NULL) { 1536 return (ENOTSUP); 1537 } 1538 1539 ASSERT(cp->cpu_id != -1); 1540 request.pcr_cmd = PSM_CPU_STOP; 1541 request.req.cpu_stop.cpuid = cp->cpu_id; 1542 request.req.cpu_stop.ctx = ctx; 1543 1544 return ((*pops->psm_cpu_ops)(&request)); 1545 } 1546 1547 int 1548 mach_cpu_add(mach_cpu_add_arg_t *argp, processorid_t *cpuidp) 1549 { 1550 int rc; 1551 struct psm_ops *pops = mach_set[0]; 1552 psm_cpu_request_t request; 1553 1554 if (pops->psm_cpu_ops == NULL) { 1555 return (ENOTSUP); 1556 } 1557 1558 request.pcr_cmd = PSM_CPU_ADD; 1559 request.req.cpu_add.argp = argp; 1560 request.req.cpu_add.cpuid = -1; 1561 rc = (*pops->psm_cpu_ops)(&request); 1562 if (rc == 0) { 1563 ASSERT(request.req.cpu_add.cpuid != -1); 1564 *cpuidp = request.req.cpu_add.cpuid; 1565 } 1566 1567 return (rc); 1568 } 1569 1570 int 1571 mach_cpu_remove(processorid_t cpuid) 1572 { 1573 struct psm_ops *pops = mach_set[0]; 1574 psm_cpu_request_t request; 1575 1576 if (pops->psm_cpu_ops == NULL) { 1577 return (ENOTSUP); 1578 } 1579 1580 request.pcr_cmd = PSM_CPU_REMOVE; 1581 request.req.cpu_remove.cpuid = cpuid; 1582 1583 return ((*pops->psm_cpu_ops)(&request)); 1584 } 1585 1586 /* 1587 * Default handler to create device node for CPU. 1588 * One reference count will be held on created device node. 1589 */ 1590 static int 1591 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp) 1592 { 1593 int rv, circ; 1594 dev_info_t *dip; 1595 static kmutex_t cpu_node_lock; 1596 static dev_info_t *cpu_nex_devi = NULL; 1597 1598 ASSERT(cp != NULL); 1599 ASSERT(dipp != NULL); 1600 *dipp = NULL; 1601 1602 if (cpu_nex_devi == NULL) { 1603 mutex_enter(&cpu_node_lock); 1604 /* First check whether cpus exists. */ 1605 cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0); 1606 /* Create cpus if it doesn't exist. */ 1607 if (cpu_nex_devi == NULL) { 1608 ndi_devi_enter(ddi_root_node(), &circ); 1609 rv = ndi_devi_alloc(ddi_root_node(), "cpus", 1610 (pnode_t)DEVI_SID_NODEID, &dip); 1611 if (rv != NDI_SUCCESS) { 1612 mutex_exit(&cpu_node_lock); 1613 cmn_err(CE_CONT, 1614 "?failed to create cpu nexus device.\n"); 1615 return (PSM_FAILURE); 1616 } 1617 ASSERT(dip != NULL); 1618 (void) ndi_devi_online(dip, 0); 1619 ndi_devi_exit(ddi_root_node(), circ); 1620 cpu_nex_devi = dip; 1621 } 1622 mutex_exit(&cpu_node_lock); 1623 } 1624 1625 /* 1626 * create a child node for cpu identified as 'cpu_id' 1627 */ 1628 ndi_devi_enter(cpu_nex_devi, &circ); 1629 dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1); 1630 if (dip == NULL) { 1631 cmn_err(CE_CONT, 1632 "?failed to create device node for cpu%d.\n", cp->cpu_id); 1633 rv = PSM_FAILURE; 1634 } else { 1635 *dipp = dip; 1636 (void) ndi_hold_devi(dip); 1637 rv = PSM_SUCCESS; 1638 } 1639 ndi_devi_exit(cpu_nex_devi, circ); 1640 1641 return (rv); 1642 } 1643 1644 /* 1645 * Create cpu device node in device tree and online it. 1646 * Return created dip with reference count held if requested. 1647 */ 1648 int 1649 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp) 1650 { 1651 int rv; 1652 dev_info_t *dip = NULL; 1653 1654 ASSERT(psm_cpu_create_devinfo != NULL); 1655 rv = psm_cpu_create_devinfo(cp, &dip); 1656 if (rv == PSM_SUCCESS) { 1657 cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi); 1658 /* Recursively attach driver for parent nexus device. */ 1659 if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) == 1660 DDI_SUCCESS) { 1661 /* Configure cpu itself and descendants. */ 1662 (void) ndi_devi_online(dip, 1663 NDI_ONLINE_ATTACH | NDI_CONFIG); 1664 } 1665 if (dipp != NULL) { 1666 *dipp = dip; 1667 } else { 1668 (void) ndi_rele_devi(dip); 1669 } 1670 } 1671 1672 return (rv); 1673 } 1674 1675 /* 1676 * The dipp contains one of following values on return: 1677 * - NULL if no device node found 1678 * - pointer to device node if found 1679 */ 1680 int 1681 mach_cpu_get_device_node(struct cpu *cp, dev_info_t **dipp) 1682 { 1683 *dipp = NULL; 1684 if (psm_cpu_get_devinfo != NULL) { 1685 if (psm_cpu_get_devinfo(cp, dipp) == PSM_SUCCESS) { 1686 return (PSM_SUCCESS); 1687 } 1688 } 1689 1690 return (PSM_FAILURE); 1691 } 1692 1693 /*ARGSUSED*/ 1694 static int 1695 mach_translate_irq(dev_info_t *dip, int irqno) 1696 { 1697 return (irqno); /* default to NO translation */ 1698 } 1699 1700 static void 1701 mach_notify_error(int level, char *errmsg) 1702 { 1703 /* 1704 * SL_FATAL is pass in once panicstr is set, deliver it 1705 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1706 * codes for the psmi handler 1707 */ 1708 if (level & SL_FATAL) 1709 (*notify_error)(CE_PANIC, errmsg); 1710 else if (level & SL_WARN) 1711 (*notify_error)(CE_WARN, errmsg); 1712 else if (level & SL_NOTE) 1713 (*notify_error)(CE_NOTE, errmsg); 1714 else if (level & SL_CONSOLE) 1715 (*notify_error)(CE_CONT, errmsg); 1716 } 1717 1718 /* 1719 * It provides the default basic intr_ops interface for the new DDI 1720 * interrupt framework if the PSM doesn't have one. 1721 * 1722 * Input: 1723 * dip - pointer to the dev_info structure of the requested device 1724 * hdlp - pointer to the internal interrupt handle structure for the 1725 * requested interrupt 1726 * intr_op - opcode for this call 1727 * result - pointer to the integer that will hold the result to be 1728 * passed back if return value is PSM_SUCCESS 1729 * 1730 * Output: 1731 * return value is either PSM_SUCCESS or PSM_FAILURE 1732 */ 1733 static int 1734 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1735 psm_intr_op_t intr_op, int *result) 1736 { 1737 struct intrspec *ispec; 1738 1739 switch (intr_op) { 1740 case PSM_INTR_OP_CHECK_MSI: 1741 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1742 DDI_INTR_TYPE_MSIX); 1743 break; 1744 case PSM_INTR_OP_ALLOC_VECTORS: 1745 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1746 *result = 1; 1747 else 1748 *result = 0; 1749 break; 1750 case PSM_INTR_OP_FREE_VECTORS: 1751 break; 1752 case PSM_INTR_OP_NAVAIL_VECTORS: 1753 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1754 *result = 1; 1755 else 1756 *result = 0; 1757 break; 1758 case PSM_INTR_OP_XLATE_VECTOR: 1759 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1760 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1761 break; 1762 case PSM_INTR_OP_GET_CAP: 1763 *result = 0; 1764 break; 1765 case PSM_INTR_OP_GET_PENDING: 1766 case PSM_INTR_OP_CLEAR_MASK: 1767 case PSM_INTR_OP_SET_MASK: 1768 case PSM_INTR_OP_GET_SHARED: 1769 case PSM_INTR_OP_SET_PRI: 1770 case PSM_INTR_OP_SET_CAP: 1771 case PSM_INTR_OP_SET_CPU: 1772 case PSM_INTR_OP_GET_INTR: 1773 default: 1774 return (PSM_FAILURE); 1775 } 1776 return (PSM_SUCCESS); 1777 } 1778 /* 1779 * Return 1 if CMT load balancing policies should be 1780 * implemented across instances of the specified hardware 1781 * sharing relationship. 1782 */ 1783 int 1784 pg_cmt_load_bal_hw(pghw_type_t hw) 1785 { 1786 if (hw == PGHW_IPIPE || 1787 hw == PGHW_FPU || 1788 hw == PGHW_PROCNODE || 1789 hw == PGHW_CHIP) 1790 return (1); 1791 else 1792 return (0); 1793 } 1794 /* 1795 * Return 1 if thread affinity polices should be implemented 1796 * for instances of the specifed hardware sharing relationship. 1797 */ 1798 int 1799 pg_cmt_affinity_hw(pghw_type_t hw) 1800 { 1801 if (hw == PGHW_CACHE) 1802 return (1); 1803 else 1804 return (0); 1805 } 1806 1807 /* 1808 * Return number of counter events requested to measure hardware capacity and 1809 * utilization and setup CPC requests for specified CPU as needed 1810 * 1811 * May return 0 when platform or processor specific code knows that no CPC 1812 * events should be programmed on this CPU or -1 when platform or processor 1813 * specific code doesn't know which counter events are best to use and common 1814 * code should decide for itself 1815 */ 1816 int 1817 /* LINTED E_FUNC_ARG_UNUSED */ 1818 cu_plat_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs) 1819 { 1820 const char *impl_name; 1821 1822 /* 1823 * Return error if pcbe_ops not set 1824 */ 1825 if (pcbe_ops == NULL) 1826 return (-1); 1827 1828 /* 1829 * Return that no CPC events should be programmed on hyperthreaded 1830 * Pentium 4 and return error for all other x86 processors to tell 1831 * common code to decide what counter events to program on those CPUs 1832 * for measuring hardware capacity and utilization 1833 */ 1834 impl_name = pcbe_ops->pcbe_impl_name(); 1835 if (impl_name != NULL && strcmp(impl_name, PCBE_IMPL_NAME_P4HT) == 0) 1836 return (0); 1837 else 1838 return (-1); 1839 } 1840