1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #define PSMI_1_5 29 #include <sys/smp_impldefs.h> 30 #include <sys/psm.h> 31 #include <sys/psm_modctl.h> 32 #include <sys/pit.h> 33 #include <sys/cmn_err.h> 34 #include <sys/strlog.h> 35 #include <sys/clock.h> 36 #include <sys/debug.h> 37 #include <sys/rtc.h> 38 #include <sys/x86_archext.h> 39 #include <sys/cpupart.h> 40 #include <sys/cpuvar.h> 41 #include <sys/pghw.h> 42 #include <sys/disp.h> 43 #include <sys/archsystm.h> 44 #include <sys/machsystm.h> 45 #include <sys/param.h> 46 #include <sys/promif.h> 47 #include <sys/mach_intr.h> 48 49 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 50 51 /* 52 * Local function prototypes 53 */ 54 static int mp_disable_intr(processorid_t cpun); 55 static void mp_enable_intr(processorid_t cpun); 56 static void mach_init(); 57 static void mach_picinit(); 58 static uint64_t mach_calchz(uint32_t pit_counter, uint64_t *processor_clks); 59 static int machhztomhz(uint64_t cpu_freq_hz); 60 static uint64_t mach_getcpufreq(void); 61 static void mach_fixcpufreq(void); 62 static int mach_clkinit(int, int *); 63 static void mach_smpinit(void); 64 static void mach_set_softintr(int ipl, struct av_softinfo *); 65 static int mach_softlvl_to_vect(int ipl); 66 static void mach_get_platform(int owner); 67 static void mach_construct_info(); 68 static int mach_translate_irq(dev_info_t *dip, int irqno); 69 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 70 psm_intr_op_t, int *); 71 static void mach_notify_error(int level, char *errmsg); 72 static hrtime_t dummy_hrtime(void); 73 static void dummy_scalehrtime(hrtime_t *); 74 static void cpu_idle(void); 75 static void cpu_wakeup(cpu_t *, int); 76 /* 77 * External reference functions 78 */ 79 extern void return_instr(); 80 extern uint64_t freq_tsc(uint32_t *); 81 #if defined(__i386) 82 extern uint64_t freq_notsc(uint32_t *); 83 #endif 84 extern void pc_gethrestime(timestruc_t *); 85 extern int cpuid_get_coreid(cpu_t *); 86 extern int cpuid_get_chipid(cpu_t *); 87 88 /* 89 * PSM functions initialization 90 */ 91 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 92 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 93 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 94 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 95 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 96 void (*psminitf)() = mach_init; 97 void (*picinitf)() = return_instr; 98 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 99 int (*ap_mlsetup)() = (int (*)(void))return_instr; 100 void (*send_dirintf)() = return_instr; 101 void (*setspl)(int) = (void (*)(int))return_instr; 102 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 103 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 104 void (*setsoftint)(int, struct av_softinfo *)= 105 (void (*)(int, struct av_softinfo *))return_instr; 106 int (*slvltovect)(int) = (int (*)(int))return_instr; 107 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 108 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 109 int (*psm_disable_intr)(int) = mp_disable_intr; 110 void (*psm_enable_intr)(int) = mp_enable_intr; 111 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 112 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 113 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 114 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 115 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 116 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 117 int (*psm_get_clockirq)(int) = NULL; 118 int (*psm_get_ipivect)(int, int) = NULL; 119 120 int (*psm_clkinit)(int) = NULL; 121 void (*psm_timer_reprogram)(hrtime_t) = NULL; 122 void (*psm_timer_enable)(void) = NULL; 123 void (*psm_timer_disable)(void) = NULL; 124 void (*psm_post_cyclic_setup)(void *arg) = NULL; 125 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 126 int *) = mach_intr_ops; 127 128 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 129 void (*hrtime_tick)(void) = return_instr; 130 131 int tsc_gethrtime_enable = 1; 132 int tsc_gethrtime_initted = 0; 133 134 /* 135 * Local Static Data 136 */ 137 static struct psm_ops mach_ops; 138 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 139 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 140 141 /* 142 * If non-zero, idle cpus will become "halted" when there's 143 * no work to do. 144 */ 145 int idle_cpu_use_hlt = 1; 146 147 148 /*ARGSUSED*/ 149 int 150 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 151 { 152 switch (hw) { 153 case PGHW_IPIPE: 154 if (x86_feature & (X86_HTT)) { 155 /* 156 * Hyper-threading is SMT 157 */ 158 return (1); 159 } else { 160 return (0); 161 } 162 case PGHW_CHIP: 163 if (x86_feature & (X86_CMP|X86_HTT)) 164 return (1); 165 else 166 return (0); 167 default: 168 return (0); 169 } 170 } 171 172 /* 173 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 174 * If pghw_type_t is an unsupported hardware type, then return -1 175 */ 176 int 177 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 178 { 179 id_t pgp_a, pgp_b; 180 181 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 182 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 183 184 if (pgp_a == -1 || pgp_b == -1) 185 return (-1); 186 187 return (pgp_a == pgp_b); 188 } 189 190 /* 191 * Return a physical instance identifier for known hardware sharing 192 * relationships 193 */ 194 id_t 195 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 196 { 197 switch (hw) { 198 case PGHW_IPIPE: 199 return (cpuid_get_coreid(cpu)); 200 case PGHW_CHIP: 201 return (cpuid_get_chipid(cpu)); 202 default: 203 return (-1); 204 } 205 } 206 207 int 208 pg_plat_hw_level(pghw_type_t hw) 209 { 210 int i; 211 static pghw_type_t hw_hier[] = { 212 PGHW_IPIPE, 213 PGHW_CHIP, 214 PGHW_NUM_COMPONENTS 215 }; 216 217 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 218 if (hw_hier[i] == hw) 219 return (i); 220 } 221 return (-1); 222 } 223 224 id_t 225 pg_plat_get_core_id(cpu_t *cpu) 226 { 227 return ((id_t)cpuid_get_coreid(cpu)); 228 } 229 230 void 231 cmp_set_nosteal_interval(void) 232 { 233 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 234 nosteal_nsec = 100000UL; 235 } 236 237 /* 238 * Routine to ensure initial callers to hrtime gets 0 as return 239 */ 240 static hrtime_t 241 dummy_hrtime(void) 242 { 243 return (0); 244 } 245 246 /* ARGSUSED */ 247 static void 248 dummy_scalehrtime(hrtime_t *ticks) 249 {} 250 251 /* 252 * Idle the present CPU until awoken via an interrupt 253 */ 254 static void 255 cpu_idle(void) 256 { 257 cpu_t *cpup = CPU; 258 processorid_t cpun = cpup->cpu_id; 259 cpupart_t *cp = cpup->cpu_part; 260 int hset_update = 1; 261 262 /* 263 * If this CPU is online, and there's multiple CPUs 264 * in the system, then we should notate our halting 265 * by adding ourselves to the partition's halted CPU 266 * bitmap. This allows other CPUs to find/awaken us when 267 * work becomes available. 268 */ 269 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 270 hset_update = 0; 271 272 /* 273 * Add ourselves to the partition's halted CPUs bitmask 274 * and set our HALTED flag, if necessary. 275 * 276 * When a thread becomes runnable, it is placed on the queue 277 * and then the halted cpuset is checked to determine who 278 * (if anyone) should be awoken. We therefore need to first 279 * add ourselves to the halted cpuset, and and then check if there 280 * is any work available. 281 * 282 * Note that memory barriers after updating the HALTED flag 283 * are not necessary since an atomic operation (updating the bitmap) 284 * immediately follows. On x86 the atomic operation acts as a 285 * memory barrier for the update of cpu_disp_flags. 286 */ 287 if (hset_update) { 288 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 289 CPUSET_ATOMIC_ADD(cp->cp_mach->mc_haltset, cpun); 290 } 291 292 /* 293 * Check to make sure there's really nothing to do. 294 * Work destined for this CPU may become available after 295 * this check. We'll be notified through the clearing of our 296 * bit in the halted CPU bitmask, and a poke. 297 */ 298 if (disp_anywork()) { 299 if (hset_update) { 300 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 301 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 302 } 303 return; 304 } 305 306 /* 307 * We're on our way to being halted. 308 * 309 * Disable interrupts now, so that we'll awaken immediately 310 * after halting if someone tries to poke us between now and 311 * the time we actually halt. 312 * 313 * We check for the presence of our bit after disabling interrupts. 314 * If it's cleared, we'll return. If the bit is cleared after 315 * we check then the poke will pop us out of the halted state. 316 * 317 * This means that the ordering of the poke and the clearing 318 * of the bit by cpu_wakeup is important. 319 * cpu_wakeup() must clear, then poke. 320 * cpu_idle() must disable interrupts, then check for the bit. 321 */ 322 cli(); 323 324 if (hset_update && !CPU_IN_SET(cp->cp_mach->mc_haltset, cpun)) { 325 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 326 sti(); 327 return; 328 } 329 330 /* 331 * The check for anything locally runnable is here for performance 332 * and isn't needed for correctness. disp_nrunnable ought to be 333 * in our cache still, so it's inexpensive to check, and if there 334 * is anything runnable we won't have to wait for the poke. 335 */ 336 if (cpup->cpu_disp->disp_nrunnable != 0) { 337 if (hset_update) { 338 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 339 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 340 } 341 sti(); 342 return; 343 } 344 345 mach_cpu_idle(); 346 347 /* 348 * We're no longer halted 349 */ 350 if (hset_update) { 351 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 352 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 353 } 354 } 355 356 357 /* 358 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 359 * Otherwise, see if other CPUs in the cpu partition are halted and need to 360 * be woken up so that they can steal the thread we placed on this CPU. 361 * This function is only used on MP systems. 362 */ 363 static void 364 cpu_wakeup(cpu_t *cpu, int bound) 365 { 366 uint_t cpu_found; 367 int result; 368 cpupart_t *cp; 369 370 cp = cpu->cpu_part; 371 if (CPU_IN_SET(cp->cp_mach->mc_haltset, cpu->cpu_id)) { 372 /* 373 * Clear the halted bit for that CPU since it will be 374 * poked in a moment. 375 */ 376 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpu->cpu_id); 377 /* 378 * We may find the current CPU present in the halted cpuset 379 * if we're in the context of an interrupt that occurred 380 * before we had a chance to clear our bit in cpu_idle(). 381 * Poking ourself is obviously unnecessary, since if 382 * we're here, we're not halted. 383 */ 384 if (cpu != CPU) 385 poke_cpu(cpu->cpu_id); 386 return; 387 } else { 388 /* 389 * This cpu isn't halted, but it's idle or undergoing a 390 * context switch. No need to awaken anyone else. 391 */ 392 if (cpu->cpu_thread == cpu->cpu_idle_thread || 393 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 394 return; 395 } 396 397 /* 398 * No need to wake up other CPUs if the thread we just enqueued 399 * is bound. 400 */ 401 if (bound) 402 return; 403 404 405 /* 406 * See if there's any other halted CPUs. If there are, then 407 * select one, and awaken it. 408 * It's possible that after we find a CPU, somebody else 409 * will awaken it before we get the chance. 410 * In that case, look again. 411 */ 412 do { 413 CPUSET_FIND(cp->cp_mach->mc_haltset, cpu_found); 414 if (cpu_found == CPUSET_NOTINSET) 415 return; 416 417 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 418 CPUSET_ATOMIC_XDEL(cp->cp_mach->mc_haltset, cpu_found, result); 419 } while (result < 0); 420 421 if (cpu_found != CPU->cpu_id) 422 poke_cpu(cpu_found); 423 } 424 425 void (*cpu_pause_handler)(volatile char *) = NULL; 426 427 static int 428 mp_disable_intr(int cpun) 429 { 430 /* 431 * switch to the offline cpu 432 */ 433 affinity_set(cpun); 434 /* 435 * raise ipl to just below cross call 436 */ 437 splx(XC_MED_PIL-1); 438 /* 439 * set base spl to prevent the next swtch to idle from 440 * lowering back to ipl 0 441 */ 442 CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1)); 443 set_base_spl(); 444 affinity_clear(); 445 return (DDI_SUCCESS); 446 } 447 448 static void 449 mp_enable_intr(int cpun) 450 { 451 /* 452 * switch to the online cpu 453 */ 454 affinity_set(cpun); 455 /* 456 * clear the interrupt active mask 457 */ 458 CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1)); 459 set_base_spl(); 460 (void) spl0(); 461 affinity_clear(); 462 } 463 464 static void 465 mach_get_platform(int owner) 466 { 467 void **srv_opsp; 468 void **clt_opsp; 469 int i; 470 int total_ops; 471 472 /* fix up psm ops */ 473 srv_opsp = (void **)mach_set[0]; 474 clt_opsp = (void **)mach_set[owner]; 475 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 476 total_ops = sizeof (struct psm_ops_ver01) / 477 sizeof (void (*)(void)); 478 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 479 /* no psm_notify_func */ 480 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 481 sizeof (void (*)(void)); 482 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 483 /* no psm_timer funcs */ 484 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 485 sizeof (void (*)(void)); 486 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 487 /* no psm_preshutdown function */ 488 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 489 sizeof (void (*)(void)); 490 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 491 /* no psm_preshutdown function */ 492 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 493 sizeof (void (*)(void)); 494 else 495 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 496 497 /* 498 * Save the version of the PSM module, in case we need to 499 * bahave differently based on version. 500 */ 501 mach_ver[0] = mach_ver[owner]; 502 503 for (i = 0; i < total_ops; i++) 504 if (clt_opsp[i] != NULL) 505 srv_opsp[i] = clt_opsp[i]; 506 } 507 508 static void 509 mach_construct_info() 510 { 511 struct psm_sw *swp; 512 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 513 int conflict_owner = 0; 514 515 if (psmsw->psw_forw == psmsw) 516 panic("No valid PSM modules found"); 517 mutex_enter(&psmsw_lock); 518 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 519 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 520 continue; 521 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 522 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 523 mach_cnt[swp->psw_infop->p_owner]++; 524 } 525 mutex_exit(&psmsw_lock); 526 527 mach_get_platform(PSM_OWN_SYS_DEFAULT); 528 529 /* check to see are there any conflicts */ 530 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 531 conflict_owner = PSM_OWN_EXCLUSIVE; 532 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 533 conflict_owner = PSM_OWN_OVERRIDE; 534 if (conflict_owner) { 535 /* remove all psm modules except uppc */ 536 cmn_err(CE_WARN, 537 "Conflicts detected on the following PSM modules:"); 538 mutex_enter(&psmsw_lock); 539 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 540 if (swp->psw_infop->p_owner == conflict_owner) 541 cmn_err(CE_WARN, "%s ", 542 swp->psw_infop->p_mach_idstring); 543 } 544 mutex_exit(&psmsw_lock); 545 cmn_err(CE_WARN, 546 "Setting the system back to SINGLE processor mode!"); 547 cmn_err(CE_WARN, 548 "Please edit /etc/mach to remove the invalid PSM module."); 549 return; 550 } 551 552 if (mach_set[PSM_OWN_EXCLUSIVE]) 553 mach_get_platform(PSM_OWN_EXCLUSIVE); 554 555 if (mach_set[PSM_OWN_OVERRIDE]) 556 mach_get_platform(PSM_OWN_OVERRIDE); 557 } 558 559 static void 560 mach_init() 561 { 562 struct psm_ops *pops; 563 564 mach_construct_info(); 565 566 pops = mach_set[0]; 567 568 /* register the interrupt and clock initialization rotuines */ 569 picinitf = mach_picinit; 570 clkinitf = mach_clkinit; 571 psm_get_clockirq = pops->psm_get_clockirq; 572 573 /* register the interrupt setup code */ 574 slvltovect = mach_softlvl_to_vect; 575 addspl = pops->psm_addspl; 576 delspl = pops->psm_delspl; 577 578 if (pops->psm_translate_irq) 579 psm_translate_irq = pops->psm_translate_irq; 580 if (pops->psm_intr_ops) 581 psm_intr_ops = pops->psm_intr_ops; 582 583 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 584 /* 585 * Time-of-day functionality now handled in TOD modules. 586 * (Warn about PSM modules that think that we're going to use 587 * their ops vectors.) 588 */ 589 if (pops->psm_tod_get) 590 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 591 (void *)pops->psm_tod_get); 592 593 if (pops->psm_tod_set) 594 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 595 (void *)pops->psm_tod_set); 596 #endif 597 598 if (pops->psm_notify_error) { 599 psm_notify_error = mach_notify_error; 600 notify_error = pops->psm_notify_error; 601 } 602 603 (*pops->psm_softinit)(); 604 605 /* 606 * Initialize the dispatcher's function hooks 607 * to enable CPU halting when idle 608 */ 609 if (idle_cpu_use_hlt) 610 idle_cpu = cpu_idle; 611 612 mach_smpinit(); 613 } 614 615 static void 616 mach_smpinit(void) 617 { 618 struct psm_ops *pops; 619 processorid_t cpu_id; 620 int cnt; 621 cpuset_t cpumask; 622 623 pops = mach_set[0]; 624 625 cpu_id = -1; 626 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 627 for (cnt = 0, CPUSET_ZERO(cpumask); cpu_id != -1; cnt++) { 628 CPUSET_ADD(cpumask, cpu_id); 629 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 630 } 631 632 mp_cpus = cpumask; 633 634 /* MP related routines */ 635 ap_mlsetup = pops->psm_post_cpu_start; 636 send_dirintf = pops->psm_send_ipi; 637 638 /* optional MP related routines */ 639 if (pops->psm_shutdown) 640 psm_shutdownf = pops->psm_shutdown; 641 if (pops->psm_preshutdown) 642 psm_preshutdownf = pops->psm_preshutdown; 643 if (pops->psm_notify_func) 644 psm_notifyf = pops->psm_notify_func; 645 if (pops->psm_set_idlecpu) 646 psm_set_idle_cpuf = pops->psm_set_idlecpu; 647 if (pops->psm_unset_idlecpu) 648 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 649 650 psm_clkinit = pops->psm_clkinit; 651 652 if (pops->psm_timer_reprogram) 653 psm_timer_reprogram = pops->psm_timer_reprogram; 654 655 if (pops->psm_timer_enable) 656 psm_timer_enable = pops->psm_timer_enable; 657 658 if (pops->psm_timer_disable) 659 psm_timer_disable = pops->psm_timer_disable; 660 661 if (pops->psm_post_cyclic_setup) 662 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 663 664 /* check for multiple cpu's */ 665 if (cnt < 2) 666 return; 667 668 /* check for MP platforms */ 669 if (pops->psm_cpu_start == NULL) 670 return; 671 672 /* 673 * Set the dispatcher hook to enable cpu "wake up" 674 * when a thread becomes runnable. 675 */ 676 if (idle_cpu_use_hlt) 677 disp_enq_thread = cpu_wakeup; 678 679 if (pops->psm_disable_intr) 680 psm_disable_intr = pops->psm_disable_intr; 681 if (pops->psm_enable_intr) 682 psm_enable_intr = pops->psm_enable_intr; 683 684 psm_get_ipivect = pops->psm_get_ipivect; 685 686 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr", 687 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 688 (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL); 689 (void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr", 690 (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO), 691 (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL); 692 693 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 694 } 695 696 static void 697 mach_picinit() 698 { 699 struct psm_ops *pops; 700 701 pops = mach_set[0]; 702 703 /* register the interrupt handlers */ 704 setlvl = pops->psm_intr_enter; 705 setlvlx = pops->psm_intr_exit; 706 707 /* initialize the interrupt hardware */ 708 (*pops->psm_picinit)(); 709 710 /* set interrupt mask for current ipl */ 711 setspl = pops->psm_setspl; 712 cli(); 713 setspl(CPU->cpu_pri); 714 } 715 716 uint_t cpu_freq; /* MHz */ 717 uint64_t cpu_freq_hz; /* measured (in hertz) */ 718 719 #define MEGA_HZ 1000000 720 721 static uint64_t 722 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 723 { 724 uint64_t cpu_hz; 725 726 if ((pit_counter == 0) || (*processor_clks == 0) || 727 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 728 return (0); 729 730 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 731 732 return (cpu_hz); 733 } 734 735 static uint64_t 736 mach_getcpufreq(void) 737 { 738 uint32_t pit_counter; 739 uint64_t processor_clks; 740 741 if (x86_feature & X86_TSC) { 742 /* 743 * We have a TSC. freq_tsc() knows how to measure the number 744 * of clock cycles sampled against the PIT. 745 */ 746 ulong_t flags = clear_int_flag(); 747 processor_clks = freq_tsc(&pit_counter); 748 restore_int_flag(flags); 749 return (mach_calchz(pit_counter, &processor_clks)); 750 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 751 #if defined(__amd64) 752 panic("mach_getcpufreq: no TSC!"); 753 #elif defined(__i386) 754 /* 755 * We are a Cyrix based on a 6x86 core or an Intel Pentium 756 * for which freq_notsc() knows how to measure the number of 757 * elapsed clock cycles sampled against the PIT 758 */ 759 ulong_t flags = clear_int_flag(); 760 processor_clks = freq_notsc(&pit_counter); 761 restore_int_flag(flags); 762 return (mach_calchz(pit_counter, &processor_clks)); 763 #endif /* __i386 */ 764 } 765 766 /* We do not know how to calculate cpu frequency for this cpu. */ 767 return (0); 768 } 769 770 /* 771 * If the clock speed of a cpu is found to be reported incorrectly, do not add 772 * to this array, instead improve the accuracy of the algorithm that determines 773 * the clock speed of the processor or extend the implementation to support the 774 * vendor as appropriate. This is here only to support adjusting the speed on 775 * older slower processors that mach_fixcpufreq() would not be able to account 776 * for otherwise. 777 */ 778 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 779 780 /* 781 * On fast processors the clock frequency that is measured may be off by 782 * a few MHz from the value printed on the part. This is a combination of 783 * the factors that for such fast parts being off by this much is within 784 * the tolerances for manufacture and because of the difficulties in the 785 * measurement that can lead to small error. This function uses some 786 * heuristics in order to tweak the value that was measured to match what 787 * is most likely printed on the part. 788 * 789 * Some examples: 790 * AMD Athlon 1000 mhz measured as 998 mhz 791 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 792 * Intel Pentium IV 1500 mhz measured as 1495mhz 793 * 794 * If in the future this function is no longer sufficient to correct 795 * for the error in the measurement, then the algorithm used to perform 796 * the measurement will have to be improved in order to increase accuracy 797 * rather than adding horrible and questionable kludges here. 798 * 799 * This is called after the cyclics subsystem because of the potential 800 * that the heuristics within may give a worse estimate of the clock 801 * frequency than the value that was measured. 802 */ 803 static void 804 mach_fixcpufreq(void) 805 { 806 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 807 808 freq = (uint32_t)cpu_freq; 809 810 /* 811 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 812 * measured speed taking into account that the 667 MHz parts were 813 * the first to round-up. 814 */ 815 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 816 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 817 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 818 819 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 820 mul = (freq + 25) / 50; 821 near50 = mul * 50; 822 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 823 824 /* Find the closer of the two */ 825 if (delta66 < delta50) { 826 fixed = near66; 827 delta = delta66; 828 } else { 829 fixed = near50; 830 delta = delta50; 831 } 832 833 if (fixed > INT_MAX) 834 return; 835 836 /* 837 * Some older parts have a core clock frequency that is not an 838 * integral multiple of 50 or 66 MHz. Check if one of the old 839 * clock frequencies is closer to the measured value than any 840 * of the integral multiples of 50 an 66, and if so set fixed 841 * and delta appropriately to represent the closest value. 842 */ 843 i = sizeof (x86_cpu_freq) / sizeof (int); 844 while (i > 0) { 845 i--; 846 847 if (x86_cpu_freq[i] <= freq) { 848 mul = freq - x86_cpu_freq[i]; 849 850 if (mul < delta) { 851 fixed = x86_cpu_freq[i]; 852 delta = mul; 853 } 854 855 break; 856 } 857 858 mul = x86_cpu_freq[i] - freq; 859 860 if (mul < delta) { 861 fixed = x86_cpu_freq[i]; 862 delta = mul; 863 } 864 } 865 866 /* 867 * Set a reasonable maximum for how much to correct the measured 868 * result by. This check is here to prevent the adjustment made 869 * by this function from being more harm than good. It is entirely 870 * possible that in the future parts will be made that are not 871 * integral multiples of 66 or 50 in clock frequency or that 872 * someone may overclock a part to some odd frequency. If the 873 * measured value is farther from the corrected value than 874 * allowed, then assume the corrected value is in error and use 875 * the measured value. 876 */ 877 if (6 < delta) 878 return; 879 880 cpu_freq = (int)fixed; 881 } 882 883 884 static int 885 machhztomhz(uint64_t cpu_freq_hz) 886 { 887 uint64_t cpu_mhz; 888 889 /* Round to nearest MHZ */ 890 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 891 892 if (cpu_mhz > INT_MAX) 893 return (0); 894 895 return ((int)cpu_mhz); 896 897 } 898 899 900 static int 901 mach_clkinit(int preferred_mode, int *set_mode) 902 { 903 struct psm_ops *pops; 904 int resolution; 905 906 pops = mach_set[0]; 907 908 cpu_freq_hz = mach_getcpufreq(); 909 910 cpu_freq = machhztomhz(cpu_freq_hz); 911 912 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 913 tsc_gethrtime_enable = 0; 914 915 if (tsc_gethrtime_enable) { 916 tsc_hrtimeinit(cpu_freq_hz); 917 gethrtimef = tsc_gethrtime; 918 gethrtimeunscaledf = tsc_gethrtimeunscaled; 919 scalehrtimef = tsc_scalehrtime; 920 hrtime_tick = tsc_tick; 921 tsc_gethrtime_initted = 1; 922 } else { 923 if (pops->psm_hrtimeinit) 924 (*pops->psm_hrtimeinit)(); 925 gethrtimef = pops->psm_gethrtime; 926 gethrtimeunscaledf = gethrtimef; 927 /* scalehrtimef will remain dummy */ 928 } 929 930 mach_fixcpufreq(); 931 932 if (mach_ver[0] >= PSM_INFO_VER01_3) { 933 if ((preferred_mode == TIMER_ONESHOT) && 934 (tsc_gethrtime_enable)) { 935 936 resolution = (*pops->psm_clkinit)(0); 937 if (resolution != 0) { 938 *set_mode = TIMER_ONESHOT; 939 return (resolution); 940 } 941 942 } 943 944 /* 945 * either periodic mode was requested or could not set to 946 * one-shot mode 947 */ 948 resolution = (*pops->psm_clkinit)(hz); 949 /* 950 * psm should be able to do periodic, so we do not check 951 * for return value of psm_clkinit here. 952 */ 953 *set_mode = TIMER_PERIODIC; 954 return (resolution); 955 } else { 956 /* 957 * PSMI interface prior to PSMI_3 does not define a return 958 * value for psm_clkinit, so the return value is ignored. 959 */ 960 (void) (*pops->psm_clkinit)(hz); 961 *set_mode = TIMER_PERIODIC; 962 return (nsec_per_tick); 963 } 964 } 965 966 /*ARGSUSED*/ 967 static void 968 mach_psm_set_softintr(int ipl, struct av_softinfo *pending) 969 { 970 struct psm_ops *pops; 971 972 /* invoke hardware interrupt */ 973 pops = mach_set[0]; 974 (*pops->psm_set_softintr)(ipl); 975 } 976 977 static int 978 mach_softlvl_to_vect(int ipl) 979 { 980 int softvect; 981 struct psm_ops *pops; 982 983 pops = mach_set[0]; 984 985 /* check for null handler for set soft interrupt call */ 986 if (pops->psm_set_softintr == NULL) { 987 setsoftint = av_set_softint_pending; 988 return (PSM_SV_SOFTWARE); 989 } 990 991 softvect = (*pops->psm_softlvl_to_irq)(ipl); 992 /* check for hardware scheme */ 993 if (softvect > PSM_SV_SOFTWARE) { 994 setsoftint = mach_psm_set_softintr; 995 return (softvect); 996 } 997 998 if (softvect == PSM_SV_SOFTWARE) 999 setsoftint = av_set_softint_pending; 1000 else /* hardware and software mixed scheme */ 1001 setsoftint = mach_set_softintr; 1002 1003 return (PSM_SV_SOFTWARE); 1004 } 1005 1006 static void 1007 mach_set_softintr(int ipl, struct av_softinfo *pending) 1008 { 1009 struct psm_ops *pops; 1010 1011 /* set software pending bits */ 1012 av_set_softint_pending(ipl, pending); 1013 1014 /* check if dosoftint will be called at the end of intr */ 1015 if (CPU_ON_INTR(CPU) || (curthread->t_intr)) 1016 return; 1017 1018 /* invoke hardware interrupt */ 1019 pops = mach_set[0]; 1020 (*pops->psm_set_softintr)(ipl); 1021 } 1022 1023 #ifdef DEBUG 1024 /* 1025 * This is here to allow us to simulate cpus that refuse to start. 1026 */ 1027 cpuset_t cpufailset; 1028 #endif 1029 1030 int 1031 mach_cpu_start(struct cpu *cp, void *ctx) 1032 { 1033 struct psm_ops *pops = mach_set[0]; 1034 processorid_t id = cp->cpu_id; 1035 1036 #ifdef DEBUG 1037 if (CPU_IN_SET(cpufailset, id)) 1038 return (0); 1039 #endif 1040 return ((*pops->psm_cpu_start)(id, ctx)); 1041 } 1042 1043 /*ARGSUSED*/ 1044 static int 1045 mach_translate_irq(dev_info_t *dip, int irqno) 1046 { 1047 return (irqno); /* default to NO translation */ 1048 } 1049 1050 static void 1051 mach_notify_error(int level, char *errmsg) 1052 { 1053 /* 1054 * SL_FATAL is pass in once panicstr is set, deliver it 1055 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1056 * codes for the psmi handler 1057 */ 1058 if (level & SL_FATAL) 1059 (*notify_error)(CE_PANIC, errmsg); 1060 else if (level & SL_WARN) 1061 (*notify_error)(CE_WARN, errmsg); 1062 else if (level & SL_NOTE) 1063 (*notify_error)(CE_NOTE, errmsg); 1064 else if (level & SL_CONSOLE) 1065 (*notify_error)(CE_CONT, errmsg); 1066 } 1067 1068 /* 1069 * It provides the default basic intr_ops interface for the new DDI 1070 * interrupt framework if the PSM doesn't have one. 1071 * 1072 * Input: 1073 * dip - pointer to the dev_info structure of the requested device 1074 * hdlp - pointer to the internal interrupt handle structure for the 1075 * requested interrupt 1076 * intr_op - opcode for this call 1077 * result - pointer to the integer that will hold the result to be 1078 * passed back if return value is PSM_SUCCESS 1079 * 1080 * Output: 1081 * return value is either PSM_SUCCESS or PSM_FAILURE 1082 */ 1083 static int 1084 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1085 psm_intr_op_t intr_op, int *result) 1086 { 1087 struct intrspec *ispec; 1088 1089 switch (intr_op) { 1090 case PSM_INTR_OP_CHECK_MSI: 1091 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1092 DDI_INTR_TYPE_MSIX); 1093 break; 1094 case PSM_INTR_OP_ALLOC_VECTORS: 1095 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1096 *result = 1; 1097 else 1098 *result = 0; 1099 break; 1100 case PSM_INTR_OP_FREE_VECTORS: 1101 break; 1102 case PSM_INTR_OP_NAVAIL_VECTORS: 1103 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1104 *result = 1; 1105 else 1106 *result = 0; 1107 break; 1108 case PSM_INTR_OP_XLATE_VECTOR: 1109 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1110 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1111 break; 1112 case PSM_INTR_OP_GET_CAP: 1113 *result = 0; 1114 break; 1115 case PSM_INTR_OP_GET_PENDING: 1116 case PSM_INTR_OP_CLEAR_MASK: 1117 case PSM_INTR_OP_SET_MASK: 1118 case PSM_INTR_OP_GET_SHARED: 1119 case PSM_INTR_OP_SET_PRI: 1120 case PSM_INTR_OP_SET_CAP: 1121 case PSM_INTR_OP_SET_CPU: 1122 case PSM_INTR_OP_GET_INTR: 1123 default: 1124 return (PSM_FAILURE); 1125 } 1126 return (PSM_SUCCESS); 1127 } 1128