1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #define PSMI_1_5 29 #include <sys/smp_impldefs.h> 30 #include <sys/psm.h> 31 #include <sys/psm_modctl.h> 32 #include <sys/pit.h> 33 #include <sys/cmn_err.h> 34 #include <sys/strlog.h> 35 #include <sys/clock.h> 36 #include <sys/debug.h> 37 #include <sys/rtc.h> 38 #include <sys/x86_archext.h> 39 #include <sys/cpupart.h> 40 #include <sys/cpuvar.h> 41 #include <sys/pghw.h> 42 #include <sys/disp.h> 43 #include <sys/cpu.h> 44 #include <sys/archsystm.h> 45 #include <sys/mach_intr.h> 46 47 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 48 49 /* 50 * Local function prototypes 51 */ 52 static int mp_disable_intr(processorid_t cpun); 53 static void mp_enable_intr(processorid_t cpun); 54 static void mach_init(); 55 static void mach_picinit(); 56 static uint64_t mach_calchz(uint32_t pit_counter, uint64_t *processor_clks); 57 static int machhztomhz(uint64_t cpu_freq_hz); 58 static uint64_t mach_getcpufreq(void); 59 static void mach_fixcpufreq(void); 60 static int mach_clkinit(int, int *); 61 static void mach_smpinit(void); 62 static void mach_set_softintr(int ipl, struct av_softinfo *); 63 static void mach_cpu_start(int cpun); 64 static int mach_softlvl_to_vect(int ipl); 65 static void mach_get_platform(int owner); 66 static void mach_construct_info(); 67 static int mach_translate_irq(dev_info_t *dip, int irqno); 68 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 69 psm_intr_op_t, int *); 70 static timestruc_t mach_tod_get(void); 71 static void mach_tod_set(timestruc_t ts); 72 static void mach_notify_error(int level, char *errmsg); 73 static hrtime_t dummy_hrtime(void); 74 static void dummy_scalehrtime(hrtime_t *); 75 static void cpu_halt(void); 76 static void cpu_wakeup(cpu_t *, int); 77 /* 78 * External reference functions 79 */ 80 extern void return_instr(); 81 extern timestruc_t (*todgetf)(void); 82 extern void (*todsetf)(timestruc_t); 83 extern long gmt_lag; 84 extern uint64_t freq_tsc(uint32_t *); 85 #if defined(__i386) 86 extern uint64_t freq_notsc(uint32_t *); 87 #endif 88 extern void pc_gethrestime(timestruc_t *); 89 extern int cpuid_get_coreid(cpu_t *); 90 extern int cpuid_get_chipid(cpu_t *); 91 92 /* 93 * PSM functions initialization 94 */ 95 void (*psm_shutdownf)(int, int) = return_instr; 96 void (*psm_preshutdownf)(int, int) = return_instr; 97 void (*psm_notifyf)(int) = return_instr; 98 void (*psm_set_idle_cpuf)(int) = return_instr; 99 void (*psm_unset_idle_cpuf)(int) = return_instr; 100 void (*psminitf)() = mach_init; 101 void (*picinitf)() = return_instr; 102 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 103 void (*cpu_startf)() = return_instr; 104 int (*ap_mlsetup)() = (int (*)(void))return_instr; 105 void (*send_dirintf)() = return_instr; 106 void (*setspl)(int) = return_instr; 107 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 108 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 109 void (*setsoftint)(int, struct av_softinfo *)= 110 (void (*)(int, struct av_softinfo *))return_instr; 111 int (*slvltovect)(int) = (int (*)(int))return_instr; 112 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 113 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 114 int (*psm_disable_intr)(int) = mp_disable_intr; 115 void (*psm_enable_intr)(int) = mp_enable_intr; 116 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 117 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 118 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 119 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 120 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 121 int (*psm_todgetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 122 int (*psm_todsetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 123 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 124 int (*psm_get_clockirq)(int) = NULL; 125 int (*psm_get_ipivect)(int, int) = NULL; 126 127 int (*psm_clkinit)(int) = NULL; 128 void (*psm_timer_reprogram)(hrtime_t) = NULL; 129 void (*psm_timer_enable)(void) = NULL; 130 void (*psm_timer_disable)(void) = NULL; 131 void (*psm_post_cyclic_setup)(void *arg) = NULL; 132 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 133 int *) = mach_intr_ops; 134 135 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 136 void (*hrtime_tick)(void) = return_instr; 137 138 int tsc_gethrtime_enable = 1; 139 int tsc_gethrtime_initted = 0; 140 141 /* 142 * Local Static Data 143 */ 144 static struct psm_ops mach_ops; 145 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 146 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 147 148 /* 149 * If non-zero, idle cpus will "halted" when there's 150 * no work to do. 151 */ 152 int halt_idle_cpus = 1; 153 154 #if defined(__amd64) 155 /* 156 * If non-zero, will use cr8 for interrupt priority masking 157 * We declare this here since install_spl is called from here 158 * (where this is checked). 159 */ 160 int intpri_use_cr8 = 0; 161 #endif /* __amd64 */ 162 163 #ifdef _SIMULATOR_SUPPORT 164 165 int simulator_run = 0; /* patch to non-zero if running under simics */ 166 167 #endif /* _SIMULATOR_SUPPORT */ 168 169 170 /*ARGSUSED*/ 171 int 172 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 173 { 174 switch (hw) { 175 case PGHW_IPIPE: 176 if (x86_feature & (X86_HTT)) { 177 /* 178 * Hyper-threading is SMT 179 */ 180 return (1); 181 } else { 182 return (0); 183 } 184 case PGHW_CHIP: 185 if (x86_feature & (X86_CMP|X86_HTT)) 186 return (1); 187 else 188 return (0); 189 default: 190 return (0); 191 } 192 } 193 194 /* 195 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 196 * If pghw_type_t is an unsupported hardware type, then return -1 197 */ 198 int 199 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 200 { 201 id_t pgp_a, pgp_b; 202 203 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 204 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 205 206 if (pgp_a == -1 || pgp_b == -1) 207 return (-1); 208 209 return (pgp_a == pgp_b); 210 } 211 212 /* 213 * Return a physical instance identifier for known hardware sharing 214 * relationships 215 */ 216 id_t 217 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 218 { 219 switch (hw) { 220 case PGHW_IPIPE: 221 return (cpuid_get_coreid(cpu)); 222 case PGHW_CHIP: 223 return (cpuid_get_chipid(cpu)); 224 default: 225 return (-1); 226 } 227 } 228 229 int 230 pg_plat_hw_level(pghw_type_t hw) 231 { 232 int i; 233 static pghw_type_t hw_hier[] = { 234 PGHW_IPIPE, 235 PGHW_CHIP, 236 PGHW_NUM_COMPONENTS 237 }; 238 239 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 240 if (hw_hier[i] == hw) 241 return (i); 242 } 243 return (-1); 244 } 245 246 id_t 247 pg_plat_get_core_id(cpu_t *cpu) 248 { 249 return ((id_t)cpuid_get_coreid(cpu)); 250 } 251 252 void 253 cmp_set_nosteal_interval(void) 254 { 255 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 256 nosteal_nsec = 100000UL; 257 } 258 259 /* 260 * Routine to ensure initial callers to hrtime gets 0 as return 261 */ 262 static hrtime_t 263 dummy_hrtime(void) 264 { 265 return (0); 266 } 267 268 /* ARGSUSED */ 269 static void 270 dummy_scalehrtime(hrtime_t *ticks) 271 {} 272 273 /* 274 * Halt the present CPU until awoken via an interrupt 275 */ 276 static void 277 cpu_halt(void) 278 { 279 cpu_t *cpup = CPU; 280 processorid_t cpun = cpup->cpu_id; 281 cpupart_t *cp = cpup->cpu_part; 282 int hset_update = 1; 283 284 /* 285 * If this CPU is online, and there's multiple CPUs 286 * in the system, then we should notate our halting 287 * by adding ourselves to the partition's halted CPU 288 * bitmap. This allows other CPUs to find/awaken us when 289 * work becomes available. 290 */ 291 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 292 hset_update = 0; 293 294 /* 295 * Add ourselves to the partition's halted CPUs bitmask 296 * and set our HALTED flag, if necessary. 297 * 298 * When a thread becomes runnable, it is placed on the queue 299 * and then the halted cpuset is checked to determine who 300 * (if anyone) should be awoken. We therefore need to first 301 * add ourselves to the halted cpuset, and and then check if there 302 * is any work available. 303 * 304 * Note that memory barriers after updating the HALTED flag 305 * are not necessary since an atomic operation (updating the bitmap) 306 * immediately follows. On x86 the atomic operation acts as a 307 * memory barrier for the update of cpu_disp_flags. 308 */ 309 if (hset_update) { 310 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 311 CPUSET_ATOMIC_ADD(cp->cp_mach->mc_haltset, cpun); 312 } 313 314 /* 315 * Check to make sure there's really nothing to do. 316 * Work destined for this CPU may become available after 317 * this check. We'll be notified through the clearing of our 318 * bit in the halted CPU bitmask, and a poke. 319 */ 320 if (disp_anywork()) { 321 if (hset_update) { 322 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 323 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 324 } 325 return; 326 } 327 328 /* 329 * We're on our way to being halted. 330 * 331 * Disable interrupts now, so that we'll awaken immediately 332 * after halting if someone tries to poke us between now and 333 * the time we actually halt. 334 * 335 * We check for the presence of our bit after disabling interrupts. 336 * If it's cleared, we'll return. If the bit is cleared after 337 * we check then the poke will pop us out of the halted state. 338 * 339 * This means that the ordering of the poke and the clearing 340 * of the bit by cpu_wakeup is important. 341 * cpu_wakeup() must clear, then poke. 342 * cpu_halt() must disable interrupts, then check for the bit. 343 */ 344 cli(); 345 346 if (hset_update && !CPU_IN_SET(cp->cp_mach->mc_haltset, cpun)) { 347 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 348 sti(); 349 return; 350 } 351 352 /* 353 * The check for anything locally runnable is here for performance 354 * and isn't needed for correctness. disp_nrunnable ought to be 355 * in our cache still, so it's inexpensive to check, and if there 356 * is anything runnable we won't have to wait for the poke. 357 */ 358 if (cpup->cpu_disp->disp_nrunnable != 0) { 359 if (hset_update) { 360 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 361 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 362 } 363 sti(); 364 return; 365 } 366 367 /* 368 * Call the halt sequence: 369 * sti 370 * hlt 371 */ 372 i86_halt(); 373 374 /* 375 * We're no longer halted 376 */ 377 if (hset_update) { 378 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 379 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 380 } 381 } 382 383 384 /* 385 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 386 * Otherwise, see if other CPUs in the cpu partition are halted and need to 387 * be woken up so that they can steal the thread we placed on this CPU. 388 * This function is only used on MP systems. 389 */ 390 static void 391 cpu_wakeup(cpu_t *cpu, int bound) 392 { 393 uint_t cpu_found; 394 int result; 395 cpupart_t *cp; 396 397 cp = cpu->cpu_part; 398 if (CPU_IN_SET(cp->cp_mach->mc_haltset, cpu->cpu_id)) { 399 /* 400 * Clear the halted bit for that CPU since it will be 401 * poked in a moment. 402 */ 403 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpu->cpu_id); 404 /* 405 * We may find the current CPU present in the halted cpuset 406 * if we're in the context of an interrupt that occurred 407 * before we had a chance to clear our bit in cpu_halt(). 408 * Poking ourself is obviously unnecessary, since if 409 * we're here, we're not halted. 410 */ 411 if (cpu != CPU) 412 poke_cpu(cpu->cpu_id); 413 return; 414 } else { 415 /* 416 * This cpu isn't halted, but it's idle or undergoing a 417 * context switch. No need to awaken anyone else. 418 */ 419 if (cpu->cpu_thread == cpu->cpu_idle_thread || 420 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 421 return; 422 } 423 424 /* 425 * No need to wake up other CPUs if the thread we just enqueued 426 * is bound. 427 */ 428 if (bound) 429 return; 430 431 432 /* 433 * See if there's any other halted CPUs. If there are, then 434 * select one, and awaken it. 435 * It's possible that after we find a CPU, somebody else 436 * will awaken it before we get the chance. 437 * In that case, look again. 438 */ 439 do { 440 CPUSET_FIND(cp->cp_mach->mc_haltset, cpu_found); 441 if (cpu_found == CPUSET_NOTINSET) 442 return; 443 444 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 445 CPUSET_ATOMIC_XDEL(cp->cp_mach->mc_haltset, cpu_found, result); 446 } while (result < 0); 447 448 if (cpu_found != CPU->cpu_id) 449 poke_cpu(cpu_found); 450 } 451 452 static int 453 mp_disable_intr(int cpun) 454 { 455 /* 456 * switch to the offline cpu 457 */ 458 affinity_set(cpun); 459 /* 460 * raise ipl to just below cross call 461 */ 462 splx(XC_MED_PIL-1); 463 /* 464 * set base spl to prevent the next swtch to idle from 465 * lowering back to ipl 0 466 */ 467 CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1)); 468 set_base_spl(); 469 affinity_clear(); 470 return (DDI_SUCCESS); 471 } 472 473 static void 474 mp_enable_intr(int cpun) 475 { 476 /* 477 * switch to the online cpu 478 */ 479 affinity_set(cpun); 480 /* 481 * clear the interrupt active mask 482 */ 483 CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1)); 484 set_base_spl(); 485 (void) spl0(); 486 affinity_clear(); 487 } 488 489 static void 490 mach_get_platform(int owner) 491 { 492 void **srv_opsp; 493 void **clt_opsp; 494 int i; 495 int total_ops; 496 497 /* fix up psm ops */ 498 srv_opsp = (void **)mach_set[0]; 499 clt_opsp = (void **)mach_set[owner]; 500 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 501 total_ops = sizeof (struct psm_ops_ver01) / 502 sizeof (void (*)(void)); 503 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 504 /* no psm_notify_func */ 505 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 506 sizeof (void (*)(void)); 507 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 508 /* no psm_timer funcs */ 509 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 510 sizeof (void (*)(void)); 511 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 512 /* no psm_preshutdown function */ 513 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 514 sizeof (void (*)(void)); 515 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 516 /* no psm_preshutdown function */ 517 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 518 sizeof (void (*)(void)); 519 else 520 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 521 522 /* 523 * Save the version of the PSM module, in case we need to 524 * bahave differently based on version. 525 */ 526 mach_ver[0] = mach_ver[owner]; 527 528 for (i = 0; i < total_ops; i++) 529 if (clt_opsp[i] != NULL) 530 srv_opsp[i] = clt_opsp[i]; 531 } 532 533 static void 534 mach_construct_info() 535 { 536 register struct psm_sw *swp; 537 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 538 int conflict_owner = 0; 539 540 if (psmsw->psw_forw == psmsw) 541 panic("No valid PSM modules found"); 542 mutex_enter(&psmsw_lock); 543 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 544 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 545 continue; 546 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 547 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 548 mach_cnt[swp->psw_infop->p_owner]++; 549 } 550 mutex_exit(&psmsw_lock); 551 552 mach_get_platform(PSM_OWN_SYS_DEFAULT); 553 554 /* check to see are there any conflicts */ 555 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 556 conflict_owner = PSM_OWN_EXCLUSIVE; 557 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 558 conflict_owner = PSM_OWN_OVERRIDE; 559 if (conflict_owner) { 560 /* remove all psm modules except uppc */ 561 cmn_err(CE_WARN, 562 "Conflicts detected on the following PSM modules:"); 563 mutex_enter(&psmsw_lock); 564 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 565 if (swp->psw_infop->p_owner == conflict_owner) 566 cmn_err(CE_WARN, "%s ", 567 swp->psw_infop->p_mach_idstring); 568 } 569 mutex_exit(&psmsw_lock); 570 cmn_err(CE_WARN, 571 "Setting the system back to SINGLE processor mode!"); 572 cmn_err(CE_WARN, 573 "Please edit /etc/mach to remove the invalid PSM module."); 574 return; 575 } 576 577 if (mach_set[PSM_OWN_EXCLUSIVE]) 578 mach_get_platform(PSM_OWN_EXCLUSIVE); 579 580 if (mach_set[PSM_OWN_OVERRIDE]) 581 mach_get_platform(PSM_OWN_OVERRIDE); 582 } 583 584 static void 585 mach_init() 586 { 587 register struct psm_ops *pops; 588 589 mach_construct_info(); 590 591 pops = mach_set[0]; 592 593 /* register the interrupt and clock initialization rotuines */ 594 picinitf = mach_picinit; 595 clkinitf = mach_clkinit; 596 psm_get_clockirq = pops->psm_get_clockirq; 597 598 /* register the interrupt setup code */ 599 slvltovect = mach_softlvl_to_vect; 600 addspl = pops->psm_addspl; 601 delspl = pops->psm_delspl; 602 603 if (pops->psm_translate_irq) 604 psm_translate_irq = pops->psm_translate_irq; 605 if (pops->psm_intr_ops) 606 psm_intr_ops = pops->psm_intr_ops; 607 if (pops->psm_tod_get) { 608 todgetf = mach_tod_get; 609 psm_todgetf = pops->psm_tod_get; 610 } 611 if (pops->psm_tod_set) { 612 todsetf = mach_tod_set; 613 psm_todsetf = pops->psm_tod_set; 614 } 615 if (pops->psm_notify_error) { 616 psm_notify_error = mach_notify_error; 617 notify_error = pops->psm_notify_error; 618 } 619 620 (*pops->psm_softinit)(); 621 622 /* 623 * Initialize the dispatcher's function hooks 624 * to enable CPU halting when idle 625 */ 626 #if defined(_SIMULATOR_SUPPORT) 627 if (halt_idle_cpus && !simulator_run) 628 idle_cpu = cpu_halt; 629 #else 630 if (halt_idle_cpus) 631 idle_cpu = cpu_halt; 632 #endif /* _SIMULATOR_SUPPORT */ 633 634 mach_smpinit(); 635 } 636 637 static void 638 mach_smpinit(void) 639 { 640 struct psm_ops *pops; 641 processorid_t cpu_id; 642 int cnt; 643 cpuset_t cpumask; 644 645 pops = mach_set[0]; 646 647 cpu_id = -1; 648 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 649 for (cnt = 0, CPUSET_ZERO(cpumask); cpu_id != -1; cnt++) { 650 CPUSET_ADD(cpumask, cpu_id); 651 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 652 } 653 654 mp_cpus = cpumask; 655 656 /* MP related routines */ 657 cpu_startf = mach_cpu_start; 658 ap_mlsetup = pops->psm_post_cpu_start; 659 send_dirintf = pops->psm_send_ipi; 660 661 /* optional MP related routines */ 662 if (pops->psm_shutdown) 663 psm_shutdownf = pops->psm_shutdown; 664 if (pops->psm_preshutdown) 665 psm_preshutdownf = pops->psm_preshutdown; 666 if (pops->psm_notify_func) 667 psm_notifyf = pops->psm_notify_func; 668 if (pops->psm_set_idlecpu) 669 psm_set_idle_cpuf = pops->psm_set_idlecpu; 670 if (pops->psm_unset_idlecpu) 671 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 672 673 psm_clkinit = pops->psm_clkinit; 674 675 if (pops->psm_timer_reprogram) 676 psm_timer_reprogram = pops->psm_timer_reprogram; 677 678 if (pops->psm_timer_enable) 679 psm_timer_enable = pops->psm_timer_enable; 680 681 if (pops->psm_timer_disable) 682 psm_timer_disable = pops->psm_timer_disable; 683 684 if (pops->psm_post_cyclic_setup) 685 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 686 687 /* check for multiple cpu's */ 688 if (cnt < 2) 689 return; 690 691 /* check for MP platforms */ 692 if (pops->psm_cpu_start == NULL) 693 return; 694 695 /* 696 * Set the dispatcher hook to enable cpu "wake up" 697 * when a thread becomes runnable. 698 */ 699 #if defined(_SIMULATOR_SUPPORT) 700 if (halt_idle_cpus && !simulator_run) { 701 disp_enq_thread = cpu_wakeup; 702 } 703 #else 704 if (halt_idle_cpus) { 705 disp_enq_thread = cpu_wakeup; 706 } 707 #endif /* _SIMULATOR_SUPPORT */ 708 709 if (pops->psm_disable_intr) 710 psm_disable_intr = pops->psm_disable_intr; 711 if (pops->psm_enable_intr) 712 psm_enable_intr = pops->psm_enable_intr; 713 714 psm_get_ipivect = pops->psm_get_ipivect; 715 716 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr", 717 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 718 (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL); 719 (void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr", 720 (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO), 721 (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL); 722 723 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 724 } 725 726 static void 727 mach_picinit() 728 { 729 struct psm_ops *pops; 730 extern void install_spl(void); /* XXX: belongs in a header file */ 731 #if defined(__amd64) && defined(DEBUG) 732 extern void *spl_patch, *slow_spl, *setsplhi_patch, *slow_setsplhi; 733 #endif 734 735 pops = mach_set[0]; 736 737 /* register the interrupt handlers */ 738 setlvl = pops->psm_intr_enter; 739 setlvlx = pops->psm_intr_exit; 740 741 /* initialize the interrupt hardware */ 742 (*pops->psm_picinit)(); 743 744 /* set interrupt mask for current ipl */ 745 setspl = pops->psm_setspl; 746 setspl(CPU->cpu_pri); 747 748 /* Install proper spl routine now that we can Program the PIC */ 749 #if defined(__amd64) 750 /* 751 * It would be better if we could check this at compile time 752 */ 753 ASSERT(((uintptr_t)&slow_setsplhi - (uintptr_t)&setsplhi_patch < 128) && 754 ((uintptr_t)&slow_spl - (uintptr_t)&spl_patch < 128)); 755 #endif 756 install_spl(); 757 } 758 759 uint_t cpu_freq; /* MHz */ 760 uint64_t cpu_freq_hz; /* measured (in hertz) */ 761 762 #define MEGA_HZ 1000000 763 764 static uint64_t 765 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 766 { 767 uint64_t cpu_hz; 768 769 if ((pit_counter == 0) || (*processor_clks == 0) || 770 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 771 return (0); 772 773 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 774 775 return (cpu_hz); 776 } 777 778 static uint64_t 779 mach_getcpufreq(void) 780 { 781 uint32_t pit_counter; 782 uint64_t processor_clks; 783 784 if (x86_feature & X86_TSC) { 785 /* 786 * We have a TSC. freq_tsc() knows how to measure the number 787 * of clock cycles sampled against the PIT. 788 */ 789 processor_clks = freq_tsc(&pit_counter); 790 return (mach_calchz(pit_counter, &processor_clks)); 791 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 792 #if defined(__amd64) 793 panic("mach_getcpufreq: no TSC!"); 794 #elif defined(__i386) 795 /* 796 * We are a Cyrix based on a 6x86 core or an Intel Pentium 797 * for which freq_notsc() knows how to measure the number of 798 * elapsed clock cycles sampled against the PIT 799 */ 800 processor_clks = freq_notsc(&pit_counter); 801 return (mach_calchz(pit_counter, &processor_clks)); 802 #endif /* __i386 */ 803 } 804 805 /* We do not know how to calculate cpu frequency for this cpu. */ 806 return (0); 807 } 808 809 /* 810 * If the clock speed of a cpu is found to be reported incorrectly, do not add 811 * to this array, instead improve the accuracy of the algorithm that determines 812 * the clock speed of the processor or extend the implementation to support the 813 * vendor as appropriate. This is here only to support adjusting the speed on 814 * older slower processors that mach_fixcpufreq() would not be able to account 815 * for otherwise. 816 */ 817 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 818 819 /* 820 * On fast processors the clock frequency that is measured may be off by 821 * a few MHz from the value printed on the part. This is a combination of 822 * the factors that for such fast parts being off by this much is within 823 * the tolerances for manufacture and because of the difficulties in the 824 * measurement that can lead to small error. This function uses some 825 * heuristics in order to tweak the value that was measured to match what 826 * is most likely printed on the part. 827 * 828 * Some examples: 829 * AMD Athlon 1000 mhz measured as 998 mhz 830 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 831 * Intel Pentium IV 1500 mhz measured as 1495mhz 832 * 833 * If in the future this function is no longer sufficient to correct 834 * for the error in the measurement, then the algorithm used to perform 835 * the measurement will have to be improved in order to increase accuracy 836 * rather than adding horrible and questionable kludges here. 837 * 838 * This is called after the cyclics subsystem because of the potential 839 * that the heuristics within may give a worse estimate of the clock 840 * frequency than the value that was measured. 841 */ 842 static void 843 mach_fixcpufreq(void) 844 { 845 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 846 847 freq = (uint32_t)cpu_freq; 848 849 /* 850 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 851 * measured speed taking into account that the 667 MHz parts were 852 * the first to round-up. 853 */ 854 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 855 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 856 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 857 858 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 859 mul = (freq + 25) / 50; 860 near50 = mul * 50; 861 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 862 863 /* Find the closer of the two */ 864 if (delta66 < delta50) { 865 fixed = near66; 866 delta = delta66; 867 } else { 868 fixed = near50; 869 delta = delta50; 870 } 871 872 if (fixed > INT_MAX) 873 return; 874 875 /* 876 * Some older parts have a core clock frequency that is not an 877 * integral multiple of 50 or 66 MHz. Check if one of the old 878 * clock frequencies is closer to the measured value than any 879 * of the integral multiples of 50 an 66, and if so set fixed 880 * and delta appropriately to represent the closest value. 881 */ 882 i = sizeof (x86_cpu_freq) / sizeof (int); 883 while (i > 0) { 884 i--; 885 886 if (x86_cpu_freq[i] <= freq) { 887 mul = freq - x86_cpu_freq[i]; 888 889 if (mul < delta) { 890 fixed = x86_cpu_freq[i]; 891 delta = mul; 892 } 893 894 break; 895 } 896 897 mul = x86_cpu_freq[i] - freq; 898 899 if (mul < delta) { 900 fixed = x86_cpu_freq[i]; 901 delta = mul; 902 } 903 } 904 905 /* 906 * Set a reasonable maximum for how much to correct the measured 907 * result by. This check is here to prevent the adjustment made 908 * by this function from being more harm than good. It is entirely 909 * possible that in the future parts will be made that are not 910 * integral multiples of 66 or 50 in clock frequency or that 911 * someone may overclock a part to some odd frequency. If the 912 * measured value is farther from the corrected value than 913 * allowed, then assume the corrected value is in error and use 914 * the measured value. 915 */ 916 if (6 < delta) 917 return; 918 919 cpu_freq = (int)fixed; 920 } 921 922 923 static int 924 machhztomhz(uint64_t cpu_freq_hz) 925 { 926 uint64_t cpu_mhz; 927 928 /* Round to nearest MHZ */ 929 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 930 931 if (cpu_mhz > INT_MAX) 932 return (0); 933 934 return ((int)cpu_mhz); 935 936 } 937 938 939 static int 940 mach_clkinit(int preferred_mode, int *set_mode) 941 { 942 register struct psm_ops *pops; 943 int resolution; 944 945 pops = mach_set[0]; 946 947 #ifdef _SIMULATOR_SUPPORT 948 if (!simulator_run) 949 cpu_freq_hz = mach_getcpufreq(); 950 else 951 cpu_freq_hz = 40000000; /* use 40 Mhz (hack for simulator) */ 952 #else 953 cpu_freq_hz = mach_getcpufreq(); 954 #endif /* _SIMULATOR_SUPPORT */ 955 956 cpu_freq = machhztomhz(cpu_freq_hz); 957 958 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 959 tsc_gethrtime_enable = 0; 960 961 if (tsc_gethrtime_enable) { 962 tsc_hrtimeinit(cpu_freq_hz); 963 gethrtimef = tsc_gethrtime; 964 gethrtimeunscaledf = tsc_gethrtimeunscaled; 965 scalehrtimef = tsc_scalehrtime; 966 hrtime_tick = tsc_tick; 967 tsc_gethrtime_initted = 1; 968 } else { 969 if (pops->psm_hrtimeinit) 970 (*pops->psm_hrtimeinit)(); 971 gethrtimef = pops->psm_gethrtime; 972 gethrtimeunscaledf = gethrtimef; 973 /* scalehrtimef will remain dummy */ 974 } 975 976 mach_fixcpufreq(); 977 978 if (mach_ver[0] >= PSM_INFO_VER01_3) { 979 if ((preferred_mode == TIMER_ONESHOT) && 980 (tsc_gethrtime_enable)) { 981 982 resolution = (*pops->psm_clkinit)(0); 983 if (resolution != 0) { 984 *set_mode = TIMER_ONESHOT; 985 return (resolution); 986 } 987 988 } 989 990 /* 991 * either periodic mode was requested or could not set to 992 * one-shot mode 993 */ 994 resolution = (*pops->psm_clkinit)(hz); 995 /* 996 * psm should be able to do periodic, so we do not check 997 * for return value of psm_clkinit here. 998 */ 999 *set_mode = TIMER_PERIODIC; 1000 return (resolution); 1001 } else { 1002 /* 1003 * PSMI interface prior to PSMI_3 does not define a return 1004 * value for psm_clkinit, so the return value is ignored. 1005 */ 1006 (void) (*pops->psm_clkinit)(hz); 1007 *set_mode = TIMER_PERIODIC; 1008 return (nsec_per_tick); 1009 } 1010 } 1011 1012 /*ARGSUSED*/ 1013 static void 1014 mach_psm_set_softintr(int ipl, struct av_softinfo *pending) 1015 { 1016 register struct psm_ops *pops; 1017 1018 /* invoke hardware interrupt */ 1019 pops = mach_set[0]; 1020 (*pops->psm_set_softintr)(ipl); 1021 } 1022 1023 static int 1024 mach_softlvl_to_vect(register int ipl) 1025 { 1026 register int softvect; 1027 register struct psm_ops *pops; 1028 1029 pops = mach_set[0]; 1030 1031 /* check for null handler for set soft interrupt call */ 1032 if (pops->psm_set_softintr == NULL) { 1033 setsoftint = av_set_softint_pending; 1034 return (PSM_SV_SOFTWARE); 1035 } 1036 1037 softvect = (*pops->psm_softlvl_to_irq)(ipl); 1038 /* check for hardware scheme */ 1039 if (softvect > PSM_SV_SOFTWARE) { 1040 setsoftint = mach_psm_set_softintr; 1041 return (softvect); 1042 } 1043 1044 if (softvect == PSM_SV_SOFTWARE) 1045 setsoftint = av_set_softint_pending; 1046 else /* hardware and software mixed scheme */ 1047 setsoftint = mach_set_softintr; 1048 1049 return (PSM_SV_SOFTWARE); 1050 } 1051 1052 static void 1053 mach_set_softintr(register int ipl, struct av_softinfo *pending) 1054 { 1055 register struct psm_ops *pops; 1056 1057 /* set software pending bits */ 1058 av_set_softint_pending(ipl, pending); 1059 1060 /* check if dosoftint will be called at the end of intr */ 1061 if (CPU_ON_INTR(CPU) || (curthread->t_intr)) 1062 return; 1063 1064 /* invoke hardware interrupt */ 1065 pops = mach_set[0]; 1066 (*pops->psm_set_softintr)(ipl); 1067 } 1068 1069 static void 1070 mach_cpu_start(register int cpun) 1071 { 1072 struct psm_ops *pops; 1073 1074 pops = mach_set[0]; 1075 1076 (*pops->psm_cpu_start)(cpun, rm_platter_va); 1077 } 1078 1079 /*ARGSUSED*/ 1080 static int 1081 mach_translate_irq(dev_info_t *dip, int irqno) 1082 { 1083 return (irqno); /* default to NO translation */ 1084 } 1085 1086 static timestruc_t 1087 mach_tod_get(void) 1088 { 1089 timestruc_t ts; 1090 todinfo_t tod; 1091 static int mach_range_warn = 1; /* warn only once */ 1092 1093 ASSERT(MUTEX_HELD(&tod_lock)); 1094 1095 /* The year returned from is the last 2 digit only */ 1096 if ((*psm_todgetf)(&tod)) { 1097 ts.tv_sec = 0; 1098 ts.tv_nsec = 0; 1099 tod_fault_reset(); 1100 return (ts); 1101 } 1102 1103 /* assume that we wrap the rtc year back to zero at 2000 */ 1104 if (tod.tod_year < 69) { 1105 if (mach_range_warn && tod.tod_year > 38) { 1106 cmn_err(CE_WARN, "hardware real-time clock is out " 1107 "of range -- time needs to be reset"); 1108 mach_range_warn = 0; 1109 } 1110 tod.tod_year += 100; 1111 } 1112 1113 /* tod_to_utc uses 1900 as base for the year */ 1114 ts.tv_sec = tod_to_utc(tod) + gmt_lag; 1115 ts.tv_nsec = 0; 1116 1117 return (ts); 1118 } 1119 1120 static void 1121 mach_tod_set(timestruc_t ts) 1122 { 1123 todinfo_t tod = utc_to_tod(ts.tv_sec - gmt_lag); 1124 1125 ASSERT(MUTEX_HELD(&tod_lock)); 1126 1127 if (tod.tod_year >= 100) 1128 tod.tod_year -= 100; 1129 1130 (*psm_todsetf)(&tod); 1131 } 1132 1133 static void 1134 mach_notify_error(int level, char *errmsg) 1135 { 1136 /* 1137 * SL_FATAL is pass in once panicstr is set, deliver it 1138 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1139 * codes for the psmi handler 1140 */ 1141 if (level & SL_FATAL) 1142 (*notify_error)(CE_PANIC, errmsg); 1143 else if (level & SL_WARN) 1144 (*notify_error)(CE_WARN, errmsg); 1145 else if (level & SL_NOTE) 1146 (*notify_error)(CE_NOTE, errmsg); 1147 else if (level & SL_CONSOLE) 1148 (*notify_error)(CE_CONT, errmsg); 1149 } 1150 1151 /* 1152 * It provides the default basic intr_ops interface for the new DDI 1153 * interrupt framework if the PSM doesn't have one. 1154 * 1155 * Input: 1156 * dip - pointer to the dev_info structure of the requested device 1157 * hdlp - pointer to the internal interrupt handle structure for the 1158 * requested interrupt 1159 * intr_op - opcode for this call 1160 * result - pointer to the integer that will hold the result to be 1161 * passed back if return value is PSM_SUCCESS 1162 * 1163 * Output: 1164 * return value is either PSM_SUCCESS or PSM_FAILURE 1165 */ 1166 static int 1167 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1168 psm_intr_op_t intr_op, int *result) 1169 { 1170 struct intrspec *ispec; 1171 1172 switch (intr_op) { 1173 case PSM_INTR_OP_CHECK_MSI: 1174 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1175 DDI_INTR_TYPE_MSIX); 1176 break; 1177 case PSM_INTR_OP_ALLOC_VECTORS: 1178 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1179 *result = 1; 1180 else 1181 *result = 0; 1182 break; 1183 case PSM_INTR_OP_FREE_VECTORS: 1184 break; 1185 case PSM_INTR_OP_NAVAIL_VECTORS: 1186 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1187 *result = 1; 1188 else 1189 *result = 0; 1190 break; 1191 case PSM_INTR_OP_XLATE_VECTOR: 1192 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1193 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1194 break; 1195 case PSM_INTR_OP_GET_CAP: 1196 *result = 0; 1197 break; 1198 case PSM_INTR_OP_GET_PENDING: 1199 case PSM_INTR_OP_CLEAR_MASK: 1200 case PSM_INTR_OP_SET_MASK: 1201 case PSM_INTR_OP_GET_SHARED: 1202 case PSM_INTR_OP_SET_PRI: 1203 case PSM_INTR_OP_SET_CAP: 1204 case PSM_INTR_OP_SET_CPU: 1205 case PSM_INTR_OP_GET_INTR: 1206 default: 1207 return (PSM_FAILURE); 1208 } 1209 return (PSM_SUCCESS); 1210 } 1211