1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #define PSMI_1_5 29 #include <sys/smp_impldefs.h> 30 #include <sys/psm.h> 31 #include <sys/psm_modctl.h> 32 #include <sys/pit.h> 33 #include <sys/cmn_err.h> 34 #include <sys/strlog.h> 35 #include <sys/clock.h> 36 #include <sys/debug.h> 37 #include <sys/rtc.h> 38 #include <sys/x86_archext.h> 39 #include <sys/cpupart.h> 40 #include <sys/cpuvar.h> 41 #include <sys/chip.h> 42 #include <sys/disp.h> 43 #include <sys/cpu.h> 44 #include <sys/archsystm.h> 45 #include <sys/mach_intr.h> 46 47 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 48 49 /* 50 * Local function prototypes 51 */ 52 static int mp_disable_intr(processorid_t cpun); 53 static void mp_enable_intr(processorid_t cpun); 54 static void mach_init(); 55 static void mach_picinit(); 56 static uint64_t mach_calchz(uint32_t pit_counter, uint64_t *processor_clks); 57 static int machhztomhz(uint64_t cpu_freq_hz); 58 static uint64_t mach_getcpufreq(void); 59 static void mach_fixcpufreq(void); 60 static int mach_clkinit(int, int *); 61 static void mach_smpinit(void); 62 static void mach_set_softintr(int ipl, struct av_softinfo *); 63 static void mach_cpu_start(int cpun); 64 static int mach_softlvl_to_vect(int ipl); 65 static void mach_get_platform(int owner); 66 static void mach_construct_info(); 67 static int mach_translate_irq(dev_info_t *dip, int irqno); 68 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 69 psm_intr_op_t, int *); 70 static timestruc_t mach_tod_get(void); 71 static void mach_tod_set(timestruc_t ts); 72 static void mach_notify_error(int level, char *errmsg); 73 static hrtime_t dummy_hrtime(void); 74 static void dummy_scalehrtime(hrtime_t *); 75 static void cpu_halt(void); 76 static void cpu_wakeup(cpu_t *, int); 77 /* 78 * External reference functions 79 */ 80 extern void return_instr(); 81 extern timestruc_t (*todgetf)(void); 82 extern void (*todsetf)(timestruc_t); 83 extern long gmt_lag; 84 extern uint64_t freq_tsc(uint32_t *); 85 #if defined(__i386) 86 extern uint64_t freq_notsc(uint32_t *); 87 #endif 88 extern void pc_gethrestime(timestruc_t *); 89 90 /* 91 * PSM functions initialization 92 */ 93 void (*psm_shutdownf)(int, int) = return_instr; 94 void (*psm_preshutdownf)(int, int) = return_instr; 95 void (*psm_notifyf)(int) = return_instr; 96 void (*psm_set_idle_cpuf)(int) = return_instr; 97 void (*psm_unset_idle_cpuf)(int) = return_instr; 98 void (*psminitf)() = mach_init; 99 void (*picinitf)() = return_instr; 100 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 101 void (*cpu_startf)() = return_instr; 102 int (*ap_mlsetup)() = (int (*)(void))return_instr; 103 void (*send_dirintf)() = return_instr; 104 void (*setspl)(int) = return_instr; 105 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 106 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 107 void (*setsoftint)(int, struct av_softinfo *)= 108 (void (*)(int, struct av_softinfo *))return_instr; 109 int (*slvltovect)(int) = (int (*)(int))return_instr; 110 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 111 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 112 int (*psm_disable_intr)(int) = mp_disable_intr; 113 void (*psm_enable_intr)(int) = mp_enable_intr; 114 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 115 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 116 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 117 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 118 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 119 int (*psm_todgetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 120 int (*psm_todsetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 121 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 122 int (*psm_get_clockirq)(int) = NULL; 123 int (*psm_get_ipivect)(int, int) = NULL; 124 125 int (*psm_clkinit)(int) = NULL; 126 void (*psm_timer_reprogram)(hrtime_t) = NULL; 127 void (*psm_timer_enable)(void) = NULL; 128 void (*psm_timer_disable)(void) = NULL; 129 void (*psm_post_cyclic_setup)(void *arg) = NULL; 130 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 131 int *) = mach_intr_ops; 132 133 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 134 void (*hrtime_tick)(void) = return_instr; 135 136 int tsc_gethrtime_enable = 1; 137 int tsc_gethrtime_initted = 0; 138 139 /* 140 * Local Static Data 141 */ 142 static struct psm_ops mach_ops; 143 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 144 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 145 146 /* 147 * If non-zero, idle cpus will "halted" when there's 148 * no work to do. 149 */ 150 int halt_idle_cpus = 1; 151 152 #if defined(__amd64) 153 /* 154 * If non-zero, will use cr8 for interrupt priority masking 155 * We declare this here since install_spl is called from here 156 * (where this is checked). 157 */ 158 int intpri_use_cr8 = 0; 159 #endif /* __amd64 */ 160 161 #ifdef _SIMULATOR_SUPPORT 162 163 int simulator_run = 0; /* patch to non-zero if running under simics */ 164 165 #endif /* _SIMULATOR_SUPPORT */ 166 167 /* ARGSUSED */ 168 void 169 chip_plat_define_chip(cpu_t *cp, chip_def_t *cd) 170 { 171 if ((x86_feature & (X86_HTT|X86_CMP)) == X86_HTT) { 172 /* 173 * Single-core Pentiums with Hyper-Threading enabled. 174 */ 175 cd->chipd_type = CHIP_SMT; 176 } else if ((x86_feature & (X86_HTT|X86_CMP)) == X86_CMP) { 177 /* 178 * Multi-core Opterons or Multi-core Pentiums with 179 * Hyper-Threading disabled. 180 */ 181 cd->chipd_type = CHIP_CMP_SPLIT_CACHE; 182 } else if ((x86_feature & (X86_HTT|X86_CMP)) == (X86_HTT|X86_CMP)) { 183 /* 184 * Multi-core Pentiums with Hyper-Threading enabled. 185 */ 186 cd->chipd_type = CHIP_CMT; 187 } else { 188 /* 189 * Single-core/single-threaded chips. 190 */ 191 cd->chipd_type = CHIP_DEFAULT; 192 } 193 194 cd->chipd_rechoose_adj = 0; 195 } 196 197 /* 198 * Routine to ensure initial callers to hrtime gets 0 as return 199 */ 200 static hrtime_t 201 dummy_hrtime(void) 202 { 203 return (0); 204 } 205 206 /* ARGSUSED */ 207 static void 208 dummy_scalehrtime(hrtime_t *ticks) 209 {} 210 211 /* 212 * Halt the present CPU until awoken via an interrupt 213 */ 214 static void 215 cpu_halt(void) 216 { 217 cpu_t *cpup = CPU; 218 processorid_t cpun = cpup->cpu_id; 219 cpupart_t *cp = cpup->cpu_part; 220 int hset_update = 1; 221 222 /* 223 * If this CPU is online, and there's multiple CPUs 224 * in the system, then we should notate our halting 225 * by adding ourselves to the partition's halted CPU 226 * bitmap. This allows other CPUs to find/awaken us when 227 * work becomes available. 228 */ 229 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 230 hset_update = 0; 231 232 /* 233 * Add ourselves to the partition's halted CPUs bitmask 234 * and set our HALTED flag, if necessary. 235 * 236 * When a thread becomes runnable, it is placed on the queue 237 * and then the halted cpuset is checked to determine who 238 * (if anyone) should be awoken. We therefore need to first 239 * add ourselves to the halted cpuset, and and then check if there 240 * is any work available. 241 * 242 * Note that memory barriers after updating the HALTED flag 243 * are not necessary since an atomic operation (updating the bitmap) 244 * immediately follows. On x86 the atomic operation acts as a 245 * memory barrier for the update of cpu_disp_flags. 246 */ 247 if (hset_update) { 248 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 249 CPUSET_ATOMIC_ADD(cp->cp_haltset, cpun); 250 } 251 252 /* 253 * Check to make sure there's really nothing to do. 254 * Work destined for this CPU may become available after 255 * this check. We'll be notified through the clearing of our 256 * bit in the halted CPU bitmask, and a poke. 257 */ 258 if (disp_anywork()) { 259 if (hset_update) { 260 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 261 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 262 } 263 return; 264 } 265 266 /* 267 * We're on our way to being halted. 268 * 269 * Disable interrupts now, so that we'll awaken immediately 270 * after halting if someone tries to poke us between now and 271 * the time we actually halt. 272 * 273 * We check for the presence of our bit after disabling interrupts. 274 * If it's cleared, we'll return. If the bit is cleared after 275 * we check then the poke will pop us out of the halted state. 276 * 277 * This means that the ordering of the poke and the clearing 278 * of the bit by cpu_wakeup is important. 279 * cpu_wakeup() must clear, then poke. 280 * cpu_halt() must disable interrupts, then check for the bit. 281 */ 282 cli(); 283 284 if (hset_update && !CPU_IN_SET(cp->cp_haltset, cpun)) { 285 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 286 sti(); 287 return; 288 } 289 290 /* 291 * The check for anything locally runnable is here for performance 292 * and isn't needed for correctness. disp_nrunnable ought to be 293 * in our cache still, so it's inexpensive to check, and if there 294 * is anything runnable we won't have to wait for the poke. 295 */ 296 if (cpup->cpu_disp->disp_nrunnable != 0) { 297 if (hset_update) { 298 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 299 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 300 } 301 sti(); 302 return; 303 } 304 305 /* 306 * Call the halt sequence: 307 * sti 308 * hlt 309 */ 310 i86_halt(); 311 312 /* 313 * We're no longer halted 314 */ 315 if (hset_update) { 316 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 317 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 318 } 319 } 320 321 322 /* 323 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 324 * Otherwise, see if other CPUs in the cpu partition are halted and need to 325 * be woken up so that they can steal the thread we placed on this CPU. 326 * This function is only used on MP systems. 327 */ 328 static void 329 cpu_wakeup(cpu_t *cpu, int bound) 330 { 331 uint_t cpu_found; 332 int result; 333 cpupart_t *cp; 334 335 cp = cpu->cpu_part; 336 if (CPU_IN_SET(cp->cp_haltset, cpu->cpu_id)) { 337 /* 338 * Clear the halted bit for that CPU since it will be 339 * poked in a moment. 340 */ 341 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpu->cpu_id); 342 /* 343 * We may find the current CPU present in the halted cpuset 344 * if we're in the context of an interrupt that occurred 345 * before we had a chance to clear our bit in cpu_halt(). 346 * Poking ourself is obviously unnecessary, since if 347 * we're here, we're not halted. 348 */ 349 if (cpu != CPU) 350 poke_cpu(cpu->cpu_id); 351 return; 352 } else { 353 /* 354 * This cpu isn't halted, but it's idle or undergoing a 355 * context switch. No need to awaken anyone else. 356 */ 357 if (cpu->cpu_thread == cpu->cpu_idle_thread || 358 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 359 return; 360 } 361 362 /* 363 * No need to wake up other CPUs if the thread we just enqueued 364 * is bound. 365 */ 366 if (bound) 367 return; 368 369 370 /* 371 * See if there's any other halted CPUs. If there are, then 372 * select one, and awaken it. 373 * It's possible that after we find a CPU, somebody else 374 * will awaken it before we get the chance. 375 * In that case, look again. 376 */ 377 do { 378 CPUSET_FIND(cp->cp_haltset, cpu_found); 379 if (cpu_found == CPUSET_NOTINSET) 380 return; 381 382 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 383 CPUSET_ATOMIC_XDEL(cp->cp_haltset, cpu_found, result); 384 } while (result < 0); 385 386 if (cpu_found != CPU->cpu_id) 387 poke_cpu(cpu_found); 388 } 389 390 static int 391 mp_disable_intr(int cpun) 392 { 393 /* 394 * switch to the offline cpu 395 */ 396 affinity_set(cpun); 397 /* 398 * raise ipl to just below cross call 399 */ 400 splx(XC_MED_PIL-1); 401 /* 402 * set base spl to prevent the next swtch to idle from 403 * lowering back to ipl 0 404 */ 405 CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1)); 406 set_base_spl(); 407 affinity_clear(); 408 return (DDI_SUCCESS); 409 } 410 411 static void 412 mp_enable_intr(int cpun) 413 { 414 /* 415 * switch to the online cpu 416 */ 417 affinity_set(cpun); 418 /* 419 * clear the interrupt active mask 420 */ 421 CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1)); 422 set_base_spl(); 423 (void) spl0(); 424 affinity_clear(); 425 } 426 427 static void 428 mach_get_platform(int owner) 429 { 430 void **srv_opsp; 431 void **clt_opsp; 432 int i; 433 int total_ops; 434 435 /* fix up psm ops */ 436 srv_opsp = (void **)mach_set[0]; 437 clt_opsp = (void **)mach_set[owner]; 438 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 439 total_ops = sizeof (struct psm_ops_ver01) / 440 sizeof (void (*)(void)); 441 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 442 /* no psm_notify_func */ 443 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 444 sizeof (void (*)(void)); 445 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 446 /* no psm_timer funcs */ 447 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 448 sizeof (void (*)(void)); 449 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 450 /* no psm_preshutdown function */ 451 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 452 sizeof (void (*)(void)); 453 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 454 /* no psm_preshutdown function */ 455 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 456 sizeof (void (*)(void)); 457 else 458 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 459 460 /* 461 * Save the version of the PSM module, in case we need to 462 * bahave differently based on version. 463 */ 464 mach_ver[0] = mach_ver[owner]; 465 466 for (i = 0; i < total_ops; i++) 467 if (clt_opsp[i] != NULL) 468 srv_opsp[i] = clt_opsp[i]; 469 } 470 471 static void 472 mach_construct_info() 473 { 474 register struct psm_sw *swp; 475 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 476 int conflict_owner = 0; 477 478 if (psmsw->psw_forw == psmsw) 479 panic("No valid PSM modules found"); 480 mutex_enter(&psmsw_lock); 481 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 482 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 483 continue; 484 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 485 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 486 mach_cnt[swp->psw_infop->p_owner]++; 487 } 488 mutex_exit(&psmsw_lock); 489 490 mach_get_platform(PSM_OWN_SYS_DEFAULT); 491 492 /* check to see are there any conflicts */ 493 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 494 conflict_owner = PSM_OWN_EXCLUSIVE; 495 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 496 conflict_owner = PSM_OWN_OVERRIDE; 497 if (conflict_owner) { 498 /* remove all psm modules except uppc */ 499 cmn_err(CE_WARN, 500 "Conflicts detected on the following PSM modules:"); 501 mutex_enter(&psmsw_lock); 502 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 503 if (swp->psw_infop->p_owner == conflict_owner) 504 cmn_err(CE_WARN, "%s ", 505 swp->psw_infop->p_mach_idstring); 506 } 507 mutex_exit(&psmsw_lock); 508 cmn_err(CE_WARN, 509 "Setting the system back to SINGLE processor mode!"); 510 cmn_err(CE_WARN, 511 "Please edit /etc/mach to remove the invalid PSM module."); 512 return; 513 } 514 515 if (mach_set[PSM_OWN_EXCLUSIVE]) 516 mach_get_platform(PSM_OWN_EXCLUSIVE); 517 518 if (mach_set[PSM_OWN_OVERRIDE]) 519 mach_get_platform(PSM_OWN_OVERRIDE); 520 } 521 522 static void 523 mach_init() 524 { 525 register struct psm_ops *pops; 526 527 mach_construct_info(); 528 529 pops = mach_set[0]; 530 531 /* register the interrupt and clock initialization rotuines */ 532 picinitf = mach_picinit; 533 clkinitf = mach_clkinit; 534 psm_get_clockirq = pops->psm_get_clockirq; 535 536 /* register the interrupt setup code */ 537 slvltovect = mach_softlvl_to_vect; 538 addspl = pops->psm_addspl; 539 delspl = pops->psm_delspl; 540 541 if (pops->psm_translate_irq) 542 psm_translate_irq = pops->psm_translate_irq; 543 if (pops->psm_intr_ops) 544 psm_intr_ops = pops->psm_intr_ops; 545 if (pops->psm_tod_get) { 546 todgetf = mach_tod_get; 547 psm_todgetf = pops->psm_tod_get; 548 } 549 if (pops->psm_tod_set) { 550 todsetf = mach_tod_set; 551 psm_todsetf = pops->psm_tod_set; 552 } 553 if (pops->psm_notify_error) { 554 psm_notify_error = mach_notify_error; 555 notify_error = pops->psm_notify_error; 556 } 557 558 (*pops->psm_softinit)(); 559 560 /* 561 * Initialize the dispatcher's function hooks 562 * to enable CPU halting when idle 563 */ 564 #if defined(_SIMULATOR_SUPPORT) 565 if (halt_idle_cpus && !simulator_run) 566 idle_cpu = cpu_halt; 567 #else 568 if (halt_idle_cpus) 569 idle_cpu = cpu_halt; 570 #endif /* _SIMULATOR_SUPPORT */ 571 572 mach_smpinit(); 573 } 574 575 static void 576 mach_smpinit(void) 577 { 578 struct psm_ops *pops; 579 processorid_t cpu_id; 580 int cnt; 581 cpuset_t cpumask; 582 583 pops = mach_set[0]; 584 585 cpu_id = -1; 586 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 587 for (cnt = 0, CPUSET_ZERO(cpumask); cpu_id != -1; cnt++) { 588 CPUSET_ADD(cpumask, cpu_id); 589 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 590 } 591 592 mp_cpus = cpumask; 593 594 /* MP related routines */ 595 cpu_startf = mach_cpu_start; 596 ap_mlsetup = pops->psm_post_cpu_start; 597 send_dirintf = pops->psm_send_ipi; 598 599 /* optional MP related routines */ 600 if (pops->psm_shutdown) 601 psm_shutdownf = pops->psm_shutdown; 602 if (pops->psm_preshutdown) 603 psm_preshutdownf = pops->psm_preshutdown; 604 if (pops->psm_notify_func) 605 psm_notifyf = pops->psm_notify_func; 606 if (pops->psm_set_idlecpu) 607 psm_set_idle_cpuf = pops->psm_set_idlecpu; 608 if (pops->psm_unset_idlecpu) 609 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 610 611 psm_clkinit = pops->psm_clkinit; 612 613 if (pops->psm_timer_reprogram) 614 psm_timer_reprogram = pops->psm_timer_reprogram; 615 616 if (pops->psm_timer_enable) 617 psm_timer_enable = pops->psm_timer_enable; 618 619 if (pops->psm_timer_disable) 620 psm_timer_disable = pops->psm_timer_disable; 621 622 if (pops->psm_post_cyclic_setup) 623 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 624 625 /* check for multiple cpu's */ 626 if (cnt < 2) 627 return; 628 629 /* check for MP platforms */ 630 if (pops->psm_cpu_start == NULL) 631 return; 632 633 /* 634 * Set the dispatcher hook to enable cpu "wake up" 635 * when a thread becomes runnable. 636 */ 637 #if defined(_SIMULATOR_SUPPORT) 638 if (halt_idle_cpus && !simulator_run) { 639 disp_enq_thread = cpu_wakeup; 640 } 641 #else 642 if (halt_idle_cpus) { 643 disp_enq_thread = cpu_wakeup; 644 } 645 #endif /* _SIMULATOR_SUPPORT */ 646 647 if (pops->psm_disable_intr) 648 psm_disable_intr = pops->psm_disable_intr; 649 if (pops->psm_enable_intr) 650 psm_enable_intr = pops->psm_enable_intr; 651 652 psm_get_ipivect = pops->psm_get_ipivect; 653 654 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr", 655 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 656 (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL); 657 (void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr", 658 (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO), 659 (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL); 660 661 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 662 } 663 664 static void 665 mach_picinit() 666 { 667 struct psm_ops *pops; 668 extern void install_spl(void); /* XXX: belongs in a header file */ 669 #if defined(__amd64) && defined(DEBUG) 670 extern void *spl_patch, *slow_spl, *setsplhi_patch, *slow_setsplhi; 671 #endif 672 673 pops = mach_set[0]; 674 675 /* register the interrupt handlers */ 676 setlvl = pops->psm_intr_enter; 677 setlvlx = pops->psm_intr_exit; 678 679 /* initialize the interrupt hardware */ 680 (*pops->psm_picinit)(); 681 682 /* set interrupt mask for current ipl */ 683 setspl = pops->psm_setspl; 684 setspl(CPU->cpu_pri); 685 686 /* Install proper spl routine now that we can Program the PIC */ 687 #if defined(__amd64) 688 /* 689 * It would be better if we could check this at compile time 690 */ 691 ASSERT(((uintptr_t)&slow_setsplhi - (uintptr_t)&setsplhi_patch < 128) && 692 ((uintptr_t)&slow_spl - (uintptr_t)&spl_patch < 128)); 693 #endif 694 install_spl(); 695 } 696 697 uint_t cpu_freq; /* MHz */ 698 uint64_t cpu_freq_hz; /* measured (in hertz) */ 699 700 #define MEGA_HZ 1000000 701 702 static uint64_t 703 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 704 { 705 uint64_t cpu_hz; 706 707 if ((pit_counter == 0) || (*processor_clks == 0) || 708 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 709 return (0); 710 711 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 712 713 return (cpu_hz); 714 } 715 716 static uint64_t 717 mach_getcpufreq(void) 718 { 719 uint32_t pit_counter; 720 uint64_t processor_clks; 721 722 if (x86_feature & X86_TSC) { 723 /* 724 * We have a TSC. freq_tsc() knows how to measure the number 725 * of clock cycles sampled against the PIT. 726 */ 727 processor_clks = freq_tsc(&pit_counter); 728 return (mach_calchz(pit_counter, &processor_clks)); 729 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 730 #if defined(__amd64) 731 panic("mach_getcpufreq: no TSC!"); 732 #elif defined(__i386) 733 /* 734 * We are a Cyrix based on a 6x86 core or an Intel Pentium 735 * for which freq_notsc() knows how to measure the number of 736 * elapsed clock cycles sampled against the PIT 737 */ 738 processor_clks = freq_notsc(&pit_counter); 739 return (mach_calchz(pit_counter, &processor_clks)); 740 #endif /* __i386 */ 741 } 742 743 /* We do not know how to calculate cpu frequency for this cpu. */ 744 return (0); 745 } 746 747 /* 748 * If the clock speed of a cpu is found to be reported incorrectly, do not add 749 * to this array, instead improve the accuracy of the algorithm that determines 750 * the clock speed of the processor or extend the implementation to support the 751 * vendor as appropriate. This is here only to support adjusting the speed on 752 * older slower processors that mach_fixcpufreq() would not be able to account 753 * for otherwise. 754 */ 755 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 756 757 /* 758 * On fast processors the clock frequency that is measured may be off by 759 * a few MHz from the value printed on the part. This is a combination of 760 * the factors that for such fast parts being off by this much is within 761 * the tolerances for manufacture and because of the difficulties in the 762 * measurement that can lead to small error. This function uses some 763 * heuristics in order to tweak the value that was measured to match what 764 * is most likely printed on the part. 765 * 766 * Some examples: 767 * AMD Athlon 1000 mhz measured as 998 mhz 768 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 769 * Intel Pentium IV 1500 mhz measured as 1495mhz 770 * 771 * If in the future this function is no longer sufficient to correct 772 * for the error in the measurement, then the algorithm used to perform 773 * the measurement will have to be improved in order to increase accuracy 774 * rather than adding horrible and questionable kludges here. 775 * 776 * This is called after the cyclics subsystem because of the potential 777 * that the heuristics within may give a worse estimate of the clock 778 * frequency than the value that was measured. 779 */ 780 static void 781 mach_fixcpufreq(void) 782 { 783 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 784 785 freq = (uint32_t)cpu_freq; 786 787 /* 788 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 789 * measured speed taking into account that the 667 MHz parts were 790 * the first to round-up. 791 */ 792 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 793 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 794 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 795 796 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 797 mul = (freq + 25) / 50; 798 near50 = mul * 50; 799 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 800 801 /* Find the closer of the two */ 802 if (delta66 < delta50) { 803 fixed = near66; 804 delta = delta66; 805 } else { 806 fixed = near50; 807 delta = delta50; 808 } 809 810 if (fixed > INT_MAX) 811 return; 812 813 /* 814 * Some older parts have a core clock frequency that is not an 815 * integral multiple of 50 or 66 MHz. Check if one of the old 816 * clock frequencies is closer to the measured value than any 817 * of the integral multiples of 50 an 66, and if so set fixed 818 * and delta appropriately to represent the closest value. 819 */ 820 i = sizeof (x86_cpu_freq) / sizeof (int); 821 while (i > 0) { 822 i--; 823 824 if (x86_cpu_freq[i] <= freq) { 825 mul = freq - x86_cpu_freq[i]; 826 827 if (mul < delta) { 828 fixed = x86_cpu_freq[i]; 829 delta = mul; 830 } 831 832 break; 833 } 834 835 mul = x86_cpu_freq[i] - freq; 836 837 if (mul < delta) { 838 fixed = x86_cpu_freq[i]; 839 delta = mul; 840 } 841 } 842 843 /* 844 * Set a reasonable maximum for how much to correct the measured 845 * result by. This check is here to prevent the adjustment made 846 * by this function from being more harm than good. It is entirely 847 * possible that in the future parts will be made that are not 848 * integral multiples of 66 or 50 in clock frequency or that 849 * someone may overclock a part to some odd frequency. If the 850 * measured value is farther from the corrected value than 851 * allowed, then assume the corrected value is in error and use 852 * the measured value. 853 */ 854 if (6 < delta) 855 return; 856 857 cpu_freq = (int)fixed; 858 } 859 860 861 static int 862 machhztomhz(uint64_t cpu_freq_hz) 863 { 864 uint64_t cpu_mhz; 865 866 /* Round to nearest MHZ */ 867 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 868 869 if (cpu_mhz > INT_MAX) 870 return (0); 871 872 return ((int)cpu_mhz); 873 874 } 875 876 877 static int 878 mach_clkinit(int preferred_mode, int *set_mode) 879 { 880 register struct psm_ops *pops; 881 int resolution; 882 883 pops = mach_set[0]; 884 885 #ifdef _SIMULATOR_SUPPORT 886 if (!simulator_run) 887 cpu_freq_hz = mach_getcpufreq(); 888 else 889 cpu_freq_hz = 40000000; /* use 40 Mhz (hack for simulator) */ 890 #else 891 cpu_freq_hz = mach_getcpufreq(); 892 #endif /* _SIMULATOR_SUPPORT */ 893 894 cpu_freq = machhztomhz(cpu_freq_hz); 895 896 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 897 tsc_gethrtime_enable = 0; 898 899 if (tsc_gethrtime_enable) { 900 tsc_hrtimeinit(cpu_freq_hz); 901 gethrtimef = tsc_gethrtime; 902 gethrtimeunscaledf = tsc_gethrtimeunscaled; 903 scalehrtimef = tsc_scalehrtime; 904 hrtime_tick = tsc_tick; 905 tsc_gethrtime_initted = 1; 906 } else { 907 if (pops->psm_hrtimeinit) 908 (*pops->psm_hrtimeinit)(); 909 gethrtimef = pops->psm_gethrtime; 910 gethrtimeunscaledf = gethrtimef; 911 /* scalehrtimef will remain dummy */ 912 } 913 914 mach_fixcpufreq(); 915 916 if (mach_ver[0] >= PSM_INFO_VER01_3) { 917 if ((preferred_mode == TIMER_ONESHOT) && 918 (tsc_gethrtime_enable)) { 919 920 resolution = (*pops->psm_clkinit)(0); 921 if (resolution != 0) { 922 *set_mode = TIMER_ONESHOT; 923 return (resolution); 924 } 925 926 } 927 928 /* 929 * either periodic mode was requested or could not set to 930 * one-shot mode 931 */ 932 resolution = (*pops->psm_clkinit)(hz); 933 /* 934 * psm should be able to do periodic, so we do not check 935 * for return value of psm_clkinit here. 936 */ 937 *set_mode = TIMER_PERIODIC; 938 return (resolution); 939 } else { 940 /* 941 * PSMI interface prior to PSMI_3 does not define a return 942 * value for psm_clkinit, so the return value is ignored. 943 */ 944 (void) (*pops->psm_clkinit)(hz); 945 *set_mode = TIMER_PERIODIC; 946 return (nsec_per_tick); 947 } 948 } 949 950 /*ARGSUSED*/ 951 static void 952 mach_psm_set_softintr(int ipl, struct av_softinfo *pending) 953 { 954 register struct psm_ops *pops; 955 956 /* invoke hardware interrupt */ 957 pops = mach_set[0]; 958 (*pops->psm_set_softintr)(ipl); 959 } 960 961 static int 962 mach_softlvl_to_vect(register int ipl) 963 { 964 register int softvect; 965 register struct psm_ops *pops; 966 967 pops = mach_set[0]; 968 969 /* check for null handler for set soft interrupt call */ 970 if (pops->psm_set_softintr == NULL) { 971 setsoftint = av_set_softint_pending; 972 return (PSM_SV_SOFTWARE); 973 } 974 975 softvect = (*pops->psm_softlvl_to_irq)(ipl); 976 /* check for hardware scheme */ 977 if (softvect > PSM_SV_SOFTWARE) { 978 setsoftint = mach_psm_set_softintr; 979 return (softvect); 980 } 981 982 if (softvect == PSM_SV_SOFTWARE) 983 setsoftint = av_set_softint_pending; 984 else /* hardware and software mixed scheme */ 985 setsoftint = mach_set_softintr; 986 987 return (PSM_SV_SOFTWARE); 988 } 989 990 static void 991 mach_set_softintr(register int ipl, struct av_softinfo *pending) 992 { 993 register struct psm_ops *pops; 994 995 /* set software pending bits */ 996 av_set_softint_pending(ipl, pending); 997 998 /* check if dosoftint will be called at the end of intr */ 999 if (CPU_ON_INTR(CPU) || (curthread->t_intr)) 1000 return; 1001 1002 /* invoke hardware interrupt */ 1003 pops = mach_set[0]; 1004 (*pops->psm_set_softintr)(ipl); 1005 } 1006 1007 static void 1008 mach_cpu_start(register int cpun) 1009 { 1010 register struct psm_ops *pops; 1011 int i; 1012 1013 pops = mach_set[0]; 1014 1015 (*pops->psm_cpu_start)(cpun, rm_platter_va); 1016 1017 /* wait for the auxillary cpu to be ready */ 1018 for (i = 20000; i; i--) { 1019 if (cpu[cpun]->cpu_flags & CPU_READY) 1020 return; 1021 drv_usecwait(100); 1022 } 1023 } 1024 1025 /*ARGSUSED*/ 1026 static int 1027 mach_translate_irq(dev_info_t *dip, int irqno) 1028 { 1029 return (irqno); /* default to NO translation */ 1030 } 1031 1032 static timestruc_t 1033 mach_tod_get(void) 1034 { 1035 timestruc_t ts; 1036 todinfo_t tod; 1037 static int mach_range_warn = 1; /* warn only once */ 1038 1039 ASSERT(MUTEX_HELD(&tod_lock)); 1040 1041 /* The year returned from is the last 2 digit only */ 1042 if ((*psm_todgetf)(&tod)) { 1043 ts.tv_sec = 0; 1044 ts.tv_nsec = 0; 1045 tod_fault_reset(); 1046 return (ts); 1047 } 1048 1049 /* assume that we wrap the rtc year back to zero at 2000 */ 1050 if (tod.tod_year < 69) { 1051 if (mach_range_warn && tod.tod_year > 38) { 1052 cmn_err(CE_WARN, "hardware real-time clock is out " 1053 "of range -- time needs to be reset"); 1054 mach_range_warn = 0; 1055 } 1056 tod.tod_year += 100; 1057 } 1058 1059 /* tod_to_utc uses 1900 as base for the year */ 1060 ts.tv_sec = tod_to_utc(tod) + gmt_lag; 1061 ts.tv_nsec = 0; 1062 1063 return (ts); 1064 } 1065 1066 static void 1067 mach_tod_set(timestruc_t ts) 1068 { 1069 todinfo_t tod = utc_to_tod(ts.tv_sec - gmt_lag); 1070 1071 ASSERT(MUTEX_HELD(&tod_lock)); 1072 1073 if (tod.tod_year >= 100) 1074 tod.tod_year -= 100; 1075 1076 (*psm_todsetf)(&tod); 1077 } 1078 1079 static void 1080 mach_notify_error(int level, char *errmsg) 1081 { 1082 /* 1083 * SL_FATAL is pass in once panicstr is set, deliver it 1084 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1085 * codes for the psmi handler 1086 */ 1087 if (level & SL_FATAL) 1088 (*notify_error)(CE_PANIC, errmsg); 1089 else if (level & SL_WARN) 1090 (*notify_error)(CE_WARN, errmsg); 1091 else if (level & SL_NOTE) 1092 (*notify_error)(CE_NOTE, errmsg); 1093 else if (level & SL_CONSOLE) 1094 (*notify_error)(CE_CONT, errmsg); 1095 } 1096 1097 /* 1098 * It provides the default basic intr_ops interface for the new DDI 1099 * interrupt framework if the PSM doesn't have one. 1100 * 1101 * Input: 1102 * dip - pointer to the dev_info structure of the requested device 1103 * hdlp - pointer to the internal interrupt handle structure for the 1104 * requested interrupt 1105 * intr_op - opcode for this call 1106 * result - pointer to the integer that will hold the result to be 1107 * passed back if return value is PSM_SUCCESS 1108 * 1109 * Output: 1110 * return value is either PSM_SUCCESS or PSM_FAILURE 1111 */ 1112 static int 1113 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1114 psm_intr_op_t intr_op, int *result) 1115 { 1116 struct intrspec *ispec; 1117 1118 switch (intr_op) { 1119 case PSM_INTR_OP_CHECK_MSI: 1120 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1121 DDI_INTR_TYPE_MSIX); 1122 break; 1123 case PSM_INTR_OP_ALLOC_VECTORS: 1124 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1125 *result = 1; 1126 else 1127 *result = 0; 1128 break; 1129 case PSM_INTR_OP_FREE_VECTORS: 1130 break; 1131 case PSM_INTR_OP_NAVAIL_VECTORS: 1132 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1133 *result = 1; 1134 else 1135 *result = 0; 1136 break; 1137 case PSM_INTR_OP_XLATE_VECTOR: 1138 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1139 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1140 break; 1141 case PSM_INTR_OP_GET_CAP: 1142 *result = 0; 1143 break; 1144 case PSM_INTR_OP_GET_PENDING: 1145 case PSM_INTR_OP_CLEAR_MASK: 1146 case PSM_INTR_OP_SET_MASK: 1147 case PSM_INTR_OP_GET_SHARED: 1148 case PSM_INTR_OP_SET_PRI: 1149 case PSM_INTR_OP_SET_CAP: 1150 case PSM_INTR_OP_SET_CPU: 1151 case PSM_INTR_OP_GET_INTR: 1152 default: 1153 return (PSM_FAILURE); 1154 } 1155 return (PSM_SUCCESS); 1156 } 1157