1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #define PSMI_1_5 30 #include <sys/smp_impldefs.h> 31 #include <sys/psm.h> 32 #include <sys/psm_modctl.h> 33 #include <sys/pit.h> 34 #include <sys/cmn_err.h> 35 #include <sys/strlog.h> 36 #include <sys/clock.h> 37 #include <sys/debug.h> 38 #include <sys/rtc.h> 39 #include <sys/x86_archext.h> 40 #include <sys/cpupart.h> 41 #include <sys/cpuvar.h> 42 #include <sys/chip.h> 43 #include <sys/disp.h> 44 #include <sys/cpu.h> 45 #include <sys/archsystm.h> 46 47 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 48 49 /* 50 * Local function prototypes 51 */ 52 static int mp_disable_intr(processorid_t cpun); 53 static void mp_enable_intr(processorid_t cpun); 54 static void mach_init(); 55 static void mach_picinit(); 56 static uint64_t mach_calchz(uint32_t pit_counter, uint64_t *processor_clks); 57 static int machhztomhz(uint64_t cpu_freq_hz); 58 static uint64_t mach_getcpufreq(void); 59 static void mach_fixcpufreq(void); 60 static int mach_clkinit(int, int *); 61 static void mach_smpinit(void); 62 static void mach_set_softintr(int ipl); 63 static void mach_cpu_start(int cpun); 64 static int mach_softlvl_to_vect(int ipl); 65 static void mach_get_platform(int owner); 66 static void mach_construct_info(); 67 static int mach_translate_irq(dev_info_t *dip, int irqno); 68 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 69 psm_intr_op_t, int *); 70 static timestruc_t mach_tod_get(void); 71 static void mach_tod_set(timestruc_t ts); 72 static void mach_notify_error(int level, char *errmsg); 73 static hrtime_t dummy_hrtime(void); 74 static void dummy_scalehrtime(hrtime_t *); 75 static void cpu_halt(void); 76 static void cpu_wakeup(cpu_t *, int); 77 /* 78 * External reference functions 79 */ 80 extern void return_instr(); 81 extern timestruc_t (*todgetf)(void); 82 extern void (*todsetf)(timestruc_t); 83 extern long gmt_lag; 84 extern uint64_t freq_tsc(uint32_t *); 85 #if defined(__i386) 86 extern uint64_t freq_notsc(uint32_t *); 87 #endif 88 extern void pc_gethrestime(timestruc_t *); 89 90 /* 91 * PSM functions initialization 92 */ 93 void (*psm_shutdownf)(int, int) = return_instr; 94 void (*psm_preshutdownf)(int, int) = return_instr; 95 void (*psm_notifyf)(int) = return_instr; 96 void (*psm_set_idle_cpuf)(int) = return_instr; 97 void (*psm_unset_idle_cpuf)(int) = return_instr; 98 void (*psminitf)() = mach_init; 99 void (*picinitf)() = return_instr; 100 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 101 void (*cpu_startf)() = return_instr; 102 int (*ap_mlsetup)() = (int (*)(void))return_instr; 103 void (*send_dirintf)() = return_instr; 104 void (*setspl)(int) = return_instr; 105 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 106 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 107 void (*setsoftint)(int) = (void (*)(int))return_instr; 108 int (*slvltovect)(int) = (int (*)(int))return_instr; 109 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 110 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 111 int (*psm_disable_intr)(int) = mp_disable_intr; 112 void (*psm_enable_intr)(int) = mp_enable_intr; 113 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 114 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 115 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 116 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 117 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 118 int (*psm_todgetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 119 int (*psm_todsetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 120 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 121 int (*psm_get_clockirq)(int) = NULL; 122 int (*psm_get_ipivect)(int, int) = NULL; 123 124 int (*psm_clkinit)(int) = NULL; 125 void (*psm_timer_reprogram)(hrtime_t) = NULL; 126 void (*psm_timer_enable)(void) = NULL; 127 void (*psm_timer_disable)(void) = NULL; 128 void (*psm_post_cyclic_setup)(void *arg) = NULL; 129 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 130 int *) = mach_intr_ops; 131 132 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 133 void (*hrtime_tick)(void) = return_instr; 134 135 int tsc_gethrtime_enable = 1; 136 int tsc_gethrtime_initted = 0; 137 138 /* 139 * Local Static Data 140 */ 141 static struct psm_ops mach_ops; 142 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 143 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 144 145 /* 146 * If non-zero, idle cpus will "halted" when there's 147 * no work to do. 148 */ 149 int halt_idle_cpus = 1; 150 151 #if defined(__amd64) 152 /* 153 * If non-zero, will use cr8 for interrupt priority masking 154 * We declare this here since install_spl is called from here 155 * (where this is checked). 156 */ 157 int intpri_use_cr8 = 0; 158 #endif /* __amd64 */ 159 160 #ifdef _SIMULATOR_SUPPORT 161 162 int simulator_run = 0; /* patch to non-zero if running under simics */ 163 164 #endif /* _SIMULATOR_SUPPORT */ 165 166 /* ARGSUSED */ 167 void 168 chip_plat_define_chip(cpu_t *cp, chip_def_t *cd) 169 { 170 if (x86_feature & (X86_HTT|X86_CMP)) 171 /* 172 * Hyperthreading is SMT 173 */ 174 cd->chipd_type = CHIP_SMT; 175 else 176 cd->chipd_type = CHIP_DEFAULT; 177 178 cd->chipd_rechoose_adj = 0; 179 } 180 181 /* 182 * Routine to ensure initial callers to hrtime gets 0 as return 183 */ 184 static hrtime_t 185 dummy_hrtime(void) 186 { 187 return (0); 188 } 189 190 /* ARGSUSED */ 191 static void 192 dummy_scalehrtime(hrtime_t *ticks) 193 {} 194 195 /* 196 * Halt the present CPU until awoken via an interrupt 197 */ 198 static void 199 cpu_halt(void) 200 { 201 cpu_t *cpup = CPU; 202 processorid_t cpun = cpup->cpu_id; 203 cpupart_t *cp = cpup->cpu_part; 204 int hset_update = 1; 205 206 /* 207 * If this CPU is online, and there's multiple CPUs 208 * in the system, then we should notate our halting 209 * by adding ourselves to the partition's halted CPU 210 * bitmap. This allows other CPUs to find/awaken us when 211 * work becomes available. 212 */ 213 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 214 hset_update = 0; 215 216 /* 217 * Add ourselves to the partition's halted CPUs bitmask 218 * and set our HALTED flag, if necessary. 219 * 220 * When a thread becomes runnable, it is placed on the queue 221 * and then the halted cpuset is checked to determine who 222 * (if anyone) should be awoken. We therefore need to first 223 * add ourselves to the halted cpuset, and and then check if there 224 * is any work available. 225 * 226 * Note that memory barriers after updating the HALTED flag 227 * are not necessary since an atomic operation (updating the bitmap) 228 * immediately follows. On x86 the atomic operation acts as a 229 * memory barrier for the update of cpu_disp_flags. 230 */ 231 if (hset_update) { 232 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 233 CPUSET_ATOMIC_ADD(cp->cp_haltset, cpun); 234 } 235 236 /* 237 * Check to make sure there's really nothing to do. 238 * Work destined for this CPU may become available after 239 * this check. We'll be notified through the clearing of our 240 * bit in the halted CPU bitmask, and a poke. 241 */ 242 if (disp_anywork()) { 243 if (hset_update) { 244 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 245 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 246 } 247 return; 248 } 249 250 /* 251 * We're on our way to being halted. 252 * 253 * Disable interrupts now, so that we'll awaken immediately 254 * after halting if someone tries to poke us between now and 255 * the time we actually halt. 256 * 257 * We check for the presence of our bit after disabling interrupts. 258 * If it's cleared, we'll return. If the bit is cleared after 259 * we check then the poke will pop us out of the halted state. 260 * 261 * This means that the ordering of the poke and the clearing 262 * of the bit by cpu_wakeup is important. 263 * cpu_wakeup() must clear, then poke. 264 * cpu_halt() must disable interrupts, then check for the bit. 265 */ 266 cli(); 267 268 if (hset_update && !CPU_IN_SET(cp->cp_haltset, cpun)) { 269 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 270 sti(); 271 return; 272 } 273 274 /* 275 * The check for anything locally runnable is here for performance 276 * and isn't needed for correctness. disp_nrunnable ought to be 277 * in our cache still, so it's inexpensive to check, and if there 278 * is anything runnable we won't have to wait for the poke. 279 */ 280 if (cpup->cpu_disp->disp_nrunnable != 0) { 281 if (hset_update) { 282 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 283 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 284 } 285 sti(); 286 return; 287 } 288 289 /* 290 * Call the halt sequence: 291 * sti 292 * hlt 293 */ 294 i86_halt(); 295 296 /* 297 * We're no longer halted 298 */ 299 if (hset_update) { 300 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 301 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 302 } 303 } 304 305 306 /* 307 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 308 * Otherwise, see if other CPUs in the cpu partition are halted and need to 309 * be woken up so that they can steal the thread we placed on this CPU. 310 * This function is only used on MP systems. 311 */ 312 static void 313 cpu_wakeup(cpu_t *cpu, int bound) 314 { 315 uint_t cpu_found; 316 int result; 317 cpupart_t *cp; 318 319 cp = cpu->cpu_part; 320 if (CPU_IN_SET(cp->cp_haltset, cpu->cpu_id)) { 321 /* 322 * Clear the halted bit for that CPU since it will be 323 * poked in a moment. 324 */ 325 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpu->cpu_id); 326 /* 327 * We may find the current CPU present in the halted cpuset 328 * if we're in the context of an interrupt that occurred 329 * before we had a chance to clear our bit in cpu_halt(). 330 * Poking ourself is obviously unnecessary, since if 331 * we're here, we're not halted. 332 */ 333 if (cpu != CPU) 334 poke_cpu(cpu->cpu_id); 335 return; 336 } else { 337 /* 338 * This cpu isn't halted, but it's idle or undergoing a 339 * context switch. No need to awaken anyone else. 340 */ 341 if (cpu->cpu_thread == cpu->cpu_idle_thread || 342 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 343 return; 344 } 345 346 /* 347 * No need to wake up other CPUs if the thread we just enqueued 348 * is bound. 349 */ 350 if (bound) 351 return; 352 353 354 /* 355 * See if there's any other halted CPUs. If there are, then 356 * select one, and awaken it. 357 * It's possible that after we find a CPU, somebody else 358 * will awaken it before we get the chance. 359 * In that case, look again. 360 */ 361 do { 362 CPUSET_FIND(cp->cp_haltset, cpu_found); 363 if (cpu_found == CPUSET_NOTINSET) 364 return; 365 366 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 367 CPUSET_ATOMIC_XDEL(cp->cp_haltset, cpu_found, result); 368 } while (result < 0); 369 370 if (cpu_found != CPU->cpu_id) 371 poke_cpu(cpu_found); 372 } 373 374 static int 375 mp_disable_intr(int cpun) 376 { 377 /* 378 * switch to the offline cpu 379 */ 380 affinity_set(cpun); 381 /* 382 * raise ipl to just below cross call 383 */ 384 splx(XC_MED_PIL-1); 385 /* 386 * set base spl to prevent the next swtch to idle from 387 * lowering back to ipl 0 388 */ 389 CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1)); 390 set_base_spl(); 391 affinity_clear(); 392 return (DDI_SUCCESS); 393 } 394 395 static void 396 mp_enable_intr(int cpun) 397 { 398 /* 399 * switch to the online cpu 400 */ 401 affinity_set(cpun); 402 /* 403 * clear the interrupt active mask 404 */ 405 CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1)); 406 set_base_spl(); 407 (void) spl0(); 408 affinity_clear(); 409 } 410 411 static void 412 mach_get_platform(int owner) 413 { 414 void **srv_opsp; 415 void **clt_opsp; 416 int i; 417 int total_ops; 418 419 /* fix up psm ops */ 420 srv_opsp = (void **)mach_set[0]; 421 clt_opsp = (void **)mach_set[owner]; 422 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 423 total_ops = sizeof (struct psm_ops_ver01) / 424 sizeof (void (*)(void)); 425 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 426 /* no psm_notify_func */ 427 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 428 sizeof (void (*)(void)); 429 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 430 /* no psm_timer funcs */ 431 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 432 sizeof (void (*)(void)); 433 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 434 /* no psm_preshutdown function */ 435 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 436 sizeof (void (*)(void)); 437 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 438 /* no psm_preshutdown function */ 439 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 440 sizeof (void (*)(void)); 441 else 442 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 443 444 /* 445 * Save the version of the PSM module, in case we need to 446 * bahave differently based on version. 447 */ 448 mach_ver[0] = mach_ver[owner]; 449 450 for (i = 0; i < total_ops; i++) 451 if (clt_opsp[i] != NULL) 452 srv_opsp[i] = clt_opsp[i]; 453 } 454 455 static void 456 mach_construct_info() 457 { 458 register struct psm_sw *swp; 459 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 460 int conflict_owner = 0; 461 462 if (psmsw->psw_forw == psmsw) 463 panic("No valid PSM modules found"); 464 mutex_enter(&psmsw_lock); 465 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 466 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 467 continue; 468 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 469 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 470 mach_cnt[swp->psw_infop->p_owner]++; 471 } 472 mutex_exit(&psmsw_lock); 473 474 mach_get_platform(PSM_OWN_SYS_DEFAULT); 475 476 /* check to see are there any conflicts */ 477 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 478 conflict_owner = PSM_OWN_EXCLUSIVE; 479 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 480 conflict_owner = PSM_OWN_OVERRIDE; 481 if (conflict_owner) { 482 /* remove all psm modules except uppc */ 483 cmn_err(CE_WARN, 484 "Conflicts detected on the following PSM modules:"); 485 mutex_enter(&psmsw_lock); 486 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 487 if (swp->psw_infop->p_owner == conflict_owner) 488 cmn_err(CE_WARN, "%s ", 489 swp->psw_infop->p_mach_idstring); 490 } 491 mutex_exit(&psmsw_lock); 492 cmn_err(CE_WARN, 493 "Setting the system back to SINGLE processor mode!"); 494 cmn_err(CE_WARN, 495 "Please edit /etc/mach to remove the invalid PSM module."); 496 return; 497 } 498 499 if (mach_set[PSM_OWN_EXCLUSIVE]) 500 mach_get_platform(PSM_OWN_EXCLUSIVE); 501 502 if (mach_set[PSM_OWN_OVERRIDE]) 503 mach_get_platform(PSM_OWN_OVERRIDE); 504 } 505 506 static void 507 mach_init() 508 { 509 register struct psm_ops *pops; 510 511 mach_construct_info(); 512 513 pops = mach_set[0]; 514 515 /* register the interrupt and clock initialization rotuines */ 516 picinitf = mach_picinit; 517 clkinitf = mach_clkinit; 518 psm_get_clockirq = pops->psm_get_clockirq; 519 520 /* register the interrupt setup code */ 521 slvltovect = mach_softlvl_to_vect; 522 addspl = pops->psm_addspl; 523 delspl = pops->psm_delspl; 524 525 if (pops->psm_translate_irq) 526 psm_translate_irq = pops->psm_translate_irq; 527 if (pops->psm_intr_ops) 528 psm_intr_ops = pops->psm_intr_ops; 529 if (pops->psm_tod_get) { 530 todgetf = mach_tod_get; 531 psm_todgetf = pops->psm_tod_get; 532 } 533 if (pops->psm_tod_set) { 534 todsetf = mach_tod_set; 535 psm_todsetf = pops->psm_tod_set; 536 } 537 if (pops->psm_notify_error) { 538 psm_notify_error = mach_notify_error; 539 notify_error = pops->psm_notify_error; 540 } 541 542 (*pops->psm_softinit)(); 543 544 /* 545 * Initialize the dispatcher's function hooks 546 * to enable CPU halting when idle 547 */ 548 #if defined(_SIMULATOR_SUPPORT) 549 if (halt_idle_cpus && !simulator_run) 550 idle_cpu = cpu_halt; 551 #else 552 if (halt_idle_cpus) 553 idle_cpu = cpu_halt; 554 #endif /* _SIMULATOR_SUPPORT */ 555 556 mach_smpinit(); 557 } 558 559 static void 560 mach_smpinit(void) 561 { 562 register struct psm_ops *pops; 563 register processorid_t cpu_id; 564 int cnt; 565 int cpumask; 566 567 pops = mach_set[0]; 568 569 cpu_id = -1; 570 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 571 for (cnt = 0, cpumask = 0; cpu_id != -1; cnt++) { 572 cpumask |= 1 << cpu_id; 573 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 574 } 575 576 mp_cpus = cpumask; 577 578 /* MP related routines */ 579 cpu_startf = mach_cpu_start; 580 ap_mlsetup = pops->psm_post_cpu_start; 581 send_dirintf = pops->psm_send_ipi; 582 583 /* optional MP related routines */ 584 if (pops->psm_shutdown) 585 psm_shutdownf = pops->psm_shutdown; 586 if (pops->psm_preshutdown) 587 psm_preshutdownf = pops->psm_preshutdown; 588 if (pops->psm_notify_func) 589 psm_notifyf = pops->psm_notify_func; 590 if (pops->psm_set_idlecpu) 591 psm_set_idle_cpuf = pops->psm_set_idlecpu; 592 if (pops->psm_unset_idlecpu) 593 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 594 595 psm_clkinit = pops->psm_clkinit; 596 597 if (pops->psm_timer_reprogram) 598 psm_timer_reprogram = pops->psm_timer_reprogram; 599 600 if (pops->psm_timer_enable) 601 psm_timer_enable = pops->psm_timer_enable; 602 603 if (pops->psm_timer_disable) 604 psm_timer_disable = pops->psm_timer_disable; 605 606 if (pops->psm_post_cyclic_setup) 607 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 608 609 /* check for multiple cpu's */ 610 if (cnt < 2) 611 return; 612 613 /* check for MP platforms */ 614 if (pops->psm_cpu_start == NULL) 615 return; 616 617 /* 618 * Set the dispatcher hook to enable cpu "wake up" 619 * when a thread becomes runnable. 620 */ 621 #if defined(_SIMULATOR_SUPPORT) 622 if (halt_idle_cpus && !simulator_run) { 623 disp_enq_thread = cpu_wakeup; 624 } 625 #else 626 if (halt_idle_cpus) { 627 disp_enq_thread = cpu_wakeup; 628 } 629 #endif /* _SIMULATOR_SUPPORT */ 630 631 if (pops->psm_disable_intr) 632 psm_disable_intr = pops->psm_disable_intr; 633 if (pops->psm_enable_intr) 634 psm_enable_intr = pops->psm_enable_intr; 635 636 psm_get_ipivect = pops->psm_get_ipivect; 637 638 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr", 639 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 640 (caddr_t)X_CALL_HIPRI, NULL, NULL); 641 (void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr", 642 (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO), 643 (caddr_t)X_CALL_MEDPRI, NULL, NULL); 644 645 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 646 } 647 648 static void 649 mach_picinit() 650 { 651 register struct psm_ops *pops; 652 extern void install_spl(void); /* XXX: belongs in a header file */ 653 #if defined(__amd64) && defined(DEBUG) 654 extern void *spl_patch, *slow_spl, *setsplhi_patch, *slow_setsplhi; 655 #endif 656 657 pops = mach_set[0]; 658 659 /* register the interrupt handlers */ 660 setlvl = pops->psm_intr_enter; 661 setlvlx = pops->psm_intr_exit; 662 663 /* initialize the interrupt hardware */ 664 (*pops->psm_picinit)(); 665 666 /* set interrupt mask for current ipl */ 667 setspl = pops->psm_setspl; 668 setspl(CPU->cpu_pri); 669 670 /* Install proper spl routine now that we can Program the PIC */ 671 #if defined(__amd64) 672 /* 673 * It would be better if we could check this at compile time 674 */ 675 ASSERT(((uintptr_t)&slow_setsplhi - (uintptr_t)&setsplhi_patch < 128) && 676 ((uintptr_t)&slow_spl - (uintptr_t)&spl_patch < 128)); 677 #endif 678 install_spl(); 679 } 680 681 uint_t cpu_freq; /* MHz */ 682 uint64_t cpu_freq_hz; /* measured (in hertz) */ 683 684 #define MEGA_HZ 1000000 685 686 static uint64_t 687 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 688 { 689 uint64_t cpu_hz; 690 691 if ((pit_counter == 0) || (*processor_clks == 0) || 692 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 693 return (0); 694 695 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 696 697 return (cpu_hz); 698 } 699 700 static uint64_t 701 mach_getcpufreq(void) 702 { 703 uint32_t pit_counter; 704 uint64_t processor_clks; 705 706 if (x86_feature & X86_TSC) { 707 /* 708 * We have a TSC. freq_tsc() knows how to measure the number 709 * of clock cycles sampled against the PIT. 710 */ 711 processor_clks = freq_tsc(&pit_counter); 712 return (mach_calchz(pit_counter, &processor_clks)); 713 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 714 #if defined(__amd64) 715 panic("mach_getcpufreq: no TSC!"); 716 #elif defined(__i386) 717 /* 718 * We are a Cyrix based on a 6x86 core or an Intel Pentium 719 * for which freq_notsc() knows how to measure the number of 720 * elapsed clock cycles sampled against the PIT 721 */ 722 processor_clks = freq_notsc(&pit_counter); 723 return (mach_calchz(pit_counter, &processor_clks)); 724 #endif /* __i386 */ 725 } 726 727 /* We do not know how to calculate cpu frequency for this cpu. */ 728 return (0); 729 } 730 731 /* 732 * If the clock speed of a cpu is found to be reported incorrectly, do not add 733 * to this array, instead improve the accuracy of the algorithm that determines 734 * the clock speed of the processor or extend the implementation to support the 735 * vendor as appropriate. This is here only to support adjusting the speed on 736 * older slower processors that mach_fixcpufreq() would not be able to account 737 * for otherwise. 738 */ 739 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 740 741 /* 742 * On fast processors the clock frequency that is measured may be off by 743 * a few MHz from the value printed on the part. This is a combination of 744 * the factors that for such fast parts being off by this much is within 745 * the tolerances for manufacture and because of the difficulties in the 746 * measurement that can lead to small error. This function uses some 747 * heuristics in order to tweak the value that was measured to match what 748 * is most likely printed on the part. 749 * 750 * Some examples: 751 * AMD Athlon 1000 mhz measured as 998 mhz 752 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 753 * Intel Pentium IV 1500 mhz measured as 1495mhz 754 * 755 * If in the future this function is no longer sufficient to correct 756 * for the error in the measurement, then the algorithm used to perform 757 * the measurement will have to be improved in order to increase accuracy 758 * rather than adding horrible and questionable kludges here. 759 * 760 * This is called after the cyclics subsystem because of the potential 761 * that the heuristics within may give a worse estimate of the clock 762 * frequency than the value that was measured. 763 */ 764 static void 765 mach_fixcpufreq(void) 766 { 767 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 768 769 freq = (uint32_t)cpu_freq; 770 771 /* 772 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 773 * measured speed taking into account that the 667 MHz parts were 774 * the first to round-up. 775 */ 776 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 777 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 778 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 779 780 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 781 mul = (freq + 25) / 50; 782 near50 = mul * 50; 783 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 784 785 /* Find the closer of the two */ 786 if (delta66 < delta50) { 787 fixed = near66; 788 delta = delta66; 789 } else { 790 fixed = near50; 791 delta = delta50; 792 } 793 794 if (fixed > INT_MAX) 795 return; 796 797 /* 798 * Some older parts have a core clock frequency that is not an 799 * integral multiple of 50 or 66 MHz. Check if one of the old 800 * clock frequencies is closer to the measured value than any 801 * of the integral multiples of 50 an 66, and if so set fixed 802 * and delta appropriately to represent the closest value. 803 */ 804 i = sizeof (x86_cpu_freq) / sizeof (int); 805 while (i > 0) { 806 i--; 807 808 if (x86_cpu_freq[i] <= freq) { 809 mul = freq - x86_cpu_freq[i]; 810 811 if (mul < delta) { 812 fixed = x86_cpu_freq[i]; 813 delta = mul; 814 } 815 816 break; 817 } 818 819 mul = x86_cpu_freq[i] - freq; 820 821 if (mul < delta) { 822 fixed = x86_cpu_freq[i]; 823 delta = mul; 824 } 825 } 826 827 /* 828 * Set a reasonable maximum for how much to correct the measured 829 * result by. This check is here to prevent the adjustment made 830 * by this function from being more harm than good. It is entirely 831 * possible that in the future parts will be made that are not 832 * integral multiples of 66 or 50 in clock frequency or that 833 * someone may overclock a part to some odd frequency. If the 834 * measured value is farther from the corrected value than 835 * allowed, then assume the corrected value is in error and use 836 * the measured value. 837 */ 838 if (6 < delta) 839 return; 840 841 cpu_freq = (int)fixed; 842 } 843 844 845 static int 846 machhztomhz(uint64_t cpu_freq_hz) 847 { 848 uint64_t cpu_mhz; 849 850 /* Round to nearest MHZ */ 851 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 852 853 if (cpu_mhz > INT_MAX) 854 return (0); 855 856 return ((int)cpu_mhz); 857 858 } 859 860 861 static int 862 mach_clkinit(int preferred_mode, int *set_mode) 863 { 864 register struct psm_ops *pops; 865 int resolution; 866 867 pops = mach_set[0]; 868 869 #ifdef _SIMULATOR_SUPPORT 870 if (!simulator_run) 871 cpu_freq_hz = mach_getcpufreq(); 872 else 873 cpu_freq_hz = 40000000; /* use 40 Mhz (hack for simulator) */ 874 #else 875 cpu_freq_hz = mach_getcpufreq(); 876 #endif /* _SIMULATOR_SUPPORT */ 877 878 cpu_freq = machhztomhz(cpu_freq_hz); 879 880 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 881 tsc_gethrtime_enable = 0; 882 883 if (tsc_gethrtime_enable) { 884 tsc_hrtimeinit(cpu_freq_hz); 885 gethrtimef = tsc_gethrtime; 886 gethrtimeunscaledf = tsc_gethrtimeunscaled; 887 scalehrtimef = tsc_scalehrtime; 888 hrtime_tick = tsc_tick; 889 tsc_gethrtime_initted = 1; 890 } else { 891 if (pops->psm_hrtimeinit) 892 (*pops->psm_hrtimeinit)(); 893 gethrtimef = pops->psm_gethrtime; 894 gethrtimeunscaledf = gethrtimef; 895 /* scalehrtimef will remain dummy */ 896 } 897 898 mach_fixcpufreq(); 899 900 if (mach_ver[0] >= PSM_INFO_VER01_3) { 901 if ((preferred_mode == TIMER_ONESHOT) && 902 (tsc_gethrtime_enable)) { 903 904 resolution = (*pops->psm_clkinit)(0); 905 if (resolution != 0) { 906 *set_mode = TIMER_ONESHOT; 907 return (resolution); 908 } 909 910 } 911 912 /* 913 * either periodic mode was requested or could not set to 914 * one-shot mode 915 */ 916 resolution = (*pops->psm_clkinit)(hz); 917 /* 918 * psm should be able to do periodic, so we do not check 919 * for return value of psm_clkinit here. 920 */ 921 *set_mode = TIMER_PERIODIC; 922 return (resolution); 923 } else { 924 /* 925 * PSMI interface prior to PSMI_3 does not define a return 926 * value for psm_clkinit, so the return value is ignored. 927 */ 928 (void) (*pops->psm_clkinit)(hz); 929 *set_mode = TIMER_PERIODIC; 930 return (nsec_per_tick); 931 } 932 } 933 934 static int 935 mach_softlvl_to_vect(register int ipl) 936 { 937 register int softvect; 938 register struct psm_ops *pops; 939 940 pops = mach_set[0]; 941 942 /* check for null handler for set soft interrupt call */ 943 if (pops->psm_set_softintr == NULL) { 944 setsoftint = set_pending; 945 return (PSM_SV_SOFTWARE); 946 } 947 948 softvect = (*pops->psm_softlvl_to_irq)(ipl); 949 /* check for hardware scheme */ 950 if (softvect > PSM_SV_SOFTWARE) { 951 setsoftint = pops->psm_set_softintr; 952 return (softvect); 953 } 954 955 if (softvect == PSM_SV_SOFTWARE) 956 setsoftint = set_pending; 957 else /* hardware and software mixed scheme */ 958 setsoftint = mach_set_softintr; 959 960 return (PSM_SV_SOFTWARE); 961 } 962 963 static void 964 mach_set_softintr(register int ipl) 965 { 966 register struct psm_ops *pops; 967 968 /* set software pending bits */ 969 set_pending(ipl); 970 971 /* check if dosoftint will be called at the end of intr */ 972 if (CPU_ON_INTR(CPU) || (curthread->t_intr)) 973 return; 974 975 /* invoke hardware interrupt */ 976 pops = mach_set[0]; 977 (*pops->psm_set_softintr)(ipl); 978 } 979 980 static void 981 mach_cpu_start(register int cpun) 982 { 983 register struct psm_ops *pops; 984 int i; 985 986 pops = mach_set[0]; 987 988 (*pops->psm_cpu_start)(cpun, rm_platter_va); 989 990 /* wait for the auxillary cpu to be ready */ 991 for (i = 20000; i; i--) { 992 if (cpu[cpun]->cpu_flags & CPU_READY) 993 return; 994 drv_usecwait(100); 995 } 996 } 997 998 /*ARGSUSED*/ 999 static int 1000 mach_translate_irq(dev_info_t *dip, int irqno) 1001 { 1002 return (irqno); /* default to NO translation */ 1003 } 1004 1005 static timestruc_t 1006 mach_tod_get(void) 1007 { 1008 timestruc_t ts; 1009 todinfo_t tod; 1010 static int mach_range_warn = 1; /* warn only once */ 1011 1012 ASSERT(MUTEX_HELD(&tod_lock)); 1013 1014 /* The year returned from is the last 2 digit only */ 1015 if ((*psm_todgetf)(&tod)) { 1016 ts.tv_sec = 0; 1017 ts.tv_nsec = 0; 1018 tod_fault_reset(); 1019 return (ts); 1020 } 1021 1022 /* assume that we wrap the rtc year back to zero at 2000 */ 1023 if (tod.tod_year < 69) { 1024 if (mach_range_warn && tod.tod_year > 38) { 1025 cmn_err(CE_WARN, "hardware real-time clock is out " 1026 "of range -- time needs to be reset"); 1027 mach_range_warn = 0; 1028 } 1029 tod.tod_year += 100; 1030 } 1031 1032 /* tod_to_utc uses 1900 as base for the year */ 1033 ts.tv_sec = tod_to_utc(tod) + gmt_lag; 1034 ts.tv_nsec = 0; 1035 1036 return (ts); 1037 } 1038 1039 static void 1040 mach_tod_set(timestruc_t ts) 1041 { 1042 todinfo_t tod = utc_to_tod(ts.tv_sec - gmt_lag); 1043 1044 ASSERT(MUTEX_HELD(&tod_lock)); 1045 1046 if (tod.tod_year >= 100) 1047 tod.tod_year -= 100; 1048 1049 (*psm_todsetf)(&tod); 1050 } 1051 1052 static void 1053 mach_notify_error(int level, char *errmsg) 1054 { 1055 /* 1056 * SL_FATAL is pass in once panicstr is set, deliver it 1057 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1058 * codes for the psmi handler 1059 */ 1060 if (level & SL_FATAL) 1061 (*notify_error)(CE_PANIC, errmsg); 1062 else if (level & SL_WARN) 1063 (*notify_error)(CE_WARN, errmsg); 1064 else if (level & SL_NOTE) 1065 (*notify_error)(CE_NOTE, errmsg); 1066 else if (level & SL_CONSOLE) 1067 (*notify_error)(CE_CONT, errmsg); 1068 } 1069 1070 /* 1071 * It provides the default basic intr_ops interface for the new DDI 1072 * interrupt framework if the PSM doesn't have one. 1073 * 1074 * Input: 1075 * dip - pointer to the dev_info structure of the requested device 1076 * hdlp - pointer to the internal interrupt handle structure for the 1077 * requested interrupt 1078 * intr_op - opcode for this call 1079 * result - pointer to the integer that will hold the result to be 1080 * passed back if return value is PSM_SUCCESS 1081 * 1082 * Output: 1083 * return value is either PSM_SUCCESS or PSM_FAILURE 1084 */ 1085 static int 1086 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1087 psm_intr_op_t intr_op, int *result) 1088 { 1089 struct intrspec *ispec; 1090 1091 switch (intr_op) { 1092 case PSM_INTR_OP_CHECK_MSI: 1093 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1094 DDI_INTR_TYPE_MSIX); 1095 break; 1096 case PSM_INTR_OP_ALLOC_VECTORS: 1097 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1098 *result = 1; 1099 else 1100 *result = 0; 1101 break; 1102 case PSM_INTR_OP_FREE_VECTORS: 1103 break; 1104 case PSM_INTR_OP_NAVAIL_VECTORS: 1105 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1106 *result = 1; 1107 else 1108 *result = 0; 1109 break; 1110 case PSM_INTR_OP_XLATE_VECTOR: 1111 ispec = (struct intrspec *)hdlp->ih_private; 1112 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1113 break; 1114 case PSM_INTR_OP_GET_CAP: 1115 *result = 0; 1116 break; 1117 case PSM_INTR_OP_GET_PENDING: 1118 case PSM_INTR_OP_CLEAR_MASK: 1119 case PSM_INTR_OP_SET_MASK: 1120 case PSM_INTR_OP_GET_SHARED: 1121 case PSM_INTR_OP_SET_PRI: 1122 case PSM_INTR_OP_SET_CAP: 1123 default: 1124 return (PSM_FAILURE); 1125 } 1126 return (PSM_SUCCESS); 1127 } 1128