1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #define PSMI_1_5 30 #include <sys/smp_impldefs.h> 31 #include <sys/psm.h> 32 #include <sys/psm_modctl.h> 33 #include <sys/pit.h> 34 #include <sys/cmn_err.h> 35 #include <sys/strlog.h> 36 #include <sys/clock.h> 37 #include <sys/debug.h> 38 #include <sys/rtc.h> 39 #include <sys/x86_archext.h> 40 #include <sys/cpupart.h> 41 #include <sys/cpuvar.h> 42 #include <sys/chip.h> 43 #include <sys/disp.h> 44 #include <sys/cpu.h> 45 #include <sys/archsystm.h> 46 #include <sys/mach_intr.h> 47 48 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 49 50 /* 51 * Local function prototypes 52 */ 53 static int mp_disable_intr(processorid_t cpun); 54 static void mp_enable_intr(processorid_t cpun); 55 static void mach_init(); 56 static void mach_picinit(); 57 static uint64_t mach_calchz(uint32_t pit_counter, uint64_t *processor_clks); 58 static int machhztomhz(uint64_t cpu_freq_hz); 59 static uint64_t mach_getcpufreq(void); 60 static void mach_fixcpufreq(void); 61 static int mach_clkinit(int, int *); 62 static void mach_smpinit(void); 63 static void mach_set_softintr(int ipl); 64 static void mach_cpu_start(int cpun); 65 static int mach_softlvl_to_vect(int ipl); 66 static void mach_get_platform(int owner); 67 static void mach_construct_info(); 68 static int mach_translate_irq(dev_info_t *dip, int irqno); 69 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 70 psm_intr_op_t, int *); 71 static timestruc_t mach_tod_get(void); 72 static void mach_tod_set(timestruc_t ts); 73 static void mach_notify_error(int level, char *errmsg); 74 static hrtime_t dummy_hrtime(void); 75 static void dummy_scalehrtime(hrtime_t *); 76 static void cpu_halt(void); 77 static void cpu_wakeup(cpu_t *, int); 78 /* 79 * External reference functions 80 */ 81 extern void return_instr(); 82 extern timestruc_t (*todgetf)(void); 83 extern void (*todsetf)(timestruc_t); 84 extern long gmt_lag; 85 extern uint64_t freq_tsc(uint32_t *); 86 #if defined(__i386) 87 extern uint64_t freq_notsc(uint32_t *); 88 #endif 89 extern void pc_gethrestime(timestruc_t *); 90 91 /* 92 * PSM functions initialization 93 */ 94 void (*psm_shutdownf)(int, int) = return_instr; 95 void (*psm_preshutdownf)(int, int) = return_instr; 96 void (*psm_notifyf)(int) = return_instr; 97 void (*psm_set_idle_cpuf)(int) = return_instr; 98 void (*psm_unset_idle_cpuf)(int) = return_instr; 99 void (*psminitf)() = mach_init; 100 void (*picinitf)() = return_instr; 101 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 102 void (*cpu_startf)() = return_instr; 103 int (*ap_mlsetup)() = (int (*)(void))return_instr; 104 void (*send_dirintf)() = return_instr; 105 void (*setspl)(int) = return_instr; 106 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 107 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 108 void (*setsoftint)(int) = (void (*)(int))return_instr; 109 int (*slvltovect)(int) = (int (*)(int))return_instr; 110 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 111 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 112 int (*psm_disable_intr)(int) = mp_disable_intr; 113 void (*psm_enable_intr)(int) = mp_enable_intr; 114 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 115 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 116 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 117 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 118 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 119 int (*psm_todgetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 120 int (*psm_todsetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 121 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 122 int (*psm_get_clockirq)(int) = NULL; 123 int (*psm_get_ipivect)(int, int) = NULL; 124 125 int (*psm_clkinit)(int) = NULL; 126 void (*psm_timer_reprogram)(hrtime_t) = NULL; 127 void (*psm_timer_enable)(void) = NULL; 128 void (*psm_timer_disable)(void) = NULL; 129 void (*psm_post_cyclic_setup)(void *arg) = NULL; 130 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 131 int *) = mach_intr_ops; 132 133 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 134 void (*hrtime_tick)(void) = return_instr; 135 136 int tsc_gethrtime_enable = 1; 137 int tsc_gethrtime_initted = 0; 138 139 /* 140 * Local Static Data 141 */ 142 static struct psm_ops mach_ops; 143 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 144 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 145 146 /* 147 * If non-zero, idle cpus will "halted" when there's 148 * no work to do. 149 */ 150 int halt_idle_cpus = 1; 151 152 #if defined(__amd64) 153 /* 154 * If non-zero, will use cr8 for interrupt priority masking 155 * We declare this here since install_spl is called from here 156 * (where this is checked). 157 */ 158 int intpri_use_cr8 = 0; 159 #endif /* __amd64 */ 160 161 #ifdef _SIMULATOR_SUPPORT 162 163 int simulator_run = 0; /* patch to non-zero if running under simics */ 164 165 #endif /* _SIMULATOR_SUPPORT */ 166 167 /* ARGSUSED */ 168 void 169 chip_plat_define_chip(cpu_t *cp, chip_def_t *cd) 170 { 171 if (x86_feature & (X86_HTT|X86_CMP)) 172 /* 173 * Hyperthreading is SMT 174 */ 175 cd->chipd_type = CHIP_SMT; 176 else 177 cd->chipd_type = CHIP_DEFAULT; 178 179 cd->chipd_rechoose_adj = 0; 180 } 181 182 /* 183 * Routine to ensure initial callers to hrtime gets 0 as return 184 */ 185 static hrtime_t 186 dummy_hrtime(void) 187 { 188 return (0); 189 } 190 191 /* ARGSUSED */ 192 static void 193 dummy_scalehrtime(hrtime_t *ticks) 194 {} 195 196 /* 197 * Halt the present CPU until awoken via an interrupt 198 */ 199 static void 200 cpu_halt(void) 201 { 202 cpu_t *cpup = CPU; 203 processorid_t cpun = cpup->cpu_id; 204 cpupart_t *cp = cpup->cpu_part; 205 int hset_update = 1; 206 207 /* 208 * If this CPU is online, and there's multiple CPUs 209 * in the system, then we should notate our halting 210 * by adding ourselves to the partition's halted CPU 211 * bitmap. This allows other CPUs to find/awaken us when 212 * work becomes available. 213 */ 214 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 215 hset_update = 0; 216 217 /* 218 * Add ourselves to the partition's halted CPUs bitmask 219 * and set our HALTED flag, if necessary. 220 * 221 * When a thread becomes runnable, it is placed on the queue 222 * and then the halted cpuset is checked to determine who 223 * (if anyone) should be awoken. We therefore need to first 224 * add ourselves to the halted cpuset, and and then check if there 225 * is any work available. 226 * 227 * Note that memory barriers after updating the HALTED flag 228 * are not necessary since an atomic operation (updating the bitmap) 229 * immediately follows. On x86 the atomic operation acts as a 230 * memory barrier for the update of cpu_disp_flags. 231 */ 232 if (hset_update) { 233 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 234 CPUSET_ATOMIC_ADD(cp->cp_haltset, cpun); 235 } 236 237 /* 238 * Check to make sure there's really nothing to do. 239 * Work destined for this CPU may become available after 240 * this check. We'll be notified through the clearing of our 241 * bit in the halted CPU bitmask, and a poke. 242 */ 243 if (disp_anywork()) { 244 if (hset_update) { 245 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 246 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 247 } 248 return; 249 } 250 251 /* 252 * We're on our way to being halted. 253 * 254 * Disable interrupts now, so that we'll awaken immediately 255 * after halting if someone tries to poke us between now and 256 * the time we actually halt. 257 * 258 * We check for the presence of our bit after disabling interrupts. 259 * If it's cleared, we'll return. If the bit is cleared after 260 * we check then the poke will pop us out of the halted state. 261 * 262 * This means that the ordering of the poke and the clearing 263 * of the bit by cpu_wakeup is important. 264 * cpu_wakeup() must clear, then poke. 265 * cpu_halt() must disable interrupts, then check for the bit. 266 */ 267 cli(); 268 269 if (hset_update && !CPU_IN_SET(cp->cp_haltset, cpun)) { 270 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 271 sti(); 272 return; 273 } 274 275 /* 276 * The check for anything locally runnable is here for performance 277 * and isn't needed for correctness. disp_nrunnable ought to be 278 * in our cache still, so it's inexpensive to check, and if there 279 * is anything runnable we won't have to wait for the poke. 280 */ 281 if (cpup->cpu_disp->disp_nrunnable != 0) { 282 if (hset_update) { 283 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 284 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 285 } 286 sti(); 287 return; 288 } 289 290 /* 291 * Call the halt sequence: 292 * sti 293 * hlt 294 */ 295 i86_halt(); 296 297 /* 298 * We're no longer halted 299 */ 300 if (hset_update) { 301 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 302 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 303 } 304 } 305 306 307 /* 308 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 309 * Otherwise, see if other CPUs in the cpu partition are halted and need to 310 * be woken up so that they can steal the thread we placed on this CPU. 311 * This function is only used on MP systems. 312 */ 313 static void 314 cpu_wakeup(cpu_t *cpu, int bound) 315 { 316 uint_t cpu_found; 317 int result; 318 cpupart_t *cp; 319 320 cp = cpu->cpu_part; 321 if (CPU_IN_SET(cp->cp_haltset, cpu->cpu_id)) { 322 /* 323 * Clear the halted bit for that CPU since it will be 324 * poked in a moment. 325 */ 326 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpu->cpu_id); 327 /* 328 * We may find the current CPU present in the halted cpuset 329 * if we're in the context of an interrupt that occurred 330 * before we had a chance to clear our bit in cpu_halt(). 331 * Poking ourself is obviously unnecessary, since if 332 * we're here, we're not halted. 333 */ 334 if (cpu != CPU) 335 poke_cpu(cpu->cpu_id); 336 return; 337 } else { 338 /* 339 * This cpu isn't halted, but it's idle or undergoing a 340 * context switch. No need to awaken anyone else. 341 */ 342 if (cpu->cpu_thread == cpu->cpu_idle_thread || 343 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 344 return; 345 } 346 347 /* 348 * No need to wake up other CPUs if the thread we just enqueued 349 * is bound. 350 */ 351 if (bound) 352 return; 353 354 355 /* 356 * See if there's any other halted CPUs. If there are, then 357 * select one, and awaken it. 358 * It's possible that after we find a CPU, somebody else 359 * will awaken it before we get the chance. 360 * In that case, look again. 361 */ 362 do { 363 CPUSET_FIND(cp->cp_haltset, cpu_found); 364 if (cpu_found == CPUSET_NOTINSET) 365 return; 366 367 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 368 CPUSET_ATOMIC_XDEL(cp->cp_haltset, cpu_found, result); 369 } while (result < 0); 370 371 if (cpu_found != CPU->cpu_id) 372 poke_cpu(cpu_found); 373 } 374 375 static int 376 mp_disable_intr(int cpun) 377 { 378 /* 379 * switch to the offline cpu 380 */ 381 affinity_set(cpun); 382 /* 383 * raise ipl to just below cross call 384 */ 385 splx(XC_MED_PIL-1); 386 /* 387 * set base spl to prevent the next swtch to idle from 388 * lowering back to ipl 0 389 */ 390 CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1)); 391 set_base_spl(); 392 affinity_clear(); 393 return (DDI_SUCCESS); 394 } 395 396 static void 397 mp_enable_intr(int cpun) 398 { 399 /* 400 * switch to the online cpu 401 */ 402 affinity_set(cpun); 403 /* 404 * clear the interrupt active mask 405 */ 406 CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1)); 407 set_base_spl(); 408 (void) spl0(); 409 affinity_clear(); 410 } 411 412 static void 413 mach_get_platform(int owner) 414 { 415 void **srv_opsp; 416 void **clt_opsp; 417 int i; 418 int total_ops; 419 420 /* fix up psm ops */ 421 srv_opsp = (void **)mach_set[0]; 422 clt_opsp = (void **)mach_set[owner]; 423 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 424 total_ops = sizeof (struct psm_ops_ver01) / 425 sizeof (void (*)(void)); 426 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 427 /* no psm_notify_func */ 428 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 429 sizeof (void (*)(void)); 430 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 431 /* no psm_timer funcs */ 432 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 433 sizeof (void (*)(void)); 434 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 435 /* no psm_preshutdown function */ 436 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 437 sizeof (void (*)(void)); 438 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 439 /* no psm_preshutdown function */ 440 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 441 sizeof (void (*)(void)); 442 else 443 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 444 445 /* 446 * Save the version of the PSM module, in case we need to 447 * bahave differently based on version. 448 */ 449 mach_ver[0] = mach_ver[owner]; 450 451 for (i = 0; i < total_ops; i++) 452 if (clt_opsp[i] != NULL) 453 srv_opsp[i] = clt_opsp[i]; 454 } 455 456 static void 457 mach_construct_info() 458 { 459 register struct psm_sw *swp; 460 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 461 int conflict_owner = 0; 462 463 if (psmsw->psw_forw == psmsw) 464 panic("No valid PSM modules found"); 465 mutex_enter(&psmsw_lock); 466 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 467 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 468 continue; 469 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 470 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 471 mach_cnt[swp->psw_infop->p_owner]++; 472 } 473 mutex_exit(&psmsw_lock); 474 475 mach_get_platform(PSM_OWN_SYS_DEFAULT); 476 477 /* check to see are there any conflicts */ 478 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 479 conflict_owner = PSM_OWN_EXCLUSIVE; 480 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 481 conflict_owner = PSM_OWN_OVERRIDE; 482 if (conflict_owner) { 483 /* remove all psm modules except uppc */ 484 cmn_err(CE_WARN, 485 "Conflicts detected on the following PSM modules:"); 486 mutex_enter(&psmsw_lock); 487 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 488 if (swp->psw_infop->p_owner == conflict_owner) 489 cmn_err(CE_WARN, "%s ", 490 swp->psw_infop->p_mach_idstring); 491 } 492 mutex_exit(&psmsw_lock); 493 cmn_err(CE_WARN, 494 "Setting the system back to SINGLE processor mode!"); 495 cmn_err(CE_WARN, 496 "Please edit /etc/mach to remove the invalid PSM module."); 497 return; 498 } 499 500 if (mach_set[PSM_OWN_EXCLUSIVE]) 501 mach_get_platform(PSM_OWN_EXCLUSIVE); 502 503 if (mach_set[PSM_OWN_OVERRIDE]) 504 mach_get_platform(PSM_OWN_OVERRIDE); 505 } 506 507 static void 508 mach_init() 509 { 510 register struct psm_ops *pops; 511 512 mach_construct_info(); 513 514 pops = mach_set[0]; 515 516 /* register the interrupt and clock initialization rotuines */ 517 picinitf = mach_picinit; 518 clkinitf = mach_clkinit; 519 psm_get_clockirq = pops->psm_get_clockirq; 520 521 /* register the interrupt setup code */ 522 slvltovect = mach_softlvl_to_vect; 523 addspl = pops->psm_addspl; 524 delspl = pops->psm_delspl; 525 526 if (pops->psm_translate_irq) 527 psm_translate_irq = pops->psm_translate_irq; 528 if (pops->psm_intr_ops) 529 psm_intr_ops = pops->psm_intr_ops; 530 if (pops->psm_tod_get) { 531 todgetf = mach_tod_get; 532 psm_todgetf = pops->psm_tod_get; 533 } 534 if (pops->psm_tod_set) { 535 todsetf = mach_tod_set; 536 psm_todsetf = pops->psm_tod_set; 537 } 538 if (pops->psm_notify_error) { 539 psm_notify_error = mach_notify_error; 540 notify_error = pops->psm_notify_error; 541 } 542 543 (*pops->psm_softinit)(); 544 545 /* 546 * Initialize the dispatcher's function hooks 547 * to enable CPU halting when idle 548 */ 549 #if defined(_SIMULATOR_SUPPORT) 550 if (halt_idle_cpus && !simulator_run) 551 idle_cpu = cpu_halt; 552 #else 553 if (halt_idle_cpus) 554 idle_cpu = cpu_halt; 555 #endif /* _SIMULATOR_SUPPORT */ 556 557 mach_smpinit(); 558 } 559 560 static void 561 mach_smpinit(void) 562 { 563 register struct psm_ops *pops; 564 register processorid_t cpu_id; 565 int cnt; 566 int cpumask; 567 568 pops = mach_set[0]; 569 570 cpu_id = -1; 571 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 572 for (cnt = 0, cpumask = 0; cpu_id != -1; cnt++) { 573 cpumask |= 1 << cpu_id; 574 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 575 } 576 577 mp_cpus = cpumask; 578 579 /* MP related routines */ 580 cpu_startf = mach_cpu_start; 581 ap_mlsetup = pops->psm_post_cpu_start; 582 send_dirintf = pops->psm_send_ipi; 583 584 /* optional MP related routines */ 585 if (pops->psm_shutdown) 586 psm_shutdownf = pops->psm_shutdown; 587 if (pops->psm_preshutdown) 588 psm_preshutdownf = pops->psm_preshutdown; 589 if (pops->psm_notify_func) 590 psm_notifyf = pops->psm_notify_func; 591 if (pops->psm_set_idlecpu) 592 psm_set_idle_cpuf = pops->psm_set_idlecpu; 593 if (pops->psm_unset_idlecpu) 594 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 595 596 psm_clkinit = pops->psm_clkinit; 597 598 if (pops->psm_timer_reprogram) 599 psm_timer_reprogram = pops->psm_timer_reprogram; 600 601 if (pops->psm_timer_enable) 602 psm_timer_enable = pops->psm_timer_enable; 603 604 if (pops->psm_timer_disable) 605 psm_timer_disable = pops->psm_timer_disable; 606 607 if (pops->psm_post_cyclic_setup) 608 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 609 610 /* check for multiple cpu's */ 611 if (cnt < 2) 612 return; 613 614 /* check for MP platforms */ 615 if (pops->psm_cpu_start == NULL) 616 return; 617 618 /* 619 * Set the dispatcher hook to enable cpu "wake up" 620 * when a thread becomes runnable. 621 */ 622 #if defined(_SIMULATOR_SUPPORT) 623 if (halt_idle_cpus && !simulator_run) { 624 disp_enq_thread = cpu_wakeup; 625 } 626 #else 627 if (halt_idle_cpus) { 628 disp_enq_thread = cpu_wakeup; 629 } 630 #endif /* _SIMULATOR_SUPPORT */ 631 632 if (pops->psm_disable_intr) 633 psm_disable_intr = pops->psm_disable_intr; 634 if (pops->psm_enable_intr) 635 psm_enable_intr = pops->psm_enable_intr; 636 637 psm_get_ipivect = pops->psm_get_ipivect; 638 639 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr", 640 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 641 (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL); 642 (void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr", 643 (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO), 644 (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL); 645 646 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 647 } 648 649 static void 650 mach_picinit() 651 { 652 register struct psm_ops *pops; 653 extern void install_spl(void); /* XXX: belongs in a header file */ 654 #if defined(__amd64) && defined(DEBUG) 655 extern void *spl_patch, *slow_spl, *setsplhi_patch, *slow_setsplhi; 656 #endif 657 658 pops = mach_set[0]; 659 660 /* register the interrupt handlers */ 661 setlvl = pops->psm_intr_enter; 662 setlvlx = pops->psm_intr_exit; 663 664 /* initialize the interrupt hardware */ 665 (*pops->psm_picinit)(); 666 667 /* set interrupt mask for current ipl */ 668 setspl = pops->psm_setspl; 669 setspl(CPU->cpu_pri); 670 671 /* Install proper spl routine now that we can Program the PIC */ 672 #if defined(__amd64) 673 /* 674 * It would be better if we could check this at compile time 675 */ 676 ASSERT(((uintptr_t)&slow_setsplhi - (uintptr_t)&setsplhi_patch < 128) && 677 ((uintptr_t)&slow_spl - (uintptr_t)&spl_patch < 128)); 678 #endif 679 install_spl(); 680 } 681 682 uint_t cpu_freq; /* MHz */ 683 uint64_t cpu_freq_hz; /* measured (in hertz) */ 684 685 #define MEGA_HZ 1000000 686 687 static uint64_t 688 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 689 { 690 uint64_t cpu_hz; 691 692 if ((pit_counter == 0) || (*processor_clks == 0) || 693 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 694 return (0); 695 696 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 697 698 return (cpu_hz); 699 } 700 701 static uint64_t 702 mach_getcpufreq(void) 703 { 704 uint32_t pit_counter; 705 uint64_t processor_clks; 706 707 if (x86_feature & X86_TSC) { 708 /* 709 * We have a TSC. freq_tsc() knows how to measure the number 710 * of clock cycles sampled against the PIT. 711 */ 712 processor_clks = freq_tsc(&pit_counter); 713 return (mach_calchz(pit_counter, &processor_clks)); 714 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 715 #if defined(__amd64) 716 panic("mach_getcpufreq: no TSC!"); 717 #elif defined(__i386) 718 /* 719 * We are a Cyrix based on a 6x86 core or an Intel Pentium 720 * for which freq_notsc() knows how to measure the number of 721 * elapsed clock cycles sampled against the PIT 722 */ 723 processor_clks = freq_notsc(&pit_counter); 724 return (mach_calchz(pit_counter, &processor_clks)); 725 #endif /* __i386 */ 726 } 727 728 /* We do not know how to calculate cpu frequency for this cpu. */ 729 return (0); 730 } 731 732 /* 733 * If the clock speed of a cpu is found to be reported incorrectly, do not add 734 * to this array, instead improve the accuracy of the algorithm that determines 735 * the clock speed of the processor or extend the implementation to support the 736 * vendor as appropriate. This is here only to support adjusting the speed on 737 * older slower processors that mach_fixcpufreq() would not be able to account 738 * for otherwise. 739 */ 740 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 741 742 /* 743 * On fast processors the clock frequency that is measured may be off by 744 * a few MHz from the value printed on the part. This is a combination of 745 * the factors that for such fast parts being off by this much is within 746 * the tolerances for manufacture and because of the difficulties in the 747 * measurement that can lead to small error. This function uses some 748 * heuristics in order to tweak the value that was measured to match what 749 * is most likely printed on the part. 750 * 751 * Some examples: 752 * AMD Athlon 1000 mhz measured as 998 mhz 753 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 754 * Intel Pentium IV 1500 mhz measured as 1495mhz 755 * 756 * If in the future this function is no longer sufficient to correct 757 * for the error in the measurement, then the algorithm used to perform 758 * the measurement will have to be improved in order to increase accuracy 759 * rather than adding horrible and questionable kludges here. 760 * 761 * This is called after the cyclics subsystem because of the potential 762 * that the heuristics within may give a worse estimate of the clock 763 * frequency than the value that was measured. 764 */ 765 static void 766 mach_fixcpufreq(void) 767 { 768 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 769 770 freq = (uint32_t)cpu_freq; 771 772 /* 773 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 774 * measured speed taking into account that the 667 MHz parts were 775 * the first to round-up. 776 */ 777 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 778 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 779 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 780 781 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 782 mul = (freq + 25) / 50; 783 near50 = mul * 50; 784 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 785 786 /* Find the closer of the two */ 787 if (delta66 < delta50) { 788 fixed = near66; 789 delta = delta66; 790 } else { 791 fixed = near50; 792 delta = delta50; 793 } 794 795 if (fixed > INT_MAX) 796 return; 797 798 /* 799 * Some older parts have a core clock frequency that is not an 800 * integral multiple of 50 or 66 MHz. Check if one of the old 801 * clock frequencies is closer to the measured value than any 802 * of the integral multiples of 50 an 66, and if so set fixed 803 * and delta appropriately to represent the closest value. 804 */ 805 i = sizeof (x86_cpu_freq) / sizeof (int); 806 while (i > 0) { 807 i--; 808 809 if (x86_cpu_freq[i] <= freq) { 810 mul = freq - x86_cpu_freq[i]; 811 812 if (mul < delta) { 813 fixed = x86_cpu_freq[i]; 814 delta = mul; 815 } 816 817 break; 818 } 819 820 mul = x86_cpu_freq[i] - freq; 821 822 if (mul < delta) { 823 fixed = x86_cpu_freq[i]; 824 delta = mul; 825 } 826 } 827 828 /* 829 * Set a reasonable maximum for how much to correct the measured 830 * result by. This check is here to prevent the adjustment made 831 * by this function from being more harm than good. It is entirely 832 * possible that in the future parts will be made that are not 833 * integral multiples of 66 or 50 in clock frequency or that 834 * someone may overclock a part to some odd frequency. If the 835 * measured value is farther from the corrected value than 836 * allowed, then assume the corrected value is in error and use 837 * the measured value. 838 */ 839 if (6 < delta) 840 return; 841 842 cpu_freq = (int)fixed; 843 } 844 845 846 static int 847 machhztomhz(uint64_t cpu_freq_hz) 848 { 849 uint64_t cpu_mhz; 850 851 /* Round to nearest MHZ */ 852 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 853 854 if (cpu_mhz > INT_MAX) 855 return (0); 856 857 return ((int)cpu_mhz); 858 859 } 860 861 862 static int 863 mach_clkinit(int preferred_mode, int *set_mode) 864 { 865 register struct psm_ops *pops; 866 int resolution; 867 868 pops = mach_set[0]; 869 870 #ifdef _SIMULATOR_SUPPORT 871 if (!simulator_run) 872 cpu_freq_hz = mach_getcpufreq(); 873 else 874 cpu_freq_hz = 40000000; /* use 40 Mhz (hack for simulator) */ 875 #else 876 cpu_freq_hz = mach_getcpufreq(); 877 #endif /* _SIMULATOR_SUPPORT */ 878 879 cpu_freq = machhztomhz(cpu_freq_hz); 880 881 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 882 tsc_gethrtime_enable = 0; 883 884 if (tsc_gethrtime_enable) { 885 tsc_hrtimeinit(cpu_freq_hz); 886 gethrtimef = tsc_gethrtime; 887 gethrtimeunscaledf = tsc_gethrtimeunscaled; 888 scalehrtimef = tsc_scalehrtime; 889 hrtime_tick = tsc_tick; 890 tsc_gethrtime_initted = 1; 891 } else { 892 if (pops->psm_hrtimeinit) 893 (*pops->psm_hrtimeinit)(); 894 gethrtimef = pops->psm_gethrtime; 895 gethrtimeunscaledf = gethrtimef; 896 /* scalehrtimef will remain dummy */ 897 } 898 899 mach_fixcpufreq(); 900 901 if (mach_ver[0] >= PSM_INFO_VER01_3) { 902 if ((preferred_mode == TIMER_ONESHOT) && 903 (tsc_gethrtime_enable)) { 904 905 resolution = (*pops->psm_clkinit)(0); 906 if (resolution != 0) { 907 *set_mode = TIMER_ONESHOT; 908 return (resolution); 909 } 910 911 } 912 913 /* 914 * either periodic mode was requested or could not set to 915 * one-shot mode 916 */ 917 resolution = (*pops->psm_clkinit)(hz); 918 /* 919 * psm should be able to do periodic, so we do not check 920 * for return value of psm_clkinit here. 921 */ 922 *set_mode = TIMER_PERIODIC; 923 return (resolution); 924 } else { 925 /* 926 * PSMI interface prior to PSMI_3 does not define a return 927 * value for psm_clkinit, so the return value is ignored. 928 */ 929 (void) (*pops->psm_clkinit)(hz); 930 *set_mode = TIMER_PERIODIC; 931 return (nsec_per_tick); 932 } 933 } 934 935 static int 936 mach_softlvl_to_vect(register int ipl) 937 { 938 register int softvect; 939 register struct psm_ops *pops; 940 941 pops = mach_set[0]; 942 943 /* check for null handler for set soft interrupt call */ 944 if (pops->psm_set_softintr == NULL) { 945 setsoftint = set_pending; 946 return (PSM_SV_SOFTWARE); 947 } 948 949 softvect = (*pops->psm_softlvl_to_irq)(ipl); 950 /* check for hardware scheme */ 951 if (softvect > PSM_SV_SOFTWARE) { 952 setsoftint = pops->psm_set_softintr; 953 return (softvect); 954 } 955 956 if (softvect == PSM_SV_SOFTWARE) 957 setsoftint = set_pending; 958 else /* hardware and software mixed scheme */ 959 setsoftint = mach_set_softintr; 960 961 return (PSM_SV_SOFTWARE); 962 } 963 964 static void 965 mach_set_softintr(register int ipl) 966 { 967 register struct psm_ops *pops; 968 969 /* set software pending bits */ 970 set_pending(ipl); 971 972 /* check if dosoftint will be called at the end of intr */ 973 if (CPU_ON_INTR(CPU) || (curthread->t_intr)) 974 return; 975 976 /* invoke hardware interrupt */ 977 pops = mach_set[0]; 978 (*pops->psm_set_softintr)(ipl); 979 } 980 981 static void 982 mach_cpu_start(register int cpun) 983 { 984 register struct psm_ops *pops; 985 int i; 986 987 pops = mach_set[0]; 988 989 (*pops->psm_cpu_start)(cpun, rm_platter_va); 990 991 /* wait for the auxillary cpu to be ready */ 992 for (i = 20000; i; i--) { 993 if (cpu[cpun]->cpu_flags & CPU_READY) 994 return; 995 drv_usecwait(100); 996 } 997 } 998 999 /*ARGSUSED*/ 1000 static int 1001 mach_translate_irq(dev_info_t *dip, int irqno) 1002 { 1003 return (irqno); /* default to NO translation */ 1004 } 1005 1006 static timestruc_t 1007 mach_tod_get(void) 1008 { 1009 timestruc_t ts; 1010 todinfo_t tod; 1011 static int mach_range_warn = 1; /* warn only once */ 1012 1013 ASSERT(MUTEX_HELD(&tod_lock)); 1014 1015 /* The year returned from is the last 2 digit only */ 1016 if ((*psm_todgetf)(&tod)) { 1017 ts.tv_sec = 0; 1018 ts.tv_nsec = 0; 1019 tod_fault_reset(); 1020 return (ts); 1021 } 1022 1023 /* assume that we wrap the rtc year back to zero at 2000 */ 1024 if (tod.tod_year < 69) { 1025 if (mach_range_warn && tod.tod_year > 38) { 1026 cmn_err(CE_WARN, "hardware real-time clock is out " 1027 "of range -- time needs to be reset"); 1028 mach_range_warn = 0; 1029 } 1030 tod.tod_year += 100; 1031 } 1032 1033 /* tod_to_utc uses 1900 as base for the year */ 1034 ts.tv_sec = tod_to_utc(tod) + gmt_lag; 1035 ts.tv_nsec = 0; 1036 1037 return (ts); 1038 } 1039 1040 static void 1041 mach_tod_set(timestruc_t ts) 1042 { 1043 todinfo_t tod = utc_to_tod(ts.tv_sec - gmt_lag); 1044 1045 ASSERT(MUTEX_HELD(&tod_lock)); 1046 1047 if (tod.tod_year >= 100) 1048 tod.tod_year -= 100; 1049 1050 (*psm_todsetf)(&tod); 1051 } 1052 1053 static void 1054 mach_notify_error(int level, char *errmsg) 1055 { 1056 /* 1057 * SL_FATAL is pass in once panicstr is set, deliver it 1058 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1059 * codes for the psmi handler 1060 */ 1061 if (level & SL_FATAL) 1062 (*notify_error)(CE_PANIC, errmsg); 1063 else if (level & SL_WARN) 1064 (*notify_error)(CE_WARN, errmsg); 1065 else if (level & SL_NOTE) 1066 (*notify_error)(CE_NOTE, errmsg); 1067 else if (level & SL_CONSOLE) 1068 (*notify_error)(CE_CONT, errmsg); 1069 } 1070 1071 /* 1072 * It provides the default basic intr_ops interface for the new DDI 1073 * interrupt framework if the PSM doesn't have one. 1074 * 1075 * Input: 1076 * dip - pointer to the dev_info structure of the requested device 1077 * hdlp - pointer to the internal interrupt handle structure for the 1078 * requested interrupt 1079 * intr_op - opcode for this call 1080 * result - pointer to the integer that will hold the result to be 1081 * passed back if return value is PSM_SUCCESS 1082 * 1083 * Output: 1084 * return value is either PSM_SUCCESS or PSM_FAILURE 1085 */ 1086 static int 1087 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1088 psm_intr_op_t intr_op, int *result) 1089 { 1090 struct intrspec *ispec; 1091 1092 switch (intr_op) { 1093 case PSM_INTR_OP_CHECK_MSI: 1094 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1095 DDI_INTR_TYPE_MSIX); 1096 break; 1097 case PSM_INTR_OP_ALLOC_VECTORS: 1098 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1099 *result = 1; 1100 else 1101 *result = 0; 1102 break; 1103 case PSM_INTR_OP_FREE_VECTORS: 1104 break; 1105 case PSM_INTR_OP_NAVAIL_VECTORS: 1106 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1107 *result = 1; 1108 else 1109 *result = 0; 1110 break; 1111 case PSM_INTR_OP_XLATE_VECTOR: 1112 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1113 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1114 break; 1115 case PSM_INTR_OP_GET_CAP: 1116 *result = 0; 1117 break; 1118 case PSM_INTR_OP_GET_PENDING: 1119 case PSM_INTR_OP_CLEAR_MASK: 1120 case PSM_INTR_OP_SET_MASK: 1121 case PSM_INTR_OP_GET_SHARED: 1122 case PSM_INTR_OP_SET_PRI: 1123 case PSM_INTR_OP_SET_CAP: 1124 case PSM_INTR_OP_SET_CPU: 1125 case PSM_INTR_OP_GET_INTR: 1126 default: 1127 return (PSM_FAILURE); 1128 } 1129 return (PSM_SUCCESS); 1130 } 1131