1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #define PSMI_1_5 30 #include <sys/smp_impldefs.h> 31 #include <sys/psm.h> 32 #include <sys/psm_modctl.h> 33 #include <sys/pit.h> 34 #include <sys/cmn_err.h> 35 #include <sys/strlog.h> 36 #include <sys/clock.h> 37 #include <sys/debug.h> 38 #include <sys/rtc.h> 39 #include <sys/x86_archext.h> 40 #include <sys/cpupart.h> 41 #include <sys/cpuvar.h> 42 #include <sys/chip.h> 43 #include <sys/disp.h> 44 #include <sys/cpu.h> 45 #include <sys/archsystm.h> 46 #include <sys/mach_intr.h> 47 48 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 49 50 /* 51 * Local function prototypes 52 */ 53 static int mp_disable_intr(processorid_t cpun); 54 static void mp_enable_intr(processorid_t cpun); 55 static void mach_init(); 56 static void mach_picinit(); 57 static uint64_t mach_calchz(uint32_t pit_counter, uint64_t *processor_clks); 58 static int machhztomhz(uint64_t cpu_freq_hz); 59 static uint64_t mach_getcpufreq(void); 60 static void mach_fixcpufreq(void); 61 static int mach_clkinit(int, int *); 62 static void mach_smpinit(void); 63 static void mach_set_softintr(int ipl, struct av_softinfo *); 64 static void mach_cpu_start(int cpun); 65 static int mach_softlvl_to_vect(int ipl); 66 static void mach_get_platform(int owner); 67 static void mach_construct_info(); 68 static int mach_translate_irq(dev_info_t *dip, int irqno); 69 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 70 psm_intr_op_t, int *); 71 static timestruc_t mach_tod_get(void); 72 static void mach_tod_set(timestruc_t ts); 73 static void mach_notify_error(int level, char *errmsg); 74 static hrtime_t dummy_hrtime(void); 75 static void dummy_scalehrtime(hrtime_t *); 76 static void cpu_halt(void); 77 static void cpu_wakeup(cpu_t *, int); 78 /* 79 * External reference functions 80 */ 81 extern void return_instr(); 82 extern timestruc_t (*todgetf)(void); 83 extern void (*todsetf)(timestruc_t); 84 extern long gmt_lag; 85 extern uint64_t freq_tsc(uint32_t *); 86 #if defined(__i386) 87 extern uint64_t freq_notsc(uint32_t *); 88 #endif 89 extern void pc_gethrestime(timestruc_t *); 90 91 /* 92 * PSM functions initialization 93 */ 94 void (*psm_shutdownf)(int, int) = return_instr; 95 void (*psm_preshutdownf)(int, int) = return_instr; 96 void (*psm_notifyf)(int) = return_instr; 97 void (*psm_set_idle_cpuf)(int) = return_instr; 98 void (*psm_unset_idle_cpuf)(int) = return_instr; 99 void (*psminitf)() = mach_init; 100 void (*picinitf)() = return_instr; 101 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 102 void (*cpu_startf)() = return_instr; 103 int (*ap_mlsetup)() = (int (*)(void))return_instr; 104 void (*send_dirintf)() = return_instr; 105 void (*setspl)(int) = return_instr; 106 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 107 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 108 void (*setsoftint)(int, struct av_softinfo *)= 109 (void (*)(int, struct av_softinfo *))return_instr; 110 int (*slvltovect)(int) = (int (*)(int))return_instr; 111 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 112 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 113 int (*psm_disable_intr)(int) = mp_disable_intr; 114 void (*psm_enable_intr)(int) = mp_enable_intr; 115 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 116 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 117 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 118 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 119 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 120 int (*psm_todgetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 121 int (*psm_todsetf)(todinfo_t *) = (int (*)(todinfo_t *))return_instr; 122 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 123 int (*psm_get_clockirq)(int) = NULL; 124 int (*psm_get_ipivect)(int, int) = NULL; 125 126 int (*psm_clkinit)(int) = NULL; 127 void (*psm_timer_reprogram)(hrtime_t) = NULL; 128 void (*psm_timer_enable)(void) = NULL; 129 void (*psm_timer_disable)(void) = NULL; 130 void (*psm_post_cyclic_setup)(void *arg) = NULL; 131 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 132 int *) = mach_intr_ops; 133 134 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 135 void (*hrtime_tick)(void) = return_instr; 136 137 int tsc_gethrtime_enable = 1; 138 int tsc_gethrtime_initted = 0; 139 140 /* 141 * Local Static Data 142 */ 143 static struct psm_ops mach_ops; 144 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 145 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 146 147 /* 148 * If non-zero, idle cpus will "halted" when there's 149 * no work to do. 150 */ 151 int halt_idle_cpus = 1; 152 153 #if defined(__amd64) 154 /* 155 * If non-zero, will use cr8 for interrupt priority masking 156 * We declare this here since install_spl is called from here 157 * (where this is checked). 158 */ 159 int intpri_use_cr8 = 0; 160 #endif /* __amd64 */ 161 162 #ifdef _SIMULATOR_SUPPORT 163 164 int simulator_run = 0; /* patch to non-zero if running under simics */ 165 166 #endif /* _SIMULATOR_SUPPORT */ 167 168 /* ARGSUSED */ 169 void 170 chip_plat_define_chip(cpu_t *cp, chip_def_t *cd) 171 { 172 if (x86_feature & (X86_HTT|X86_CMP)) 173 /* 174 * Hyperthreading is SMT 175 */ 176 cd->chipd_type = CHIP_SMT; 177 else 178 cd->chipd_type = CHIP_DEFAULT; 179 180 cd->chipd_rechoose_adj = 0; 181 } 182 183 /* 184 * Routine to ensure initial callers to hrtime gets 0 as return 185 */ 186 static hrtime_t 187 dummy_hrtime(void) 188 { 189 return (0); 190 } 191 192 /* ARGSUSED */ 193 static void 194 dummy_scalehrtime(hrtime_t *ticks) 195 {} 196 197 /* 198 * Halt the present CPU until awoken via an interrupt 199 */ 200 static void 201 cpu_halt(void) 202 { 203 cpu_t *cpup = CPU; 204 processorid_t cpun = cpup->cpu_id; 205 cpupart_t *cp = cpup->cpu_part; 206 int hset_update = 1; 207 208 /* 209 * If this CPU is online, and there's multiple CPUs 210 * in the system, then we should notate our halting 211 * by adding ourselves to the partition's halted CPU 212 * bitmap. This allows other CPUs to find/awaken us when 213 * work becomes available. 214 */ 215 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 216 hset_update = 0; 217 218 /* 219 * Add ourselves to the partition's halted CPUs bitmask 220 * and set our HALTED flag, if necessary. 221 * 222 * When a thread becomes runnable, it is placed on the queue 223 * and then the halted cpuset is checked to determine who 224 * (if anyone) should be awoken. We therefore need to first 225 * add ourselves to the halted cpuset, and and then check if there 226 * is any work available. 227 * 228 * Note that memory barriers after updating the HALTED flag 229 * are not necessary since an atomic operation (updating the bitmap) 230 * immediately follows. On x86 the atomic operation acts as a 231 * memory barrier for the update of cpu_disp_flags. 232 */ 233 if (hset_update) { 234 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 235 CPUSET_ATOMIC_ADD(cp->cp_haltset, cpun); 236 } 237 238 /* 239 * Check to make sure there's really nothing to do. 240 * Work destined for this CPU may become available after 241 * this check. We'll be notified through the clearing of our 242 * bit in the halted CPU bitmask, and a poke. 243 */ 244 if (disp_anywork()) { 245 if (hset_update) { 246 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 247 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 248 } 249 return; 250 } 251 252 /* 253 * We're on our way to being halted. 254 * 255 * Disable interrupts now, so that we'll awaken immediately 256 * after halting if someone tries to poke us between now and 257 * the time we actually halt. 258 * 259 * We check for the presence of our bit after disabling interrupts. 260 * If it's cleared, we'll return. If the bit is cleared after 261 * we check then the poke will pop us out of the halted state. 262 * 263 * This means that the ordering of the poke and the clearing 264 * of the bit by cpu_wakeup is important. 265 * cpu_wakeup() must clear, then poke. 266 * cpu_halt() must disable interrupts, then check for the bit. 267 */ 268 cli(); 269 270 if (hset_update && !CPU_IN_SET(cp->cp_haltset, cpun)) { 271 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 272 sti(); 273 return; 274 } 275 276 /* 277 * The check for anything locally runnable is here for performance 278 * and isn't needed for correctness. disp_nrunnable ought to be 279 * in our cache still, so it's inexpensive to check, and if there 280 * is anything runnable we won't have to wait for the poke. 281 */ 282 if (cpup->cpu_disp->disp_nrunnable != 0) { 283 if (hset_update) { 284 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 285 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 286 } 287 sti(); 288 return; 289 } 290 291 /* 292 * Call the halt sequence: 293 * sti 294 * hlt 295 */ 296 i86_halt(); 297 298 /* 299 * We're no longer halted 300 */ 301 if (hset_update) { 302 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 303 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpun); 304 } 305 } 306 307 308 /* 309 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 310 * Otherwise, see if other CPUs in the cpu partition are halted and need to 311 * be woken up so that they can steal the thread we placed on this CPU. 312 * This function is only used on MP systems. 313 */ 314 static void 315 cpu_wakeup(cpu_t *cpu, int bound) 316 { 317 uint_t cpu_found; 318 int result; 319 cpupart_t *cp; 320 321 cp = cpu->cpu_part; 322 if (CPU_IN_SET(cp->cp_haltset, cpu->cpu_id)) { 323 /* 324 * Clear the halted bit for that CPU since it will be 325 * poked in a moment. 326 */ 327 CPUSET_ATOMIC_DEL(cp->cp_haltset, cpu->cpu_id); 328 /* 329 * We may find the current CPU present in the halted cpuset 330 * if we're in the context of an interrupt that occurred 331 * before we had a chance to clear our bit in cpu_halt(). 332 * Poking ourself is obviously unnecessary, since if 333 * we're here, we're not halted. 334 */ 335 if (cpu != CPU) 336 poke_cpu(cpu->cpu_id); 337 return; 338 } else { 339 /* 340 * This cpu isn't halted, but it's idle or undergoing a 341 * context switch. No need to awaken anyone else. 342 */ 343 if (cpu->cpu_thread == cpu->cpu_idle_thread || 344 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 345 return; 346 } 347 348 /* 349 * No need to wake up other CPUs if the thread we just enqueued 350 * is bound. 351 */ 352 if (bound) 353 return; 354 355 356 /* 357 * See if there's any other halted CPUs. If there are, then 358 * select one, and awaken it. 359 * It's possible that after we find a CPU, somebody else 360 * will awaken it before we get the chance. 361 * In that case, look again. 362 */ 363 do { 364 CPUSET_FIND(cp->cp_haltset, cpu_found); 365 if (cpu_found == CPUSET_NOTINSET) 366 return; 367 368 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 369 CPUSET_ATOMIC_XDEL(cp->cp_haltset, cpu_found, result); 370 } while (result < 0); 371 372 if (cpu_found != CPU->cpu_id) 373 poke_cpu(cpu_found); 374 } 375 376 static int 377 mp_disable_intr(int cpun) 378 { 379 /* 380 * switch to the offline cpu 381 */ 382 affinity_set(cpun); 383 /* 384 * raise ipl to just below cross call 385 */ 386 splx(XC_MED_PIL-1); 387 /* 388 * set base spl to prevent the next swtch to idle from 389 * lowering back to ipl 0 390 */ 391 CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1)); 392 set_base_spl(); 393 affinity_clear(); 394 return (DDI_SUCCESS); 395 } 396 397 static void 398 mp_enable_intr(int cpun) 399 { 400 /* 401 * switch to the online cpu 402 */ 403 affinity_set(cpun); 404 /* 405 * clear the interrupt active mask 406 */ 407 CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1)); 408 set_base_spl(); 409 (void) spl0(); 410 affinity_clear(); 411 } 412 413 static void 414 mach_get_platform(int owner) 415 { 416 void **srv_opsp; 417 void **clt_opsp; 418 int i; 419 int total_ops; 420 421 /* fix up psm ops */ 422 srv_opsp = (void **)mach_set[0]; 423 clt_opsp = (void **)mach_set[owner]; 424 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 425 total_ops = sizeof (struct psm_ops_ver01) / 426 sizeof (void (*)(void)); 427 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 428 /* no psm_notify_func */ 429 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 430 sizeof (void (*)(void)); 431 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 432 /* no psm_timer funcs */ 433 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 434 sizeof (void (*)(void)); 435 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 436 /* no psm_preshutdown function */ 437 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 438 sizeof (void (*)(void)); 439 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 440 /* no psm_preshutdown function */ 441 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 442 sizeof (void (*)(void)); 443 else 444 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 445 446 /* 447 * Save the version of the PSM module, in case we need to 448 * bahave differently based on version. 449 */ 450 mach_ver[0] = mach_ver[owner]; 451 452 for (i = 0; i < total_ops; i++) 453 if (clt_opsp[i] != NULL) 454 srv_opsp[i] = clt_opsp[i]; 455 } 456 457 static void 458 mach_construct_info() 459 { 460 register struct psm_sw *swp; 461 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 462 int conflict_owner = 0; 463 464 if (psmsw->psw_forw == psmsw) 465 panic("No valid PSM modules found"); 466 mutex_enter(&psmsw_lock); 467 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 468 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 469 continue; 470 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 471 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 472 mach_cnt[swp->psw_infop->p_owner]++; 473 } 474 mutex_exit(&psmsw_lock); 475 476 mach_get_platform(PSM_OWN_SYS_DEFAULT); 477 478 /* check to see are there any conflicts */ 479 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 480 conflict_owner = PSM_OWN_EXCLUSIVE; 481 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 482 conflict_owner = PSM_OWN_OVERRIDE; 483 if (conflict_owner) { 484 /* remove all psm modules except uppc */ 485 cmn_err(CE_WARN, 486 "Conflicts detected on the following PSM modules:"); 487 mutex_enter(&psmsw_lock); 488 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 489 if (swp->psw_infop->p_owner == conflict_owner) 490 cmn_err(CE_WARN, "%s ", 491 swp->psw_infop->p_mach_idstring); 492 } 493 mutex_exit(&psmsw_lock); 494 cmn_err(CE_WARN, 495 "Setting the system back to SINGLE processor mode!"); 496 cmn_err(CE_WARN, 497 "Please edit /etc/mach to remove the invalid PSM module."); 498 return; 499 } 500 501 if (mach_set[PSM_OWN_EXCLUSIVE]) 502 mach_get_platform(PSM_OWN_EXCLUSIVE); 503 504 if (mach_set[PSM_OWN_OVERRIDE]) 505 mach_get_platform(PSM_OWN_OVERRIDE); 506 } 507 508 static void 509 mach_init() 510 { 511 register struct psm_ops *pops; 512 513 mach_construct_info(); 514 515 pops = mach_set[0]; 516 517 /* register the interrupt and clock initialization rotuines */ 518 picinitf = mach_picinit; 519 clkinitf = mach_clkinit; 520 psm_get_clockirq = pops->psm_get_clockirq; 521 522 /* register the interrupt setup code */ 523 slvltovect = mach_softlvl_to_vect; 524 addspl = pops->psm_addspl; 525 delspl = pops->psm_delspl; 526 527 if (pops->psm_translate_irq) 528 psm_translate_irq = pops->psm_translate_irq; 529 if (pops->psm_intr_ops) 530 psm_intr_ops = pops->psm_intr_ops; 531 if (pops->psm_tod_get) { 532 todgetf = mach_tod_get; 533 psm_todgetf = pops->psm_tod_get; 534 } 535 if (pops->psm_tod_set) { 536 todsetf = mach_tod_set; 537 psm_todsetf = pops->psm_tod_set; 538 } 539 if (pops->psm_notify_error) { 540 psm_notify_error = mach_notify_error; 541 notify_error = pops->psm_notify_error; 542 } 543 544 (*pops->psm_softinit)(); 545 546 /* 547 * Initialize the dispatcher's function hooks 548 * to enable CPU halting when idle 549 */ 550 #if defined(_SIMULATOR_SUPPORT) 551 if (halt_idle_cpus && !simulator_run) 552 idle_cpu = cpu_halt; 553 #else 554 if (halt_idle_cpus) 555 idle_cpu = cpu_halt; 556 #endif /* _SIMULATOR_SUPPORT */ 557 558 mach_smpinit(); 559 } 560 561 static void 562 mach_smpinit(void) 563 { 564 register struct psm_ops *pops; 565 register processorid_t cpu_id; 566 int cnt; 567 int cpumask; 568 569 pops = mach_set[0]; 570 571 cpu_id = -1; 572 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 573 for (cnt = 0, cpumask = 0; cpu_id != -1; cnt++) { 574 cpumask |= 1 << cpu_id; 575 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 576 } 577 578 mp_cpus = cpumask; 579 580 /* MP related routines */ 581 cpu_startf = mach_cpu_start; 582 ap_mlsetup = pops->psm_post_cpu_start; 583 send_dirintf = pops->psm_send_ipi; 584 585 /* optional MP related routines */ 586 if (pops->psm_shutdown) 587 psm_shutdownf = pops->psm_shutdown; 588 if (pops->psm_preshutdown) 589 psm_preshutdownf = pops->psm_preshutdown; 590 if (pops->psm_notify_func) 591 psm_notifyf = pops->psm_notify_func; 592 if (pops->psm_set_idlecpu) 593 psm_set_idle_cpuf = pops->psm_set_idlecpu; 594 if (pops->psm_unset_idlecpu) 595 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 596 597 psm_clkinit = pops->psm_clkinit; 598 599 if (pops->psm_timer_reprogram) 600 psm_timer_reprogram = pops->psm_timer_reprogram; 601 602 if (pops->psm_timer_enable) 603 psm_timer_enable = pops->psm_timer_enable; 604 605 if (pops->psm_timer_disable) 606 psm_timer_disable = pops->psm_timer_disable; 607 608 if (pops->psm_post_cyclic_setup) 609 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 610 611 /* check for multiple cpu's */ 612 if (cnt < 2) 613 return; 614 615 /* check for MP platforms */ 616 if (pops->psm_cpu_start == NULL) 617 return; 618 619 /* 620 * Set the dispatcher hook to enable cpu "wake up" 621 * when a thread becomes runnable. 622 */ 623 #if defined(_SIMULATOR_SUPPORT) 624 if (halt_idle_cpus && !simulator_run) { 625 disp_enq_thread = cpu_wakeup; 626 } 627 #else 628 if (halt_idle_cpus) { 629 disp_enq_thread = cpu_wakeup; 630 } 631 #endif /* _SIMULATOR_SUPPORT */ 632 633 if (pops->psm_disable_intr) 634 psm_disable_intr = pops->psm_disable_intr; 635 if (pops->psm_enable_intr) 636 psm_enable_intr = pops->psm_enable_intr; 637 638 psm_get_ipivect = pops->psm_get_ipivect; 639 640 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr", 641 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 642 (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL); 643 (void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr", 644 (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO), 645 (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL); 646 647 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 648 } 649 650 static void 651 mach_picinit() 652 { 653 register struct psm_ops *pops; 654 extern void install_spl(void); /* XXX: belongs in a header file */ 655 #if defined(__amd64) && defined(DEBUG) 656 extern void *spl_patch, *slow_spl, *setsplhi_patch, *slow_setsplhi; 657 #endif 658 659 pops = mach_set[0]; 660 661 /* register the interrupt handlers */ 662 setlvl = pops->psm_intr_enter; 663 setlvlx = pops->psm_intr_exit; 664 665 /* initialize the interrupt hardware */ 666 (*pops->psm_picinit)(); 667 668 /* set interrupt mask for current ipl */ 669 setspl = pops->psm_setspl; 670 setspl(CPU->cpu_pri); 671 672 /* Install proper spl routine now that we can Program the PIC */ 673 #if defined(__amd64) 674 /* 675 * It would be better if we could check this at compile time 676 */ 677 ASSERT(((uintptr_t)&slow_setsplhi - (uintptr_t)&setsplhi_patch < 128) && 678 ((uintptr_t)&slow_spl - (uintptr_t)&spl_patch < 128)); 679 #endif 680 install_spl(); 681 } 682 683 uint_t cpu_freq; /* MHz */ 684 uint64_t cpu_freq_hz; /* measured (in hertz) */ 685 686 #define MEGA_HZ 1000000 687 688 static uint64_t 689 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 690 { 691 uint64_t cpu_hz; 692 693 if ((pit_counter == 0) || (*processor_clks == 0) || 694 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 695 return (0); 696 697 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 698 699 return (cpu_hz); 700 } 701 702 static uint64_t 703 mach_getcpufreq(void) 704 { 705 uint32_t pit_counter; 706 uint64_t processor_clks; 707 708 if (x86_feature & X86_TSC) { 709 /* 710 * We have a TSC. freq_tsc() knows how to measure the number 711 * of clock cycles sampled against the PIT. 712 */ 713 processor_clks = freq_tsc(&pit_counter); 714 return (mach_calchz(pit_counter, &processor_clks)); 715 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 716 #if defined(__amd64) 717 panic("mach_getcpufreq: no TSC!"); 718 #elif defined(__i386) 719 /* 720 * We are a Cyrix based on a 6x86 core or an Intel Pentium 721 * for which freq_notsc() knows how to measure the number of 722 * elapsed clock cycles sampled against the PIT 723 */ 724 processor_clks = freq_notsc(&pit_counter); 725 return (mach_calchz(pit_counter, &processor_clks)); 726 #endif /* __i386 */ 727 } 728 729 /* We do not know how to calculate cpu frequency for this cpu. */ 730 return (0); 731 } 732 733 /* 734 * If the clock speed of a cpu is found to be reported incorrectly, do not add 735 * to this array, instead improve the accuracy of the algorithm that determines 736 * the clock speed of the processor or extend the implementation to support the 737 * vendor as appropriate. This is here only to support adjusting the speed on 738 * older slower processors that mach_fixcpufreq() would not be able to account 739 * for otherwise. 740 */ 741 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 742 743 /* 744 * On fast processors the clock frequency that is measured may be off by 745 * a few MHz from the value printed on the part. This is a combination of 746 * the factors that for such fast parts being off by this much is within 747 * the tolerances for manufacture and because of the difficulties in the 748 * measurement that can lead to small error. This function uses some 749 * heuristics in order to tweak the value that was measured to match what 750 * is most likely printed on the part. 751 * 752 * Some examples: 753 * AMD Athlon 1000 mhz measured as 998 mhz 754 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 755 * Intel Pentium IV 1500 mhz measured as 1495mhz 756 * 757 * If in the future this function is no longer sufficient to correct 758 * for the error in the measurement, then the algorithm used to perform 759 * the measurement will have to be improved in order to increase accuracy 760 * rather than adding horrible and questionable kludges here. 761 * 762 * This is called after the cyclics subsystem because of the potential 763 * that the heuristics within may give a worse estimate of the clock 764 * frequency than the value that was measured. 765 */ 766 static void 767 mach_fixcpufreq(void) 768 { 769 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 770 771 freq = (uint32_t)cpu_freq; 772 773 /* 774 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 775 * measured speed taking into account that the 667 MHz parts were 776 * the first to round-up. 777 */ 778 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 779 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 780 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 781 782 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 783 mul = (freq + 25) / 50; 784 near50 = mul * 50; 785 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 786 787 /* Find the closer of the two */ 788 if (delta66 < delta50) { 789 fixed = near66; 790 delta = delta66; 791 } else { 792 fixed = near50; 793 delta = delta50; 794 } 795 796 if (fixed > INT_MAX) 797 return; 798 799 /* 800 * Some older parts have a core clock frequency that is not an 801 * integral multiple of 50 or 66 MHz. Check if one of the old 802 * clock frequencies is closer to the measured value than any 803 * of the integral multiples of 50 an 66, and if so set fixed 804 * and delta appropriately to represent the closest value. 805 */ 806 i = sizeof (x86_cpu_freq) / sizeof (int); 807 while (i > 0) { 808 i--; 809 810 if (x86_cpu_freq[i] <= freq) { 811 mul = freq - x86_cpu_freq[i]; 812 813 if (mul < delta) { 814 fixed = x86_cpu_freq[i]; 815 delta = mul; 816 } 817 818 break; 819 } 820 821 mul = x86_cpu_freq[i] - freq; 822 823 if (mul < delta) { 824 fixed = x86_cpu_freq[i]; 825 delta = mul; 826 } 827 } 828 829 /* 830 * Set a reasonable maximum for how much to correct the measured 831 * result by. This check is here to prevent the adjustment made 832 * by this function from being more harm than good. It is entirely 833 * possible that in the future parts will be made that are not 834 * integral multiples of 66 or 50 in clock frequency or that 835 * someone may overclock a part to some odd frequency. If the 836 * measured value is farther from the corrected value than 837 * allowed, then assume the corrected value is in error and use 838 * the measured value. 839 */ 840 if (6 < delta) 841 return; 842 843 cpu_freq = (int)fixed; 844 } 845 846 847 static int 848 machhztomhz(uint64_t cpu_freq_hz) 849 { 850 uint64_t cpu_mhz; 851 852 /* Round to nearest MHZ */ 853 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 854 855 if (cpu_mhz > INT_MAX) 856 return (0); 857 858 return ((int)cpu_mhz); 859 860 } 861 862 863 static int 864 mach_clkinit(int preferred_mode, int *set_mode) 865 { 866 register struct psm_ops *pops; 867 int resolution; 868 869 pops = mach_set[0]; 870 871 #ifdef _SIMULATOR_SUPPORT 872 if (!simulator_run) 873 cpu_freq_hz = mach_getcpufreq(); 874 else 875 cpu_freq_hz = 40000000; /* use 40 Mhz (hack for simulator) */ 876 #else 877 cpu_freq_hz = mach_getcpufreq(); 878 #endif /* _SIMULATOR_SUPPORT */ 879 880 cpu_freq = machhztomhz(cpu_freq_hz); 881 882 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 883 tsc_gethrtime_enable = 0; 884 885 if (tsc_gethrtime_enable) { 886 tsc_hrtimeinit(cpu_freq_hz); 887 gethrtimef = tsc_gethrtime; 888 gethrtimeunscaledf = tsc_gethrtimeunscaled; 889 scalehrtimef = tsc_scalehrtime; 890 hrtime_tick = tsc_tick; 891 tsc_gethrtime_initted = 1; 892 } else { 893 if (pops->psm_hrtimeinit) 894 (*pops->psm_hrtimeinit)(); 895 gethrtimef = pops->psm_gethrtime; 896 gethrtimeunscaledf = gethrtimef; 897 /* scalehrtimef will remain dummy */ 898 } 899 900 mach_fixcpufreq(); 901 902 if (mach_ver[0] >= PSM_INFO_VER01_3) { 903 if ((preferred_mode == TIMER_ONESHOT) && 904 (tsc_gethrtime_enable)) { 905 906 resolution = (*pops->psm_clkinit)(0); 907 if (resolution != 0) { 908 *set_mode = TIMER_ONESHOT; 909 return (resolution); 910 } 911 912 } 913 914 /* 915 * either periodic mode was requested or could not set to 916 * one-shot mode 917 */ 918 resolution = (*pops->psm_clkinit)(hz); 919 /* 920 * psm should be able to do periodic, so we do not check 921 * for return value of psm_clkinit here. 922 */ 923 *set_mode = TIMER_PERIODIC; 924 return (resolution); 925 } else { 926 /* 927 * PSMI interface prior to PSMI_3 does not define a return 928 * value for psm_clkinit, so the return value is ignored. 929 */ 930 (void) (*pops->psm_clkinit)(hz); 931 *set_mode = TIMER_PERIODIC; 932 return (nsec_per_tick); 933 } 934 } 935 936 /*ARGSUSED*/ 937 static void 938 mach_psm_set_softintr(int ipl, struct av_softinfo *pending) 939 { 940 register struct psm_ops *pops; 941 942 /* invoke hardware interrupt */ 943 pops = mach_set[0]; 944 (*pops->psm_set_softintr)(ipl); 945 } 946 947 static int 948 mach_softlvl_to_vect(register int ipl) 949 { 950 register int softvect; 951 register struct psm_ops *pops; 952 953 pops = mach_set[0]; 954 955 /* check for null handler for set soft interrupt call */ 956 if (pops->psm_set_softintr == NULL) { 957 setsoftint = av_set_softint_pending; 958 return (PSM_SV_SOFTWARE); 959 } 960 961 softvect = (*pops->psm_softlvl_to_irq)(ipl); 962 /* check for hardware scheme */ 963 if (softvect > PSM_SV_SOFTWARE) { 964 setsoftint = mach_psm_set_softintr; 965 return (softvect); 966 } 967 968 if (softvect == PSM_SV_SOFTWARE) 969 setsoftint = av_set_softint_pending; 970 else /* hardware and software mixed scheme */ 971 setsoftint = mach_set_softintr; 972 973 return (PSM_SV_SOFTWARE); 974 } 975 976 static void 977 mach_set_softintr(register int ipl, struct av_softinfo *pending) 978 { 979 register struct psm_ops *pops; 980 981 /* set software pending bits */ 982 av_set_softint_pending(ipl, pending); 983 984 /* check if dosoftint will be called at the end of intr */ 985 if (CPU_ON_INTR(CPU) || (curthread->t_intr)) 986 return; 987 988 /* invoke hardware interrupt */ 989 pops = mach_set[0]; 990 (*pops->psm_set_softintr)(ipl); 991 } 992 993 static void 994 mach_cpu_start(register int cpun) 995 { 996 register struct psm_ops *pops; 997 int i; 998 999 pops = mach_set[0]; 1000 1001 (*pops->psm_cpu_start)(cpun, rm_platter_va); 1002 1003 /* wait for the auxillary cpu to be ready */ 1004 for (i = 20000; i; i--) { 1005 if (cpu[cpun]->cpu_flags & CPU_READY) 1006 return; 1007 drv_usecwait(100); 1008 } 1009 } 1010 1011 /*ARGSUSED*/ 1012 static int 1013 mach_translate_irq(dev_info_t *dip, int irqno) 1014 { 1015 return (irqno); /* default to NO translation */ 1016 } 1017 1018 static timestruc_t 1019 mach_tod_get(void) 1020 { 1021 timestruc_t ts; 1022 todinfo_t tod; 1023 static int mach_range_warn = 1; /* warn only once */ 1024 1025 ASSERT(MUTEX_HELD(&tod_lock)); 1026 1027 /* The year returned from is the last 2 digit only */ 1028 if ((*psm_todgetf)(&tod)) { 1029 ts.tv_sec = 0; 1030 ts.tv_nsec = 0; 1031 tod_fault_reset(); 1032 return (ts); 1033 } 1034 1035 /* assume that we wrap the rtc year back to zero at 2000 */ 1036 if (tod.tod_year < 69) { 1037 if (mach_range_warn && tod.tod_year > 38) { 1038 cmn_err(CE_WARN, "hardware real-time clock is out " 1039 "of range -- time needs to be reset"); 1040 mach_range_warn = 0; 1041 } 1042 tod.tod_year += 100; 1043 } 1044 1045 /* tod_to_utc uses 1900 as base for the year */ 1046 ts.tv_sec = tod_to_utc(tod) + gmt_lag; 1047 ts.tv_nsec = 0; 1048 1049 return (ts); 1050 } 1051 1052 static void 1053 mach_tod_set(timestruc_t ts) 1054 { 1055 todinfo_t tod = utc_to_tod(ts.tv_sec - gmt_lag); 1056 1057 ASSERT(MUTEX_HELD(&tod_lock)); 1058 1059 if (tod.tod_year >= 100) 1060 tod.tod_year -= 100; 1061 1062 (*psm_todsetf)(&tod); 1063 } 1064 1065 static void 1066 mach_notify_error(int level, char *errmsg) 1067 { 1068 /* 1069 * SL_FATAL is pass in once panicstr is set, deliver it 1070 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1071 * codes for the psmi handler 1072 */ 1073 if (level & SL_FATAL) 1074 (*notify_error)(CE_PANIC, errmsg); 1075 else if (level & SL_WARN) 1076 (*notify_error)(CE_WARN, errmsg); 1077 else if (level & SL_NOTE) 1078 (*notify_error)(CE_NOTE, errmsg); 1079 else if (level & SL_CONSOLE) 1080 (*notify_error)(CE_CONT, errmsg); 1081 } 1082 1083 /* 1084 * It provides the default basic intr_ops interface for the new DDI 1085 * interrupt framework if the PSM doesn't have one. 1086 * 1087 * Input: 1088 * dip - pointer to the dev_info structure of the requested device 1089 * hdlp - pointer to the internal interrupt handle structure for the 1090 * requested interrupt 1091 * intr_op - opcode for this call 1092 * result - pointer to the integer that will hold the result to be 1093 * passed back if return value is PSM_SUCCESS 1094 * 1095 * Output: 1096 * return value is either PSM_SUCCESS or PSM_FAILURE 1097 */ 1098 static int 1099 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1100 psm_intr_op_t intr_op, int *result) 1101 { 1102 struct intrspec *ispec; 1103 1104 switch (intr_op) { 1105 case PSM_INTR_OP_CHECK_MSI: 1106 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1107 DDI_INTR_TYPE_MSIX); 1108 break; 1109 case PSM_INTR_OP_ALLOC_VECTORS: 1110 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1111 *result = 1; 1112 else 1113 *result = 0; 1114 break; 1115 case PSM_INTR_OP_FREE_VECTORS: 1116 break; 1117 case PSM_INTR_OP_NAVAIL_VECTORS: 1118 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1119 *result = 1; 1120 else 1121 *result = 0; 1122 break; 1123 case PSM_INTR_OP_XLATE_VECTOR: 1124 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1125 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1126 break; 1127 case PSM_INTR_OP_GET_CAP: 1128 *result = 0; 1129 break; 1130 case PSM_INTR_OP_GET_PENDING: 1131 case PSM_INTR_OP_CLEAR_MASK: 1132 case PSM_INTR_OP_SET_MASK: 1133 case PSM_INTR_OP_GET_SHARED: 1134 case PSM_INTR_OP_SET_PRI: 1135 case PSM_INTR_OP_SET_CAP: 1136 case PSM_INTR_OP_SET_CPU: 1137 case PSM_INTR_OP_GET_INTR: 1138 default: 1139 return (PSM_FAILURE); 1140 } 1141 return (PSM_SUCCESS); 1142 } 1143