1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/thread.h> 30 #include <sys/cpuvar.h> 31 #include <sys/t_lock.h> 32 #include <sys/param.h> 33 #include <sys/proc.h> 34 #include <sys/disp.h> 35 #include <sys/mmu.h> 36 #include <sys/class.h> 37 #include <sys/cmn_err.h> 38 #include <sys/debug.h> 39 #include <sys/asm_linkage.h> 40 #include <sys/x_call.h> 41 #include <sys/systm.h> 42 #include <sys/var.h> 43 #include <sys/vtrace.h> 44 #include <vm/hat.h> 45 #include <sys/mmu.h> 46 #include <vm/as.h> 47 #include <vm/seg_kmem.h> 48 #include <sys/segments.h> 49 #include <sys/kmem.h> 50 #include <sys/stack.h> 51 #include <sys/smp_impldefs.h> 52 #include <sys/x86_archext.h> 53 #include <sys/machsystm.h> 54 #include <sys/traptrace.h> 55 #include <sys/clock.h> 56 #include <sys/cpc_impl.h> 57 #include <sys/chip.h> 58 #include <sys/dtrace.h> 59 #include <sys/archsystm.h> 60 #include <sys/fp.h> 61 #include <sys/reboot.h> 62 #include <sys/kdi.h> 63 #include <vm/hat_i86.h> 64 #include <sys/memnode.h> 65 #include <sys/pci_cfgspace.h> 66 #include <sys/cpu_module.h> 67 68 struct cpu cpus[1]; /* CPU data */ 69 struct cpu *cpu[NCPU] = {&cpus[0]}; /* pointers to all CPUs */ 70 cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 71 72 /* 73 * Useful for disabling MP bring-up for an MP capable kernel 74 * (a kernel that was built with MP defined) 75 */ 76 int use_mp = 1; 77 78 int mp_cpus = 0x1; /* to be set by platform specific module */ 79 80 /* 81 * This variable is used by the hat layer to decide whether or not 82 * critical sections are needed to prevent race conditions. For sun4m, 83 * this variable is set once enough MP initialization has been done in 84 * order to allow cross calls. 85 */ 86 int flushes_require_xcalls = 0; 87 ulong_t cpu_ready_set = 1; 88 89 extern void real_mode_start(void); 90 extern void real_mode_end(void); 91 static void mp_startup(void); 92 93 static void cpu_sep_enable(void); 94 static void cpu_sep_disable(void); 95 static void cpu_asysc_enable(void); 96 static void cpu_asysc_disable(void); 97 98 extern int tsc_gethrtime_enable; 99 100 /* 101 * Init CPU info - get CPU type info for processor_info system call. 102 */ 103 void 104 init_cpu_info(struct cpu *cp) 105 { 106 processor_info_t *pi = &cp->cpu_type_info; 107 char buf[CPU_IDSTRLEN]; 108 109 /* 110 * Get clock-frequency property for the CPU. 111 */ 112 pi->pi_clock = cpu_freq; 113 114 (void) strcpy(pi->pi_processor_type, "i386"); 115 if (fpu_exists) 116 (void) strcpy(pi->pi_fputypes, "i387 compatible"); 117 118 (void) cpuid_getidstr(cp, buf, sizeof (buf)); 119 120 cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 121 (void) strcpy(cp->cpu_idstr, buf); 122 123 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); 124 125 (void) cpuid_getbrandstr(cp, buf, sizeof (buf)); 126 cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 127 (void) strcpy(cp->cpu_brandstr, buf); 128 129 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); 130 } 131 132 /* 133 * Configure syscall support on this CPU. 134 */ 135 /*ARGSUSED*/ 136 static void 137 init_cpu_syscall(struct cpu *cp) 138 { 139 kpreempt_disable(); 140 141 #if defined(__amd64) 142 if (x86_feature & X86_ASYSC) { 143 144 #if !defined(__lint) 145 /* 146 * The syscall instruction imposes a certain ordering on 147 * segment selectors, so we double-check that ordering 148 * here. 149 */ 150 ASSERT(KDS_SEL == KCS_SEL + 8); 151 ASSERT(UDS_SEL == U32CS_SEL + 8); 152 ASSERT(UCS_SEL == U32CS_SEL + 16); 153 #endif 154 /* 155 * Turn syscall/sysret extensions on. 156 */ 157 cpu_asysc_enable(); 158 159 /* 160 * Program the magic registers .. 161 */ 162 wrmsr(MSR_AMD_STAR, ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 163 32); 164 wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall); 165 wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32); 166 167 /* 168 * This list of flags is masked off the incoming 169 * %rfl when we enter the kernel. 170 */ 171 wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T)); 172 } 173 #endif 174 175 /* 176 * On 32-bit kernels, we use sysenter/sysexit because it's too 177 * hard to use syscall/sysret, and it is more portable anyway. 178 * 179 * On 64-bit kernels on Nocona machines, the 32-bit syscall 180 * variant isn't available to 32-bit applications, but sysenter is. 181 */ 182 if (x86_feature & X86_SEP) { 183 184 #if !defined(__lint) 185 /* 186 * The sysenter instruction imposes a certain ordering on 187 * segment selectors, so we double-check that ordering 188 * here. See "sysenter" in Intel document 245471-012, "IA-32 189 * Intel Architecture Software Developer's Manual Volume 2: 190 * Instruction Set Reference" 191 */ 192 ASSERT(KDS_SEL == KCS_SEL + 8); 193 194 ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3)); 195 ASSERT32(UDS_SEL == UCS_SEL + 8); 196 197 ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3)); 198 ASSERT64(UDS_SEL == U32CS_SEL + 8); 199 #endif 200 201 cpu_sep_enable(); 202 203 /* 204 * resume() sets this value to the base of the threads stack 205 * via a context handler. 206 */ 207 wrmsr(MSR_INTC_SEP_ESP, 0ULL); 208 wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter); 209 } 210 211 kpreempt_enable(); 212 } 213 214 /* 215 * Multiprocessor initialization. 216 * 217 * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the 218 * startup and idle threads for the specified CPU. 219 */ 220 static void 221 mp_startup_init(int cpun) 222 { 223 #if defined(__amd64) 224 extern void *long_mode_64(void); 225 #endif /* __amd64 */ 226 227 struct cpu *cp; 228 struct tss *ntss; 229 kthread_id_t tp; 230 caddr_t sp; 231 int size; 232 proc_t *procp; 233 extern void idle(); 234 235 struct cpu_tables *tablesp; 236 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 237 238 #ifdef TRAPTRACE 239 trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; 240 #endif 241 242 ASSERT(cpun < NCPU && cpu[cpun] == NULL); 243 244 if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) { 245 panic("mp_startup_init: cpu%d: " 246 "no memory for cpu structure", cpun); 247 /*NOTREACHED*/ 248 } 249 procp = curthread->t_procp; 250 251 mutex_enter(&cpu_lock); 252 /* 253 * Initialize the dispatcher first. 254 */ 255 disp_cpu_init(cp); 256 mutex_exit(&cpu_lock); 257 258 cpu_vm_data_init(cp); 259 260 /* 261 * Allocate and initialize the startup thread for this CPU. 262 * Interrupt and process switch stacks get allocated later 263 * when the CPU starts running. 264 */ 265 tp = thread_create(NULL, 0, NULL, NULL, 0, procp, 266 TS_STOPPED, maxclsyspri); 267 268 /* 269 * Set state to TS_ONPROC since this thread will start running 270 * as soon as the CPU comes online. 271 * 272 * All the other fields of the thread structure are setup by 273 * thread_create(). 274 */ 275 THREAD_ONPROC(tp, cp); 276 tp->t_preempt = 1; 277 tp->t_bound_cpu = cp; 278 tp->t_affinitycnt = 1; 279 tp->t_cpu = cp; 280 tp->t_disp_queue = cp->cpu_disp; 281 282 /* 283 * Setup thread to start in mp_startup. 284 */ 285 sp = tp->t_stk; 286 tp->t_pc = (uintptr_t)mp_startup; 287 tp->t_sp = (uintptr_t)(sp - MINFRAME); 288 289 cp->cpu_id = cpun; 290 cp->cpu_self = cp; 291 cp->cpu_mask = 1 << cpun; 292 cp->cpu_thread = tp; 293 cp->cpu_lwp = NULL; 294 cp->cpu_dispthread = tp; 295 cp->cpu_dispatch_pri = DISP_PRIO(tp); 296 297 /* 298 * cpu_base_spl must be set explicitly here to prevent any blocking 299 * operations in mp_startup from causing the spl of the cpu to drop 300 * to 0 (allowing device interrupts before we're ready) in resume(). 301 * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY. 302 * As an extra bit of security on DEBUG kernels, this is enforced with 303 * an assertion in mp_startup() -- before cpu_base_spl is set to its 304 * proper value. 305 */ 306 cp->cpu_base_spl = ipltospl(LOCK_LEVEL); 307 308 /* 309 * Now, initialize per-CPU idle thread for this CPU. 310 */ 311 tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); 312 313 cp->cpu_idle_thread = tp; 314 315 tp->t_preempt = 1; 316 tp->t_bound_cpu = cp; 317 tp->t_affinitycnt = 1; 318 tp->t_cpu = cp; 319 tp->t_disp_queue = cp->cpu_disp; 320 321 /* 322 * Bootstrap the CPU for CMT aware scheduling 323 * The rest of the initialization will happen from 324 * mp_startup() 325 */ 326 chip_bootstrap_cpu(cp); 327 328 /* 329 * Perform CPC intialization on the new CPU. 330 */ 331 kcpc_hw_init(cp); 332 333 /* 334 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 335 * for each CPU. 336 */ 337 338 setup_vaddr_for_ppcopy(cp); 339 340 /* 341 * Allocate space for page directory, stack, tss, gdt and idt. 342 * This assumes that kmem_alloc will return memory which is aligned 343 * to the next higher power of 2 or a page(if size > MAXABIG) 344 * If this assumption goes wrong at any time due to change in 345 * kmem alloc, things may not work as the page directory has to be 346 * page aligned 347 */ 348 if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL) 349 panic("mp_startup_init: cpu%d cannot allocate tables", cpun); 350 351 if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) { 352 kmem_free(tablesp, sizeof (struct cpu_tables)); 353 size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE; 354 tablesp = kmem_zalloc(size, KM_NOSLEEP); 355 tablesp = (struct cpu_tables *) 356 (((uintptr_t)tablesp + MMU_STD_PAGESIZE) & 357 MMU_STD_PAGEMASK); 358 } 359 360 ntss = cp->cpu_tss = &tablesp->ct_tss; 361 cp->cpu_gdt = tablesp->ct_gdt; 362 bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t))); 363 364 #if defined(__amd64) 365 366 /* 367 * #DF (double fault). 368 */ 369 ntss->tss_ist1 = 370 (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 371 372 #elif defined(__i386) 373 374 ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp = 375 (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 376 377 ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL; 378 379 ntss->tss_eip = (uint32_t)mp_startup; 380 381 ntss->tss_cs = KCS_SEL; 382 ntss->tss_fs = KFS_SEL; 383 ntss->tss_gs = KGS_SEL; 384 385 /* 386 * setup kernel %gs. 387 */ 388 set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA, 389 SEL_KPL, 0, 1); 390 391 #endif /* __i386 */ 392 393 /* 394 * Set I/O bit map offset equal to size of TSS segment limit 395 * for no I/O permission map. This will cause all user I/O 396 * instructions to generate #gp fault. 397 */ 398 ntss->tss_bitmapbase = sizeof (*ntss); 399 400 /* 401 * setup kernel tss. 402 */ 403 set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, 404 sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL); 405 406 /* 407 * If we have more than one node, each cpu gets a copy of IDT 408 * local to its node. If this is a Pentium box, we use cpu 0's 409 * IDT. cpu 0's IDT has been made read-only to workaround the 410 * cmpxchgl register bug 411 */ 412 cp->cpu_idt = CPU->cpu_idt; 413 if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) { 414 cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP); 415 bcopy(idt0, cp->cpu_idt, sizeof (idt0)); 416 } 417 418 /* 419 * Get interrupt priority data from cpu 0 420 */ 421 cp->cpu_pri_data = CPU->cpu_pri_data; 422 423 hat_cpu_online(cp); 424 425 /* Should remove all entries for the current process/thread here */ 426 427 /* 428 * Fill up the real mode platter to make it easy for real mode code to 429 * kick it off. This area should really be one passed by boot to kernel 430 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 431 * have identical physical and virtual address in paged mode. 432 */ 433 real_mode_platter->rm_idt_base = cp->cpu_idt; 434 real_mode_platter->rm_idt_lim = sizeof (idt0) - 1; 435 real_mode_platter->rm_gdt_base = cp->cpu_gdt; 436 real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1; 437 real_mode_platter->rm_pdbr = getcr3(); 438 real_mode_platter->rm_cpu = cpun; 439 real_mode_platter->rm_x86feature = x86_feature; 440 real_mode_platter->rm_cr4 = cr4_value; 441 442 #if defined(__amd64) 443 if (getcr3() > 0xffffffffUL) 444 panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 445 "located above 4G in physical memory (@ 0x%llx).", 446 (unsigned long long)getcr3()); 447 448 /* 449 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 450 * by code in real_mode_start(): 451 * 452 * GDT[0]: NULL selector 453 * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 454 * 455 * Clear the IDT as interrupts will be off and a limit of 0 will cause 456 * the CPU to triple fault and reset on an NMI, seemingly as reasonable 457 * a course of action as any other, though it may cause the entire 458 * platform to reset in some cases... 459 */ 460 real_mode_platter->rm_temp_gdt[0] = 0ULL; 461 real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 462 463 real_mode_platter->rm_temp_gdt_lim = (ushort_t) 464 (sizeof (real_mode_platter->rm_temp_gdt) - 1); 465 real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 466 (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt); 467 468 real_mode_platter->rm_temp_idt_lim = 0; 469 real_mode_platter->rm_temp_idt_base = 0; 470 471 /* 472 * Since the CPU needs to jump to protected mode using an identity 473 * mapped address, we need to calculate it here. 474 */ 475 real_mode_platter->rm_longmode64_addr = rm_platter_pa + 476 ((uint32_t)long_mode_64 - (uint32_t)real_mode_start); 477 #endif /* __amd64 */ 478 479 #ifdef TRAPTRACE 480 /* 481 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this 482 * CPU. 483 */ 484 ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); 485 ttc->ttc_next = ttc->ttc_first; 486 ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; 487 #endif 488 489 /* 490 * Record that we have another CPU. 491 */ 492 mutex_enter(&cpu_lock); 493 /* 494 * Initialize the interrupt threads for this CPU 495 */ 496 cpu_intr_alloc(cp, NINTR_THREADS); 497 /* 498 * Add CPU to list of available CPUs. It'll be on the active list 499 * after mp_startup(). 500 */ 501 cpu_add_unit(cp); 502 mutex_exit(&cpu_lock); 503 } 504 505 /* 506 * Apply workarounds for known errata, and warn about those that are absent. 507 * 508 * System vendors occasionally create configurations which contain different 509 * revisions of the CPUs that are almost but not exactly the same. At the 510 * time of writing, this meant that their clock rates were the same, their 511 * feature sets were the same, but the required workaround were -not- 512 * necessarily the same. So, this routine is invoked on -every- CPU soon 513 * after starting to make sure that the resulting system contains the most 514 * pessimal set of workarounds needed to cope with *any* of the CPUs in the 515 * system. 516 * 517 * workaround_errata is invoked early in mlsetup() for CPU 0, and in 518 * mp_startup() for all slave CPUs. Slaves process workaround_errata prior 519 * to acknowledging their readiness to the master, so this routine will 520 * never be executed by multiple CPUs in parallel, thus making updates to 521 * global data safe. 522 * 523 * These workarounds are based on Rev 3.57 of the Revision Guide for 524 * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005. 525 */ 526 527 #if defined(OPTERON_ERRATUM_91) 528 int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ 529 #endif 530 531 #if defined(OPTERON_ERRATUM_93) 532 int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ 533 #endif 534 535 #if defined(OPTERON_ERRATUM_100) 536 int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ 537 #endif 538 539 #if defined(OPTERON_ERRATUM_109) 540 int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ 541 #endif 542 543 #if defined(OPTERON_ERRATUM_121) 544 int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ 545 #endif 546 547 #if defined(OPTERON_ERRATUM_122) 548 int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ 549 #endif 550 551 #if defined(OPTERON_ERRATUM_123) 552 int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ 553 #endif 554 555 #if defined(OPTERON_ERRATUM_131) 556 int opteron_erratum_131; /* if non-zero -> at least one cpu has it */ 557 #endif 558 559 #if defined(OPTERON_WORKAROUND_6336786) 560 int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */ 561 int opteron_workaround_6336786_UP = 0; /* Not needed for UP */ 562 #endif 563 564 #define WARNING(cpu, n) \ 565 cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d", \ 566 (cpu)->cpu_id, (n)) 567 568 uint_t 569 workaround_errata(struct cpu *cpu) 570 { 571 uint_t missing = 0; 572 573 ASSERT(cpu == CPU); 574 575 /*LINTED*/ 576 if (cpuid_opteron_erratum(cpu, 88) > 0) { 577 /* 578 * SWAPGS May Fail To Read Correct GS Base 579 */ 580 #if defined(OPTERON_ERRATUM_88) 581 /* 582 * The workaround is an mfence in the relevant assembler code 583 */ 584 #else 585 WARNING(cpu, 88); 586 missing++; 587 #endif 588 } 589 590 if (cpuid_opteron_erratum(cpu, 91) > 0) { 591 /* 592 * Software Prefetches May Report A Page Fault 593 */ 594 #if defined(OPTERON_ERRATUM_91) 595 /* 596 * fix is in trap.c 597 */ 598 opteron_erratum_91++; 599 #else 600 WARNING(cpu, 91); 601 missing++; 602 #endif 603 } 604 605 if (cpuid_opteron_erratum(cpu, 93) > 0) { 606 /* 607 * RSM Auto-Halt Restart Returns to Incorrect RIP 608 */ 609 #if defined(OPTERON_ERRATUM_93) 610 /* 611 * fix is in trap.c 612 */ 613 opteron_erratum_93++; 614 #else 615 WARNING(cpu, 93); 616 missing++; 617 #endif 618 } 619 620 /*LINTED*/ 621 if (cpuid_opteron_erratum(cpu, 95) > 0) { 622 /* 623 * RET Instruction May Return to Incorrect EIP 624 */ 625 #if defined(OPTERON_ERRATUM_95) 626 #if defined(_LP64) 627 /* 628 * Workaround this by ensuring that 32-bit user code and 629 * 64-bit kernel code never occupy the same address 630 * range mod 4G. 631 */ 632 if (_userlimit32 > 0xc0000000ul) 633 *(uintptr_t *)&_userlimit32 = 0xc0000000ul; 634 635 /*LINTED*/ 636 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); 637 #endif /* _LP64 */ 638 #else 639 WARNING(cpu, 95); 640 missing++; 641 #endif /* OPTERON_ERRATUM_95 */ 642 } 643 644 if (cpuid_opteron_erratum(cpu, 100) > 0) { 645 /* 646 * Compatibility Mode Branches Transfer to Illegal Address 647 */ 648 #if defined(OPTERON_ERRATUM_100) 649 /* 650 * fix is in trap.c 651 */ 652 opteron_erratum_100++; 653 #else 654 WARNING(cpu, 100); 655 missing++; 656 #endif 657 } 658 659 /*LINTED*/ 660 if (cpuid_opteron_erratum(cpu, 108) > 0) { 661 /* 662 * CPUID Instruction May Return Incorrect Model Number In 663 * Some Processors 664 */ 665 #if defined(OPTERON_ERRATUM_108) 666 /* 667 * (Our cpuid-handling code corrects the model number on 668 * those processors) 669 */ 670 #else 671 WARNING(cpu, 108); 672 missing++; 673 #endif 674 } 675 676 /*LINTED*/ 677 if (cpuid_opteron_erratum(cpu, 109) > 0) { 678 /* 679 * Certain Reverse REP MOVS May Produce Unpredictable Behaviour 680 */ 681 #if defined(OPTERON_ERRATUM_109) 682 683 /* workaround is to print a warning to upgrade BIOS */ 684 if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 685 opteron_erratum_109++; 686 #else 687 WARNING(cpu, 109); 688 missing++; 689 #endif 690 } 691 /*LINTED*/ 692 if (cpuid_opteron_erratum(cpu, 121) > 0) { 693 /* 694 * Sequential Execution Across Non_Canonical Boundary Caused 695 * Processor Hang 696 */ 697 #if defined(OPTERON_ERRATUM_121) 698 static int lma; 699 700 if (opteron_erratum_121) 701 opteron_erratum_121++; 702 703 /* 704 * Erratum 121 is only present in long (64 bit) mode. 705 * Workaround is to include the page immediately before the 706 * va hole to eliminate the possibility of system hangs due to 707 * sequential execution across the va hole boundary. 708 */ 709 if (lma == 0) { 710 /* 711 * check LMA once: assume all cpus are in long mode 712 * or not. 713 */ 714 lma = 1; 715 716 if (rdmsr(MSR_AMD_EFER) & AMD_EFER_LMA) { 717 if (hole_start) { 718 hole_start -= PAGESIZE; 719 } else { 720 /* 721 * hole_start not yet initialized by 722 * mmu_init. Initialize hole_start 723 * with value to be subtracted. 724 */ 725 hole_start = PAGESIZE; 726 } 727 opteron_erratum_121++; 728 } 729 } 730 #else 731 WARNING(cpu, 121); 732 missing++; 733 #endif 734 } 735 736 /*LINTED*/ 737 if (cpuid_opteron_erratum(cpu, 122) > 0) { 738 /* 739 * TLB Flush Filter May Cause Cohenrency Problem in 740 * Multiprocessor Systems 741 */ 742 #if defined(OPTERON_ERRATUM_122) 743 /* 744 * Erratum 122 is only present in MP configurations (multi-core 745 * or multi-processor). 746 */ 747 748 if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 || 749 cpuid_get_ncpu_per_chip(cpu) > 1) { 750 /* disable TLB Flush Filter */ 751 wrmsr(MSR_AMD_HWCR, rdmsr(MSR_AMD_HWCR) | 752 (uint64_t)(uintptr_t)AMD_HWCR_FFDIS); 753 opteron_erratum_122++; 754 } 755 756 #else 757 WARNING(cpu, 122); 758 missing++; 759 #endif 760 } 761 762 #if defined(OPTERON_ERRATUM_123) 763 /*LINTED*/ 764 if (cpuid_opteron_erratum(cpu, 123) > 0) { 765 /* 766 * Bypassed Reads May Cause Data Corruption of System Hang in 767 * Dual Core Processors 768 */ 769 /* 770 * Erratum 123 applies only to multi-core cpus. 771 */ 772 773 if (cpuid_get_ncpu_per_chip(cpu) > 1) { 774 /* workaround is to print a warning to upgrade BIOS */ 775 if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 776 opteron_erratum_123++; 777 } 778 } 779 #endif 780 781 #if defined(OPTERON_ERRATUM_131) 782 /*LINTED*/ 783 if (cpuid_opteron_erratum(cpu, 131) > 0) { 784 /* 785 * Multiprocessor Systems with Four or More Cores May Deadlock 786 * Waiting for a Probe Response 787 */ 788 /* 789 * Erratum 131 applies to any system with four or more cores. 790 */ 791 if ((opteron_erratum_131 == 0) && ((lgrp_plat_node_cnt * 792 cpuid_get_ncpu_per_chip(cpu)) >= 4)) { 793 /* 794 * Workaround is to print a warning to upgrade 795 * the BIOS 796 */ 797 if (!(rdmsr(MSR_AMD_NB_CFG) & AMD_NB_CFG_SRQ_HEARTBEAT)) 798 opteron_erratum_131++; 799 } 800 } 801 #endif 802 803 #if defined(OPTERON_WORKAROUND_6336786) 804 /* 805 * This isn't really erratum, but for convenience the 806 * detection/workaround code lives here and in cpuid_opteron_erratum. 807 */ 808 if (cpuid_opteron_erratum(cpu, 6336786) > 0) { 809 int node; 810 uint8_t data; 811 812 /* 813 * Disable C1-Clock ramping on multi-core/multi-processor 814 * K8 platforms to guard against TSC drift. 815 */ 816 if (opteron_workaround_6336786) { 817 opteron_workaround_6336786++; 818 } else if ((lgrp_plat_node_cnt * 819 cpuid_get_ncpu_per_chip(cpu) >= 2) || 820 opteron_workaround_6336786_UP) { 821 for (node = 0; node < lgrp_plat_node_cnt; node++) { 822 /* 823 * Clear PMM7[1:0] (function 3, offset 0x87) 824 * Northbridge device is the node id + 24. 825 */ 826 data = pci_getb_func(0, node + 24, 3, 0x87); 827 data &= 0xFC; 828 pci_putb_func(0, node + 24, 3, 0x87, data); 829 } 830 opteron_workaround_6336786++; 831 } 832 } 833 #endif 834 return (missing); 835 } 836 837 void 838 workaround_errata_end() 839 { 840 #if defined(OPTERON_ERRATUM_109) 841 if (opteron_erratum_109) { 842 cmn_err(CE_WARN, 843 "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 844 " processor\nerratum 109 was not detected; updating your" 845 " system's BIOS to a version\ncontaining this" 846 " microcode patch is HIGHLY recommended or erroneous" 847 " system\noperation may occur.\n"); 848 } 849 #endif /* OPTERON_ERRATUM_109 */ 850 #if defined(OPTERON_ERRATUM_123) 851 if (opteron_erratum_123) { 852 cmn_err(CE_WARN, 853 "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 854 " processor\nerratum 123 was not detected; updating your" 855 " system's BIOS to a version\ncontaining this" 856 " microcode patch is HIGHLY recommended or erroneous" 857 " system\noperation may occur.\n"); 858 } 859 #endif /* OPTERON_ERRATUM_123 */ 860 #if defined(OPTERON_ERRATUM_131) 861 if (opteron_erratum_131) { 862 cmn_err(CE_WARN, 863 "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 864 " processor\nerratum 131 was not detected; updating your" 865 " system's BIOS to a version\ncontaining this" 866 " microcode patch is HIGHLY recommended or erroneous" 867 " system\noperation may occur.\n"); 868 } 869 #endif /* OPTERON_ERRATUM_131 */ 870 } 871 872 static ushort_t *mp_map_warm_reset_vector(); 873 static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector); 874 875 /*ARGSUSED*/ 876 void 877 start_other_cpus(int cprboot) 878 { 879 unsigned who; 880 int cpuid = 0; 881 int delays = 0; 882 int started_cpu; 883 ushort_t *warm_reset_vector = NULL; 884 extern int procset; 885 886 /* 887 * Initialize our own cpu_info. 888 */ 889 init_cpu_info(CPU); 890 891 /* 892 * Initialize our syscall handlers 893 */ 894 init_cpu_syscall(CPU); 895 896 /* 897 * if only 1 cpu or not using MP, skip the rest of this 898 */ 899 if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) { 900 if (use_mp == 0) 901 cmn_err(CE_CONT, "?***** Not in MP mode\n"); 902 goto done; 903 } 904 905 /* 906 * perform such initialization as is needed 907 * to be able to take CPUs on- and off-line. 908 */ 909 cpu_pause_init(); 910 911 xc_init(); /* initialize processor crosscalls */ 912 913 /* 914 * Copy the real mode code at "real_mode_start" to the 915 * page at rm_platter_va. 916 */ 917 warm_reset_vector = mp_map_warm_reset_vector(); 918 if (warm_reset_vector == NULL) 919 goto done; 920 921 bcopy((caddr_t)real_mode_start, 922 (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code, 923 (size_t)real_mode_end - (size_t)real_mode_start); 924 925 flushes_require_xcalls = 1; 926 927 affinity_set(CPU_CURRENT); 928 929 for (who = 0; who < NCPU; who++) { 930 if (who == cpuid) 931 continue; 932 933 if ((mp_cpus & (1 << who)) == 0) 934 continue; 935 936 mp_startup_init(who); 937 started_cpu = 1; 938 (*cpu_startf)(who, rm_platter_pa); 939 940 while ((procset & (1 << who)) == 0) { 941 942 delay(1); 943 if (++delays > (20 * hz)) { 944 945 cmn_err(CE_WARN, 946 "cpu%d failed to start", who); 947 948 mutex_enter(&cpu_lock); 949 cpu[who]->cpu_flags = 0; 950 cpu_vm_data_destroy(cpu[who]); 951 cpu_del_unit(who); 952 mutex_exit(&cpu_lock); 953 954 started_cpu = 0; 955 break; 956 } 957 } 958 if (!started_cpu) 959 continue; 960 if (tsc_gethrtime_enable) 961 tsc_sync_master(who); 962 963 964 if (dtrace_cpu_init != NULL) { 965 /* 966 * DTrace CPU initialization expects cpu_lock 967 * to be held. 968 */ 969 mutex_enter(&cpu_lock); 970 (*dtrace_cpu_init)(who); 971 mutex_exit(&cpu_lock); 972 } 973 } 974 975 affinity_clear(); 976 977 for (who = 0; who < NCPU; who++) { 978 if (who == cpuid) 979 continue; 980 981 if (!(procset & (1 << who))) 982 continue; 983 984 while (!(cpu_ready_set & (1 << who))) 985 delay(1); 986 } 987 988 done: 989 workaround_errata_end(); 990 991 if (warm_reset_vector != NULL) 992 mp_unmap_warm_reset_vector(warm_reset_vector); 993 hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 994 HAT_UNLOAD); 995 } 996 997 /* 998 * Dummy functions - no i86pc platforms support dynamic cpu allocation. 999 */ 1000 /*ARGSUSED*/ 1001 int 1002 mp_cpu_configure(int cpuid) 1003 { 1004 return (ENOTSUP); /* not supported */ 1005 } 1006 1007 /*ARGSUSED*/ 1008 int 1009 mp_cpu_unconfigure(int cpuid) 1010 { 1011 return (ENOTSUP); /* not supported */ 1012 } 1013 1014 /* 1015 * Startup function for 'other' CPUs (besides boot cpu). 1016 * Called from real_mode_start (after *ap_mlsetup). 1017 * 1018 * WARNING: until CPU_READY is set, mp_startup and routines called by 1019 * mp_startup should not call routines (e.g. kmem_free) that could call 1020 * hat_unload which requires CPU_READY to be set. 1021 */ 1022 void 1023 mp_startup(void) 1024 { 1025 struct cpu *cp = CPU; 1026 extern int procset; 1027 uint_t new_x86_feature; 1028 1029 new_x86_feature = cpuid_pass1(cp); 1030 1031 /* 1032 * We need to Sync MTRR with cpu0's MTRR. We have to do 1033 * this with interrupts disabled. 1034 */ 1035 if (x86_feature & X86_MTRR) 1036 mtrr_sync(); 1037 1038 /* 1039 * Initialize this CPU's syscall handlers 1040 */ 1041 init_cpu_syscall(cp); 1042 1043 /* 1044 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the 1045 * highest level at which a routine is permitted to block on 1046 * an adaptive mutex (allows for cpu poke interrupt in case 1047 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks 1048 * device interrupts that may end up in the hat layer issuing cross 1049 * calls before CPU_READY is set. 1050 */ 1051 (void) splx(ipltospl(LOCK_LEVEL)); 1052 1053 /* 1054 * Do a sanity check to make sure this new CPU is a sane thing 1055 * to add to the collection of processors running this system. 1056 * 1057 * XXX Clearly this needs to get more sophisticated, if x86 1058 * systems start to get built out of heterogenous CPUs; as is 1059 * likely to happen once the number of processors in a configuration 1060 * gets large enough. 1061 */ 1062 if ((x86_feature & new_x86_feature) != x86_feature) { 1063 cmn_err(CE_CONT, "?cpu%d: %b\n", 1064 cp->cpu_id, new_x86_feature, FMT_X86_FEATURE); 1065 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); 1066 } 1067 1068 /* 1069 * We could be more sophisticated here, and just mark the CPU 1070 * as "faulted" but at this point we'll opt for the easier 1071 * answer of dieing horribly. Provided the boot cpu is ok, 1072 * the system can be recovered by booting with use_mp set to zero. 1073 */ 1074 if (workaround_errata(cp) != 0) 1075 panic("critical workaround(s) missing for cpu%d", cp->cpu_id); 1076 1077 cpuid_pass2(cp); 1078 cpuid_pass3(cp); 1079 (void) cpuid_pass4(cp); 1080 1081 init_cpu_info(cp); 1082 1083 mutex_enter(&cpu_lock); 1084 procset |= 1 << cp->cpu_id; 1085 mutex_exit(&cpu_lock); 1086 1087 if (tsc_gethrtime_enable) 1088 tsc_sync_slave(); 1089 1090 mutex_enter(&cpu_lock); 1091 /* 1092 * It's unfortunate that chip_cpu_init() has to be called here. 1093 * It really belongs in cpu_add_unit(), but unfortunately it is 1094 * dependent on the cpuid probing, which must be done in the 1095 * context of the current CPU. Care must be taken on x86 to ensure 1096 * that mp_startup can safely block even though chip_cpu_init() and 1097 * cpu_add_active() have not yet been called. 1098 */ 1099 chip_cpu_init(cp); 1100 chip_cpu_startup(cp); 1101 1102 cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS; 1103 cpu_add_active(cp); 1104 mutex_exit(&cpu_lock); 1105 1106 add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi); 1107 1108 /* The base spl should still be at LOCK LEVEL here */ 1109 ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL)); 1110 set_base_spl(); /* Restore the spl to its proper value */ 1111 1112 (void) spl0(); /* enable interrupts */ 1113 1114 /* 1115 * Set up the CPU module for this CPU. This can't be done before 1116 * this CPU is made CPU_READY, because we may (in heterogeneous systems) 1117 * need to go load another CPU module. The act of attempting to load 1118 * a module may trigger a cross-call, which will ASSERT unless this 1119 * cpu is CPU_READY. 1120 */ 1121 cmi_init(); 1122 1123 if (x86_feature & X86_MCA) 1124 cmi_mca_init(); 1125 1126 if (boothowto & RB_DEBUG) 1127 kdi_dvec_cpu_init(cp); 1128 1129 /* 1130 * Setting the bit in cpu_ready_set must be the last operation in 1131 * processor initialization; the boot CPU will continue to boot once 1132 * it sees this bit set for all active CPUs. 1133 */ 1134 CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); 1135 1136 /* 1137 * Because mp_startup() gets fired off after init() starts, we 1138 * can't use the '?' trick to do 'boot -v' printing - so we 1139 * always direct the 'cpu .. online' messages to the log. 1140 */ 1141 cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 1142 cp->cpu_id); 1143 1144 /* 1145 * Now we are done with the startup thread, so free it up. 1146 */ 1147 thread_exit(); 1148 panic("mp_startup: cannot return"); 1149 /*NOTREACHED*/ 1150 } 1151 1152 1153 /* 1154 * Start CPU on user request. 1155 */ 1156 /* ARGSUSED */ 1157 int 1158 mp_cpu_start(struct cpu *cp) 1159 { 1160 ASSERT(MUTEX_HELD(&cpu_lock)); 1161 return (0); 1162 } 1163 1164 /* 1165 * Stop CPU on user request. 1166 */ 1167 /* ARGSUSED */ 1168 int 1169 mp_cpu_stop(struct cpu *cp) 1170 { 1171 extern int cbe_psm_timer_mode; 1172 ASSERT(MUTEX_HELD(&cpu_lock)); 1173 1174 /* 1175 * If TIMER_PERIODIC mode is used, CPU0 is the one running it; 1176 * can't stop it. (This is true only for machines with no TSC.) 1177 */ 1178 1179 if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0)) 1180 return (1); 1181 1182 return (0); 1183 } 1184 1185 /* 1186 * Power on CPU. 1187 */ 1188 /* ARGSUSED */ 1189 int 1190 mp_cpu_poweron(struct cpu *cp) 1191 { 1192 ASSERT(MUTEX_HELD(&cpu_lock)); 1193 return (ENOTSUP); /* not supported */ 1194 } 1195 1196 /* 1197 * Power off CPU. 1198 */ 1199 /* ARGSUSED */ 1200 int 1201 mp_cpu_poweroff(struct cpu *cp) 1202 { 1203 ASSERT(MUTEX_HELD(&cpu_lock)); 1204 return (ENOTSUP); /* not supported */ 1205 } 1206 1207 1208 /* 1209 * Take the specified CPU out of participation in interrupts. 1210 */ 1211 int 1212 cpu_disable_intr(struct cpu *cp) 1213 { 1214 if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) 1215 return (EBUSY); 1216 1217 cp->cpu_flags &= ~CPU_ENABLE; 1218 return (0); 1219 } 1220 1221 /* 1222 * Allow the specified CPU to participate in interrupts. 1223 */ 1224 void 1225 cpu_enable_intr(struct cpu *cp) 1226 { 1227 ASSERT(MUTEX_HELD(&cpu_lock)); 1228 cp->cpu_flags |= CPU_ENABLE; 1229 psm_enable_intr(cp->cpu_id); 1230 } 1231 1232 1233 1234 static ushort_t * 1235 mp_map_warm_reset_vector() 1236 { 1237 ushort_t *warm_reset_vector; 1238 1239 if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 1240 sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 1241 return (NULL); 1242 1243 /* 1244 * setup secondary cpu bios boot up vector 1245 */ 1246 *warm_reset_vector = (ushort_t)((caddr_t) 1247 ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 1248 + ((ulong_t)rm_platter_va & 0xf)); 1249 warm_reset_vector++; 1250 *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 1251 1252 --warm_reset_vector; 1253 return (warm_reset_vector); 1254 } 1255 1256 static void 1257 mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector) 1258 { 1259 psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 1260 } 1261 1262 void 1263 mp_cpu_faulted_enter(struct cpu *cp) 1264 { 1265 cmi_faulted_enter(cp); 1266 } 1267 1268 void 1269 mp_cpu_faulted_exit(struct cpu *cp) 1270 { 1271 cmi_faulted_exit(cp); 1272 } 1273 1274 /* 1275 * The following two routines are used as context operators on threads belonging 1276 * to processes with a private LDT (see sysi86). Due to the rarity of such 1277 * processes, these routines are currently written for best code readability and 1278 * organization rather than speed. We could avoid checking x86_feature at every 1279 * context switch by installing different context ops, depending on the 1280 * x86_feature flags, at LDT creation time -- one for each combination of fast 1281 * syscall feature flags. 1282 */ 1283 1284 /*ARGSUSED*/ 1285 void 1286 cpu_fast_syscall_disable(void *arg) 1287 { 1288 if (x86_feature & X86_SEP) 1289 cpu_sep_disable(); 1290 if (x86_feature & X86_ASYSC) 1291 cpu_asysc_disable(); 1292 } 1293 1294 /*ARGSUSED*/ 1295 void 1296 cpu_fast_syscall_enable(void *arg) 1297 { 1298 if (x86_feature & X86_SEP) 1299 cpu_sep_enable(); 1300 if (x86_feature & X86_ASYSC) 1301 cpu_asysc_enable(); 1302 } 1303 1304 static void 1305 cpu_sep_enable(void) 1306 { 1307 ASSERT(x86_feature & X86_SEP); 1308 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1309 1310 wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL); 1311 } 1312 1313 static void 1314 cpu_sep_disable(void) 1315 { 1316 ASSERT(x86_feature & X86_SEP); 1317 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1318 1319 /* 1320 * Setting the SYSENTER_CS_MSR register to 0 causes software executing 1321 * the sysenter or sysexit instruction to trigger a #gp fault. 1322 */ 1323 wrmsr(MSR_INTC_SEP_CS, 0ULL); 1324 } 1325 1326 static void 1327 cpu_asysc_enable(void) 1328 { 1329 ASSERT(x86_feature & X86_ASYSC); 1330 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1331 1332 wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) | 1333 (uint64_t)(uintptr_t)AMD_EFER_SCE); 1334 } 1335 1336 static void 1337 cpu_asysc_disable(void) 1338 { 1339 ASSERT(x86_feature & X86_ASYSC); 1340 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1341 1342 /* 1343 * Turn off the SCE (syscall enable) bit in the EFER register. Software 1344 * executing syscall or sysret with this bit off will incur a #ud trap. 1345 */ 1346 wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) & 1347 ~((uint64_t)(uintptr_t)AMD_EFER_SCE)); 1348 } 1349