1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/thread.h> 31 #include <sys/cpuvar.h> 32 #include <sys/t_lock.h> 33 #include <sys/param.h> 34 #include <sys/proc.h> 35 #include <sys/disp.h> 36 #include <sys/mmu.h> 37 #include <sys/class.h> 38 #include <sys/cmn_err.h> 39 #include <sys/debug.h> 40 #include <sys/asm_linkage.h> 41 #include <sys/x_call.h> 42 #include <sys/systm.h> 43 #include <sys/var.h> 44 #include <sys/vtrace.h> 45 #include <vm/hat.h> 46 #include <sys/mmu.h> 47 #include <vm/as.h> 48 #include <vm/seg_kmem.h> 49 #include <sys/segments.h> 50 #include <sys/kmem.h> 51 #include <sys/stack.h> 52 #include <sys/smp_impldefs.h> 53 #include <sys/x86_archext.h> 54 #include <sys/machsystm.h> 55 #include <sys/traptrace.h> 56 #include <sys/clock.h> 57 #include <sys/cpc_impl.h> 58 #include <sys/chip.h> 59 #include <sys/dtrace.h> 60 #include <sys/archsystm.h> 61 #include <sys/fp.h> 62 #include <sys/reboot.h> 63 #include <sys/kdi.h> 64 #include <vm/hat_i86.h> 65 #include <sys/memnode.h> 66 67 struct cpu cpus[1]; /* CPU data */ 68 struct cpu *cpu[NCPU] = {&cpus[0]}; /* pointers to all CPUs */ 69 cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 70 71 /* 72 * Useful for disabling MP bring-up for an MP capable kernel 73 * (a kernel that was built with MP defined) 74 */ 75 int use_mp = 1; 76 77 int mp_cpus = 0x1; /* to be set by platform specific module */ 78 79 /* 80 * This variable is used by the hat layer to decide whether or not 81 * critical sections are needed to prevent race conditions. For sun4m, 82 * this variable is set once enough MP initialization has been done in 83 * order to allow cross calls. 84 */ 85 int flushes_require_xcalls = 0; 86 ulong_t cpu_ready_set = 1; 87 88 extern void real_mode_start(void); 89 extern void real_mode_end(void); 90 static void mp_startup(void); 91 92 static void cpu_sep_enable(void); 93 static void cpu_sep_disable(void); 94 static void cpu_asysc_enable(void); 95 static void cpu_asysc_disable(void); 96 97 extern int tsc_gethrtime_enable; 98 99 /* 100 * Init CPU info - get CPU type info for processor_info system call. 101 */ 102 void 103 init_cpu_info(struct cpu *cp) 104 { 105 processor_info_t *pi = &cp->cpu_type_info; 106 char buf[CPU_IDSTRLEN]; 107 108 /* 109 * Get clock-frequency property for the CPU. 110 */ 111 pi->pi_clock = cpu_freq; 112 113 (void) strcpy(pi->pi_processor_type, "i386"); 114 if (fpu_exists) 115 (void) strcpy(pi->pi_fputypes, "i387 compatible"); 116 117 (void) cpuid_getidstr(cp, buf, sizeof (buf)); 118 119 cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 120 (void) strcpy(cp->cpu_idstr, buf); 121 122 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); 123 124 (void) cpuid_getbrandstr(cp, buf, sizeof (buf)); 125 cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 126 (void) strcpy(cp->cpu_brandstr, buf); 127 128 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); 129 } 130 131 /* 132 * Configure syscall support on this CPU. 133 */ 134 /*ARGSUSED*/ 135 static void 136 init_cpu_syscall(struct cpu *cp) 137 { 138 uint64_t value; 139 140 kpreempt_disable(); 141 142 #if defined(__amd64) 143 if (x86_feature & X86_ASYSC) { 144 145 #if !defined(__lint) 146 /* 147 * The syscall instruction imposes a certain ordering on 148 * segment selectors, so we double-check that ordering 149 * here. 150 */ 151 ASSERT(KDS_SEL == KCS_SEL + 8); 152 ASSERT(UDS_SEL == U32CS_SEL + 8); 153 ASSERT(UCS_SEL == U32CS_SEL + 16); 154 #endif 155 /* 156 * Turn syscall/sysret extensions on. 157 */ 158 cpu_asysc_enable(); 159 160 /* 161 * Program the magic registers .. 162 */ 163 value = ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32; 164 wrmsr(MSR_AMD_STAR, &value); 165 value = (uintptr_t)sys_syscall; 166 wrmsr(MSR_AMD_LSTAR, &value); 167 value = (uintptr_t)sys_syscall32; 168 wrmsr(MSR_AMD_CSTAR, &value); 169 170 /* 171 * This list of flags is masked off the incoming 172 * %rfl when we enter the kernel. 173 */ 174 value = PS_IE | PS_T; 175 wrmsr(MSR_AMD_SFMASK, &value); 176 } 177 #endif 178 179 /* 180 * On 32-bit kernels, we use sysenter/sysexit because it's too 181 * hard to use syscall/sysret, and it is more portable anyway. 182 * 183 * On 64-bit kernels on Nocona machines, the 32-bit syscall 184 * variant isn't available to 32-bit applications, but sysenter is. 185 */ 186 if (x86_feature & X86_SEP) { 187 188 #if !defined(__lint) 189 /* 190 * The sysenter instruction imposes a certain ordering on 191 * segment selectors, so we double-check that ordering 192 * here. See "sysenter" in Intel document 245471-012, "IA-32 193 * Intel Architecture Software Developer's Manual Volume 2: 194 * Instruction Set Reference" 195 */ 196 ASSERT(KDS_SEL == KCS_SEL + 8); 197 198 ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3)); 199 ASSERT32(UDS_SEL == UCS_SEL + 8); 200 201 ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3)); 202 ASSERT64(UDS_SEL == U32CS_SEL + 8); 203 #endif 204 205 cpu_sep_enable(); 206 207 /* 208 * resume() sets this value to the base of the threads stack 209 * via a context handler. 210 */ 211 value = 0; 212 wrmsr(MSR_INTC_SEP_ESP, &value); 213 214 value = (uintptr_t)sys_sysenter; 215 wrmsr(MSR_INTC_SEP_EIP, &value); 216 } 217 218 kpreempt_enable(); 219 } 220 221 /* 222 * Multiprocessor initialization. 223 * 224 * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the 225 * startup and idle threads for the specified CPU. 226 */ 227 static void 228 mp_startup_init(int cpun) 229 { 230 #if defined(__amd64) 231 extern void *long_mode_64(void); 232 #endif /* __amd64 */ 233 234 struct cpu *cp; 235 struct tss *ntss; 236 kthread_id_t tp; 237 caddr_t sp; 238 int size; 239 proc_t *procp; 240 extern void idle(); 241 extern void init_intr_threads(struct cpu *); 242 243 struct cpu_tables *tablesp; 244 extern chip_t cpu0_chip; 245 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 246 247 #ifdef TRAPTRACE 248 trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; 249 #endif 250 251 ASSERT(cpun < NCPU && cpu[cpun] == NULL); 252 253 if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) { 254 panic("mp_startup_init: cpu%d: " 255 "no memory for cpu structure", cpun); 256 /*NOTREACHED*/ 257 } 258 procp = curthread->t_procp; 259 260 mutex_enter(&cpu_lock); 261 /* 262 * Initialize the dispatcher first. 263 */ 264 disp_cpu_init(cp); 265 mutex_exit(&cpu_lock); 266 267 /* 268 * Allocate and initialize the startup thread for this CPU. 269 * Interrupt and process switch stacks get allocated later 270 * when the CPU starts running. 271 */ 272 tp = thread_create(NULL, 0, NULL, NULL, 0, procp, 273 TS_STOPPED, maxclsyspri); 274 275 /* 276 * Set state to TS_ONPROC since this thread will start running 277 * as soon as the CPU comes online. 278 * 279 * All the other fields of the thread structure are setup by 280 * thread_create(). 281 */ 282 THREAD_ONPROC(tp, cp); 283 tp->t_preempt = 1; 284 tp->t_bound_cpu = cp; 285 tp->t_affinitycnt = 1; 286 tp->t_cpu = cp; 287 tp->t_disp_queue = cp->cpu_disp; 288 289 /* 290 * Setup thread to start in mp_startup. 291 */ 292 sp = tp->t_stk; 293 tp->t_pc = (uintptr_t)mp_startup; 294 tp->t_sp = (uintptr_t)(sp - MINFRAME); 295 296 cp->cpu_id = cpun; 297 cp->cpu_self = cp; 298 cp->cpu_mask = 1 << cpun; 299 cp->cpu_thread = tp; 300 cp->cpu_lwp = NULL; 301 cp->cpu_dispthread = tp; 302 cp->cpu_dispatch_pri = DISP_PRIO(tp); 303 304 /* 305 * Bootstrap cpu_chip in case mp_startup blocks 306 */ 307 cp->cpu_chip = &cpu0_chip; 308 309 /* 310 * Now, initialize per-CPU idle thread for this CPU. 311 */ 312 tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); 313 314 cp->cpu_idle_thread = tp; 315 316 tp->t_preempt = 1; 317 tp->t_bound_cpu = cp; 318 tp->t_affinitycnt = 1; 319 tp->t_cpu = cp; 320 tp->t_disp_queue = cp->cpu_disp; 321 322 /* 323 * Perform CPC intialization on the new CPU. 324 */ 325 kcpc_hw_init(cp); 326 327 /* 328 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 329 * for each CPU. 330 */ 331 332 setup_vaddr_for_ppcopy(cp); 333 334 /* 335 * Allocate space for page directory, stack, tss, gdt and idt. 336 * This assumes that kmem_alloc will return memory which is aligned 337 * to the next higher power of 2 or a page(if size > MAXABIG) 338 * If this assumption goes wrong at any time due to change in 339 * kmem alloc, things may not work as the page directory has to be 340 * page aligned 341 */ 342 if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL) 343 panic("mp_startup_init: cpu%d cannot allocate tables", cpun); 344 345 if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) { 346 kmem_free(tablesp, sizeof (struct cpu_tables)); 347 size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE; 348 tablesp = kmem_zalloc(size, KM_NOSLEEP); 349 tablesp = (struct cpu_tables *) 350 (((uintptr_t)tablesp + MMU_STD_PAGESIZE) & 351 MMU_STD_PAGEMASK); 352 } 353 354 ntss = cp->cpu_tss = &tablesp->ct_tss; 355 cp->cpu_gdt = tablesp->ct_gdt; 356 bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t))); 357 358 #if defined(__amd64) 359 360 /* 361 * #DF (double fault). 362 */ 363 ntss->tss_ist1 = 364 (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 365 366 #elif defined(__i386) 367 368 ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp = 369 (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 370 371 ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL; 372 373 ntss->tss_eip = (uint32_t)mp_startup; 374 375 ntss->tss_cs = KCS_SEL; 376 ntss->tss_fs = KFS_SEL; 377 ntss->tss_gs = KGS_SEL; 378 379 /* 380 * setup kernel %gs. 381 */ 382 set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA, 383 SEL_KPL, 0, 1); 384 385 #endif /* __i386 */ 386 387 /* 388 * Set I/O bit map offset equal to size of TSS segment limit 389 * for no I/O permission map. This will cause all user I/O 390 * instructions to generate #gp fault. 391 */ 392 ntss->tss_bitmapbase = sizeof (*ntss); 393 394 /* 395 * setup kernel tss. 396 */ 397 set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, 398 sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL); 399 400 /* 401 * If we have more than one node, each cpu gets a copy of IDT 402 * local to its node. If this is a Pentium box, we use cpu 0's 403 * IDT. cpu 0's IDT has been made read-only to workaround the 404 * cmpxchgl register bug 405 */ 406 cp->cpu_idt = CPU->cpu_idt; 407 if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) { 408 cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP); 409 bcopy(idt0, cp->cpu_idt, sizeof (idt0)); 410 } 411 412 /* 413 * Get interrupt priority data from cpu 0 414 */ 415 cp->cpu_pri_data = CPU->cpu_pri_data; 416 417 hat_cpu_online(cp); 418 419 /* Should remove all entries for the current process/thread here */ 420 421 /* 422 * Fill up the real mode platter to make it easy for real mode code to 423 * kick it off. This area should really be one passed by boot to kernel 424 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 425 * have identical physical and virtual address in paged mode. 426 */ 427 real_mode_platter->rm_idt_base = cp->cpu_idt; 428 real_mode_platter->rm_idt_lim = sizeof (idt0) - 1; 429 real_mode_platter->rm_gdt_base = cp->cpu_gdt; 430 real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1; 431 real_mode_platter->rm_pdbr = getcr3(); 432 real_mode_platter->rm_cpu = cpun; 433 real_mode_platter->rm_x86feature = x86_feature; 434 real_mode_platter->rm_cr4 = cr4_value; 435 436 #if defined(__amd64) 437 if (getcr3() > 0xffffffffUL) 438 panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 439 "located above 4G in physical memory (@ 0x%llx).", 440 (unsigned long long)getcr3()); 441 442 /* 443 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 444 * by code in real_mode_start(): 445 * 446 * GDT[0]: NULL selector 447 * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 448 * 449 * Clear the IDT as interrupts will be off and a limit of 0 will cause 450 * the CPU to triple fault and reset on an NMI, seemingly as reasonable 451 * a course of action as any other, though it may cause the entire 452 * platform to reset in some cases... 453 */ 454 real_mode_platter->rm_temp_gdt[0] = 0ULL; 455 real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 456 457 real_mode_platter->rm_temp_gdt_lim = (ushort_t) 458 (sizeof (real_mode_platter->rm_temp_gdt) - 1); 459 real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 460 (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt); 461 462 real_mode_platter->rm_temp_idt_lim = 0; 463 real_mode_platter->rm_temp_idt_base = 0; 464 465 /* 466 * Since the CPU needs to jump to protected mode using an identity 467 * mapped address, we need to calculate it here. 468 */ 469 real_mode_platter->rm_longmode64_addr = rm_platter_pa + 470 ((uint32_t)long_mode_64 - (uint32_t)real_mode_start); 471 #endif /* __amd64 */ 472 473 #ifdef TRAPTRACE 474 /* 475 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this 476 * CPU. 477 */ 478 ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); 479 ttc->ttc_next = ttc->ttc_first; 480 ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; 481 #endif 482 483 /* 484 * Record that we have another CPU. 485 */ 486 mutex_enter(&cpu_lock); 487 /* 488 * Initialize the interrupt threads for this CPU 489 */ 490 init_intr_threads(cp); 491 /* 492 * Add CPU to list of available CPUs. It'll be on the active list 493 * after mp_startup(). 494 */ 495 cpu_add_unit(cp); 496 mutex_exit(&cpu_lock); 497 } 498 499 /* 500 * Apply workarounds for known errata, and warn about those that are absent. 501 * 502 * System vendors occasionally create configurations which contain different 503 * revisions of the CPUs that are almost but not exactly the same. At the 504 * time of writing, this meant that their clock rates were the same, their 505 * feature sets were the same, but the required workaround were -not- 506 * necessarily the same. So, this routine is invoked on -every- CPU soon 507 * after starting to make sure that the resulting system contains the most 508 * pessimal set of workarounds needed to cope with *any* of the CPUs in the 509 * system. 510 * 511 * These workarounds are based on Rev 3.50 of the Revision Guide for 512 * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, May 2005. 513 */ 514 515 #if defined(OPTERON_ERRATUM_91) 516 int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ 517 #endif 518 519 #if defined(OPTERON_ERRATUM_93) 520 int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ 521 #endif 522 523 #if defined(OPTERON_ERRATUM_100) 524 int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ 525 #endif 526 527 #if defined(OPTERON_ERRATUM_109) 528 int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ 529 #endif 530 531 #if defined(OPTERON_ERRATUM_121) 532 int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ 533 #endif 534 535 #if defined(OPTERON_ERRATUM_122) 536 int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ 537 #endif 538 539 #if defined(OPTERON_ERRATUM_123) 540 int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ 541 #endif 542 543 544 #define WARNING(cpu, n) \ 545 cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d", \ 546 (cpu)->cpu_id, (n)) 547 548 uint_t 549 workaround_errata(struct cpu *cpu) 550 { 551 uint_t missing = 0; 552 553 ASSERT(cpu == CPU); 554 555 /*LINTED*/ 556 if (cpuid_opteron_erratum(cpu, 88) > 0) { 557 /* 558 * SWAPGS May Fail To Read Correct GS Base 559 */ 560 #if defined(OPTERON_ERRATUM_88) 561 /* 562 * The workaround is an mfence in the relevant assembler code 563 */ 564 #else 565 WARNING(cpu, 88); 566 missing++; 567 #endif 568 } 569 570 if (cpuid_opteron_erratum(cpu, 91) > 0) { 571 /* 572 * Software Prefetches May Report A Page Fault 573 */ 574 #if defined(OPTERON_ERRATUM_91) 575 /* 576 * fix is in trap.c 577 */ 578 opteron_erratum_91++; 579 #else 580 WARNING(cpu, 91); 581 missing++; 582 #endif 583 } 584 585 if (cpuid_opteron_erratum(cpu, 93) > 0) { 586 /* 587 * RSM Auto-Halt Restart Returns to Incorrect RIP 588 */ 589 #if defined(OPTERON_ERRATUM_93) 590 /* 591 * fix is in trap.c 592 */ 593 opteron_erratum_93++; 594 #else 595 WARNING(cpu, 93); 596 missing++; 597 #endif 598 } 599 600 /*LINTED*/ 601 if (cpuid_opteron_erratum(cpu, 95) > 0) { 602 /* 603 * RET Instruction May Return to Incorrect EIP 604 */ 605 #if defined(OPTERON_ERRATUM_95) 606 #if defined(_LP64) 607 /* 608 * Workaround this by ensuring that 32-bit user code and 609 * 64-bit kernel code never occupy the same address 610 * range mod 4G. 611 */ 612 if (_userlimit32 > 0xc0000000ul) 613 *(uintptr_t *)&_userlimit32 = 0xc0000000ul; 614 615 /*LINTED*/ 616 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); 617 #endif /* _LP64 */ 618 #else 619 WARNING(cpu, 95); 620 missing++; 621 #endif /* OPTERON_ERRATUM_95 */ 622 } 623 624 if (cpuid_opteron_erratum(cpu, 100) > 0) { 625 /* 626 * Compatibility Mode Branches Transfer to Illegal Address 627 */ 628 #if defined(OPTERON_ERRATUM_100) 629 /* 630 * fix is in trap.c 631 */ 632 opteron_erratum_100++; 633 #else 634 WARNING(cpu, 100); 635 missing++; 636 #endif 637 } 638 639 /*LINTED*/ 640 if (cpuid_opteron_erratum(cpu, 108) > 0) { 641 /* 642 * CPUID Instruction May Return Incorrect Model Number In 643 * Some Processors 644 */ 645 #if defined(OPTERON_ERRATUM_108) 646 /* 647 * (Our cpuid-handling code corrects the model number on 648 * those processors) 649 */ 650 #else 651 WARNING(cpu, 108); 652 missing++; 653 #endif 654 } 655 656 /*LINTED*/ 657 if (cpuid_opteron_erratum(cpu, 109) > 0) { 658 /* 659 * Certain Reverse REP MOVS May Produce Unpredictable Behaviour 660 */ 661 #if defined(OPTERON_ERRATUM_109) 662 uint64_t patchlevel; 663 664 (void) rdmsr(MSR_AMD_PATCHLEVEL, &patchlevel); 665 /* workaround is to print a warning to upgrade BIOS */ 666 if (patchlevel == 0) 667 opteron_erratum_109++; 668 #else 669 WARNING(cpu, 109); 670 missing++; 671 #endif 672 } 673 /*LINTED*/ 674 if (cpuid_opteron_erratum(cpu, 121) > 0) { 675 /* 676 * Sequential Execution Across Non_Canonical Boundary Caused 677 * Processor Hang 678 */ 679 #if defined(OPTERON_ERRATUM_121) 680 static int lma; 681 682 if (opteron_erratum_121) 683 opteron_erratum_121++; 684 685 /* 686 * Erratum 121 is only present in long (64 bit) mode. 687 * Workaround is to include the page immediately before the 688 * va hole to eliminate the possibility of system hangs due to 689 * sequential execution across the va hole boundary. 690 */ 691 if (lma == 0) { 692 uint64_t efer; 693 694 /* 695 * check LMA once: assume all cpus are in long mode 696 * or not. 697 */ 698 lma = 1; 699 700 (void) rdmsr(MSR_AMD_EFER, &efer); 701 if (efer & AMD_EFER_LMA) { 702 if (hole_start) { 703 hole_start -= PAGESIZE; 704 } else { 705 /* 706 * hole_start not yet initialized by 707 * mmu_init. Initialize hole_start 708 * with value to be subtracted. 709 */ 710 hole_start = PAGESIZE; 711 } 712 opteron_erratum_121++; 713 } 714 } 715 #else 716 WARNING(cpu, 121); 717 missing++; 718 #endif 719 } 720 721 /*LINTED*/ 722 if (cpuid_opteron_erratum(cpu, 122) > 0) { 723 /* 724 * TLB Flush Filter May Cause Cohenrency Problem in 725 * Multiprocessor Systems 726 */ 727 #if defined(OPTERON_ERRATUM_122) 728 /* 729 * Erratum 122 is only present in MP configurations (multi-core 730 * or multi-processor). 731 */ 732 733 if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 || 734 cpuid_get_ncpu_per_chip(cpu) > 1) { 735 uint64_t hwcrval; 736 737 /* disable TLB Flush Filter */ 738 (void) rdmsr(MSR_AMD_HWCR, &hwcrval); 739 hwcrval |= AMD_HWCR_FFDIS; 740 wrmsr(MSR_AMD_HWCR, &hwcrval); 741 opteron_erratum_122++; 742 } 743 744 #else 745 WARNING(cpu, 122); 746 missing++; 747 #endif 748 } 749 /*LINTED*/ 750 if (cpuid_opteron_erratum(cpu, 123) > 0) { 751 /* 752 * Bypassed Reads May Cause Data Corruption of System Hang in 753 * Dual Core Processors 754 */ 755 #if defined(OPTERON_ERRATUM_123) 756 /* 757 * Erratum 123 applies only to multi-core cpus. 758 */ 759 760 if (cpuid_get_ncpu_per_chip(cpu) > 1) { 761 uint64_t patchlevel; 762 763 (void) rdmsr(MSR_AMD_PATCHLEVEL, &patchlevel); 764 /* workaround is to print a warning to upgrade BIOS */ 765 if (patchlevel == 0) 766 opteron_erratum_123++; 767 } 768 #else 769 WARNING(cpu, 123); 770 missing++; 771 #endif 772 } 773 return (missing); 774 } 775 776 void 777 workaround_errata_end() 778 { 779 #if defined(OPTERON_ERRATUM_109) 780 if (opteron_erratum_109) { 781 cmn_err(CE_WARN, "!BIOS microcode patch for AMD Processor" 782 " Erratum 109 was not detected. Updating BIOS with the" 783 " microcode patch is highly recommended."); 784 } 785 #endif 786 #if defined(OPTERON_ERRATUM_123) 787 if (opteron_erratum_123) { 788 cmn_err(CE_WARN, "!BIOS microcode patch for AMD Processor" 789 " Erratum 123 was not detected. Updating BIOS with the" 790 " microcode patch is highly recommended."); 791 } 792 #endif 793 } 794 795 static ushort_t *mp_map_warm_reset_vector(); 796 static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector); 797 798 /*ARGSUSED*/ 799 void 800 start_other_cpus(int cprboot) 801 { 802 unsigned who; 803 int cpuid = getbootcpuid(); 804 int delays = 0; 805 int started_cpu; 806 ushort_t *warm_reset_vector = NULL; 807 extern int procset; 808 809 /* 810 * Initialize our own cpu_info. 811 */ 812 init_cpu_info(CPU); 813 814 /* 815 * Initialize our syscall handlers 816 */ 817 init_cpu_syscall(CPU); 818 819 /* 820 * if only 1 cpu or not using MP, skip the rest of this 821 */ 822 if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) { 823 if (use_mp == 0) 824 cmn_err(CE_CONT, "?***** Not in MP mode\n"); 825 goto done; 826 } 827 828 /* 829 * perform such initialization as is needed 830 * to be able to take CPUs on- and off-line. 831 */ 832 cpu_pause_init(); 833 834 xc_init(); /* initialize processor crosscalls */ 835 836 /* 837 * Copy the real mode code at "real_mode_start" to the 838 * page at rm_platter_va. 839 */ 840 warm_reset_vector = mp_map_warm_reset_vector(); 841 if (warm_reset_vector == NULL) 842 goto done; 843 844 bcopy((caddr_t)real_mode_start, 845 (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code, 846 (size_t)real_mode_end - (size_t)real_mode_start); 847 848 flushes_require_xcalls = 1; 849 850 affinity_set(CPU_CURRENT); 851 852 for (who = 0; who < NCPU; who++) { 853 if (who == cpuid) 854 continue; 855 856 if ((mp_cpus & (1 << who)) == 0) 857 continue; 858 859 mp_startup_init(who); 860 started_cpu = 1; 861 (*cpu_startf)(who, rm_platter_pa); 862 863 while ((procset & (1 << who)) == 0) { 864 865 delay(1); 866 if (++delays > (20 * hz)) { 867 868 cmn_err(CE_WARN, 869 "cpu%d failed to start", who); 870 871 mutex_enter(&cpu_lock); 872 cpu[who]->cpu_flags = 0; 873 cpu_del_unit(who); 874 mutex_exit(&cpu_lock); 875 876 started_cpu = 0; 877 break; 878 } 879 } 880 if (!started_cpu) 881 continue; 882 if (tsc_gethrtime_enable) 883 tsc_sync_master(who); 884 885 886 if (dtrace_cpu_init != NULL) { 887 /* 888 * DTrace CPU initialization expects cpu_lock 889 * to be held. 890 */ 891 mutex_enter(&cpu_lock); 892 (*dtrace_cpu_init)(who); 893 mutex_exit(&cpu_lock); 894 } 895 } 896 897 affinity_clear(); 898 899 for (who = 0; who < NCPU; who++) { 900 if (who == cpuid) 901 continue; 902 903 if (!(procset & (1 << who))) 904 continue; 905 906 while (!(cpu_ready_set & (1 << who))) 907 delay(1); 908 } 909 910 done: 911 workaround_errata_end(); 912 913 if (warm_reset_vector != NULL) 914 mp_unmap_warm_reset_vector(warm_reset_vector); 915 hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 916 HAT_UNLOAD); 917 } 918 919 /* 920 * Dummy functions - no i86pc platforms support dynamic cpu allocation. 921 */ 922 /*ARGSUSED*/ 923 int 924 mp_cpu_configure(int cpuid) 925 { 926 return (ENOTSUP); /* not supported */ 927 } 928 929 /*ARGSUSED*/ 930 int 931 mp_cpu_unconfigure(int cpuid) 932 { 933 return (ENOTSUP); /* not supported */ 934 } 935 936 /* 937 * Startup function for 'other' CPUs (besides boot cpu). 938 * Resumed from cpu_startup. 939 */ 940 void 941 mp_startup(void) 942 { 943 struct cpu *cp = CPU; 944 extern int procset; 945 uint_t new_x86_feature; 946 947 new_x86_feature = cpuid_pass1(cp); 948 949 /* 950 * We need to Sync MTRR with cpu0's MTRR. We have to do 951 * this with interrupts disabled. 952 */ 953 if (x86_feature & X86_MTRR) 954 mtrr_sync(); 955 /* 956 * Enable machine check architecture 957 */ 958 if (x86_feature & X86_MCA) 959 setup_mca(); 960 961 /* 962 * Initialize this CPU's syscall handlers 963 */ 964 init_cpu_syscall(cp); 965 966 /* 967 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the 968 * highest level at which a routine is permitted to block on 969 * an adaptive mutex (allows for cpu poke interrupt in case 970 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks 971 * device interrupts that may end up in the hat layer issuing cross 972 * calls before CPU_READY is set. 973 */ 974 (void) splx(ipltospl(LOCK_LEVEL)); 975 976 /* 977 * Do a sanity check to make sure this new CPU is a sane thing 978 * to add to the collection of processors running this system. 979 * 980 * XXX Clearly this needs to get more sophisticated, if x86 981 * systems start to get built out of heterogenous CPUs; as is 982 * likely to happen once the number of processors in a configuration 983 * gets large enough. 984 */ 985 if ((x86_feature & new_x86_feature) != x86_feature) { 986 cmn_err(CE_CONT, "?cpu%d: %b\n", 987 cp->cpu_id, new_x86_feature, FMT_X86_FEATURE); 988 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); 989 } 990 991 /* 992 * We could be more sophisticated here, and just mark the CPU 993 * as "faulted" but at this point we'll opt for the easier 994 * answer of dieing horribly. Provided the boot cpu is ok, 995 * the system can be recovered by booting with use_mp set to zero. 996 */ 997 if (workaround_errata(cp) != 0) 998 panic("critical workaround(s) missing for cpu%d", cp->cpu_id); 999 1000 cpuid_pass2(cp); 1001 cpuid_pass3(cp); 1002 (void) cpuid_pass4(cp); 1003 1004 init_cpu_info(cp); 1005 1006 add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi); 1007 1008 mutex_enter(&cpu_lock); 1009 procset |= 1 << cp->cpu_id; 1010 mutex_exit(&cpu_lock); 1011 1012 if (tsc_gethrtime_enable) 1013 tsc_sync_slave(); 1014 1015 mutex_enter(&cpu_lock); 1016 /* 1017 * It's unfortunate that chip_cpu_init() has to be called here. 1018 * It really belongs in cpu_add_unit(), but unfortunately it is 1019 * dependent on the cpuid probing, which must be done in the 1020 * context of the current CPU. Care must be taken on x86 to ensure 1021 * that mp_startup can safely block even though chip_cpu_init() and 1022 * cpu_add_active() have not yet been called. 1023 */ 1024 chip_cpu_init(cp); 1025 chip_cpu_startup(cp); 1026 1027 cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS; 1028 cpu_add_active(cp); 1029 mutex_exit(&cpu_lock); 1030 1031 (void) spl0(); /* enable interrupts */ 1032 1033 if (boothowto & RB_DEBUG) 1034 kdi_dvec_cpu_init(cp); 1035 1036 /* 1037 * Setting the bit in cpu_ready_set must be the last operation in 1038 * processor initialization; the boot CPU will continue to boot once 1039 * it sees this bit set for all active CPUs. 1040 */ 1041 CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); 1042 1043 /* 1044 * Because mp_startup() gets fired off after init() starts, we 1045 * can't use the '?' trick to do 'boot -v' printing - so we 1046 * always direct the 'cpu .. online' messages to the log. 1047 */ 1048 cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 1049 cp->cpu_id); 1050 1051 /* 1052 * Now we are done with the startup thread, so free it up. 1053 */ 1054 thread_exit(); 1055 panic("mp_startup: cannot return"); 1056 /*NOTREACHED*/ 1057 } 1058 1059 1060 /* 1061 * Start CPU on user request. 1062 */ 1063 /* ARGSUSED */ 1064 int 1065 mp_cpu_start(struct cpu *cp) 1066 { 1067 ASSERT(MUTEX_HELD(&cpu_lock)); 1068 if (cp->cpu_id == getbootcpuid()) 1069 return (EBUSY); /* Cannot start boot CPU */ 1070 return (0); 1071 } 1072 1073 /* 1074 * Stop CPU on user request. 1075 */ 1076 /* ARGSUSED */ 1077 int 1078 mp_cpu_stop(struct cpu *cp) 1079 { 1080 ASSERT(MUTEX_HELD(&cpu_lock)); 1081 if (cp->cpu_id == getbootcpuid()) 1082 return (EBUSY); /* Cannot stop boot CPU */ 1083 1084 return (0); 1085 } 1086 1087 /* 1088 * Power on CPU. 1089 */ 1090 /* ARGSUSED */ 1091 int 1092 mp_cpu_poweron(struct cpu *cp) 1093 { 1094 ASSERT(MUTEX_HELD(&cpu_lock)); 1095 return (ENOTSUP); /* not supported */ 1096 } 1097 1098 /* 1099 * Power off CPU. 1100 */ 1101 /* ARGSUSED */ 1102 int 1103 mp_cpu_poweroff(struct cpu *cp) 1104 { 1105 ASSERT(MUTEX_HELD(&cpu_lock)); 1106 return (ENOTSUP); /* not supported */ 1107 } 1108 1109 1110 /* 1111 * Take the specified CPU out of participation in interrupts. 1112 */ 1113 int 1114 cpu_disable_intr(struct cpu *cp) 1115 { 1116 /* 1117 * cannot disable interrupts on boot cpu 1118 */ 1119 if (cp == cpu[getbootcpuid()]) 1120 return (EBUSY); 1121 1122 if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) 1123 return (EBUSY); 1124 1125 cp->cpu_flags &= ~CPU_ENABLE; 1126 return (0); 1127 } 1128 1129 /* 1130 * Allow the specified CPU to participate in interrupts. 1131 */ 1132 void 1133 cpu_enable_intr(struct cpu *cp) 1134 { 1135 ASSERT(MUTEX_HELD(&cpu_lock)); 1136 if (cp == cpu[getbootcpuid()]) 1137 return; 1138 1139 cp->cpu_flags |= CPU_ENABLE; 1140 psm_enable_intr(cp->cpu_id); 1141 } 1142 1143 1144 /* 1145 * return the cpu id of the initial startup cpu 1146 */ 1147 processorid_t 1148 getbootcpuid(void) 1149 { 1150 return (0); 1151 } 1152 1153 static ushort_t * 1154 mp_map_warm_reset_vector() 1155 { 1156 ushort_t *warm_reset_vector; 1157 1158 if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 1159 sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 1160 return (NULL); 1161 1162 /* 1163 * setup secondary cpu bios boot up vector 1164 */ 1165 *warm_reset_vector = (ushort_t)((caddr_t) 1166 ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 1167 + ((ulong_t)rm_platter_va & 0xf)); 1168 warm_reset_vector++; 1169 *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 1170 1171 --warm_reset_vector; 1172 return (warm_reset_vector); 1173 } 1174 1175 static void 1176 mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector) 1177 { 1178 psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 1179 } 1180 1181 /*ARGSUSED*/ 1182 void 1183 mp_cpu_faulted_enter(struct cpu *cp) 1184 {} 1185 1186 /*ARGSUSED*/ 1187 void 1188 mp_cpu_faulted_exit(struct cpu *cp) 1189 {} 1190 1191 /* 1192 * The following two routines are used as context operators on threads belonging 1193 * to processes with a private LDT (see sysi86). Due to the rarity of such 1194 * processes, these routines are currently written for best code readability and 1195 * organization rather than speed. We could avoid checking x86_feature at every 1196 * context switch by installing different context ops, depending on the 1197 * x86_feature flags, at LDT creation time -- one for each combination of fast 1198 * syscall feature flags. 1199 */ 1200 1201 /*ARGSUSED*/ 1202 void 1203 cpu_fast_syscall_disable(void *arg) 1204 { 1205 if (x86_feature & X86_SEP) 1206 cpu_sep_disable(); 1207 if (x86_feature & X86_ASYSC) 1208 cpu_asysc_disable(); 1209 } 1210 1211 /*ARGSUSED*/ 1212 void 1213 cpu_fast_syscall_enable(void *arg) 1214 { 1215 if (x86_feature & X86_SEP) 1216 cpu_sep_enable(); 1217 if (x86_feature & X86_ASYSC) 1218 cpu_asysc_enable(); 1219 } 1220 1221 static void 1222 cpu_sep_enable(void) 1223 { 1224 uint64_t value; 1225 1226 ASSERT(x86_feature & X86_SEP); 1227 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1228 1229 value = KCS_SEL; 1230 wrmsr(MSR_INTC_SEP_CS, &value); 1231 } 1232 1233 static void 1234 cpu_sep_disable(void) 1235 { 1236 uint64_t value; 1237 1238 ASSERT(x86_feature & X86_SEP); 1239 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1240 1241 /* 1242 * Setting the SYSENTER_CS_MSR register to 0 causes software executing 1243 * the sysenter or sysexit instruction to trigger a #gp fault. 1244 */ 1245 value = 0; 1246 wrmsr(MSR_INTC_SEP_CS, &value); 1247 } 1248 1249 static void 1250 cpu_asysc_enable(void) 1251 { 1252 uint64_t value; 1253 1254 ASSERT(x86_feature & X86_ASYSC); 1255 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1256 1257 (void) rdmsr(MSR_AMD_EFER, &value); 1258 value |= AMD_EFER_SCE; 1259 wrmsr(MSR_AMD_EFER, &value); 1260 } 1261 1262 static void 1263 cpu_asysc_disable(void) 1264 { 1265 uint64_t value; 1266 1267 ASSERT(x86_feature & X86_ASYSC); 1268 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1269 1270 /* 1271 * Turn off the SCE (syscall enable) bit in the EFER register. Software 1272 * executing syscall or sysret with this bit off will incur a #ud trap. 1273 */ 1274 (void) rdmsr(MSR_AMD_EFER, &value); 1275 value &= ~AMD_EFER_SCE; 1276 wrmsr(MSR_AMD_EFER, &value); 1277 } 1278