1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/thread.h> 31 #include <sys/cpuvar.h> 32 #include <sys/t_lock.h> 33 #include <sys/param.h> 34 #include <sys/proc.h> 35 #include <sys/disp.h> 36 #include <sys/mmu.h> 37 #include <sys/class.h> 38 #include <sys/cmn_err.h> 39 #include <sys/debug.h> 40 #include <sys/asm_linkage.h> 41 #include <sys/x_call.h> 42 #include <sys/systm.h> 43 #include <sys/var.h> 44 #include <sys/vtrace.h> 45 #include <vm/hat.h> 46 #include <sys/mmu.h> 47 #include <vm/as.h> 48 #include <vm/seg_kmem.h> 49 #include <sys/segments.h> 50 #include <sys/kmem.h> 51 #include <sys/stack.h> 52 #include <sys/smp_impldefs.h> 53 #include <sys/x86_archext.h> 54 #include <sys/machsystm.h> 55 #include <sys/traptrace.h> 56 #include <sys/clock.h> 57 #include <sys/cpc_impl.h> 58 #include <sys/chip.h> 59 #include <sys/dtrace.h> 60 #include <sys/archsystm.h> 61 #include <sys/fp.h> 62 #include <sys/reboot.h> 63 #include <sys/kdi.h> 64 #include <vm/hat_i86.h> 65 #include <sys/memnode.h> 66 67 struct cpu cpus[1]; /* CPU data */ 68 struct cpu *cpu[NCPU] = {&cpus[0]}; /* pointers to all CPUs */ 69 cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 70 71 /* 72 * Useful for disabling MP bring-up for an MP capable kernel 73 * (a kernel that was built with MP defined) 74 */ 75 int use_mp = 1; 76 77 int mp_cpus = 0x1; /* to be set by platform specific module */ 78 79 /* 80 * This variable is used by the hat layer to decide whether or not 81 * critical sections are needed to prevent race conditions. For sun4m, 82 * this variable is set once enough MP initialization has been done in 83 * order to allow cross calls. 84 */ 85 int flushes_require_xcalls = 0; 86 ulong_t cpu_ready_set = 1; 87 88 extern void real_mode_start(void); 89 extern void real_mode_end(void); 90 static void mp_startup(void); 91 92 static void cpu_sep_enable(void); 93 static void cpu_sep_disable(void); 94 static void cpu_asysc_enable(void); 95 static void cpu_asysc_disable(void); 96 97 extern int tsc_gethrtime_enable; 98 99 /* 100 * Init CPU info - get CPU type info for processor_info system call. 101 */ 102 void 103 init_cpu_info(struct cpu *cp) 104 { 105 processor_info_t *pi = &cp->cpu_type_info; 106 char buf[CPU_IDSTRLEN]; 107 108 /* 109 * Get clock-frequency property for the CPU. 110 */ 111 pi->pi_clock = cpu_freq; 112 113 (void) strcpy(pi->pi_processor_type, "i386"); 114 if (fpu_exists) 115 (void) strcpy(pi->pi_fputypes, "i387 compatible"); 116 117 (void) cpuid_getidstr(cp, buf, sizeof (buf)); 118 119 cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 120 (void) strcpy(cp->cpu_idstr, buf); 121 122 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); 123 124 (void) cpuid_getbrandstr(cp, buf, sizeof (buf)); 125 cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 126 (void) strcpy(cp->cpu_brandstr, buf); 127 128 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); 129 } 130 131 /* 132 * Configure syscall support on this CPU. 133 */ 134 /*ARGSUSED*/ 135 static void 136 init_cpu_syscall(struct cpu *cp) 137 { 138 uint64_t value; 139 140 kpreempt_disable(); 141 142 #if defined(__amd64) 143 if (x86_feature & X86_ASYSC) { 144 145 #if !defined(__lint) 146 /* 147 * The syscall instruction imposes a certain ordering on 148 * segment selectors, so we double-check that ordering 149 * here. 150 */ 151 ASSERT(KDS_SEL == KCS_SEL + 8); 152 ASSERT(UDS_SEL == U32CS_SEL + 8); 153 ASSERT(UCS_SEL == U32CS_SEL + 16); 154 #endif 155 /* 156 * Turn syscall/sysret extensions on. 157 */ 158 cpu_asysc_enable(); 159 160 /* 161 * Program the magic registers .. 162 */ 163 value = ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32; 164 wrmsr(MSR_AMD_STAR, &value); 165 value = (uintptr_t)sys_syscall; 166 wrmsr(MSR_AMD_LSTAR, &value); 167 value = (uintptr_t)sys_syscall32; 168 wrmsr(MSR_AMD_CSTAR, &value); 169 170 /* 171 * This list of flags is masked off the incoming 172 * %rfl when we enter the kernel. 173 */ 174 value = PS_IE | PS_T; 175 wrmsr(MSR_AMD_SFMASK, &value); 176 } 177 #endif 178 179 /* 180 * On 32-bit kernels, we use sysenter/sysexit because it's too 181 * hard to use syscall/sysret, and it is more portable anyway. 182 * 183 * On 64-bit kernels on Nocona machines, the 32-bit syscall 184 * variant isn't available to 32-bit applications, but sysenter is. 185 */ 186 if (x86_feature & X86_SEP) { 187 188 #if !defined(__lint) 189 /* 190 * The sysenter instruction imposes a certain ordering on 191 * segment selectors, so we double-check that ordering 192 * here. See "sysenter" in Intel document 245471-012, "IA-32 193 * Intel Architecture Software Developer's Manual Volume 2: 194 * Instruction Set Reference" 195 */ 196 ASSERT(KDS_SEL == KCS_SEL + 8); 197 198 ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3)); 199 ASSERT32(UDS_SEL == UCS_SEL + 8); 200 201 ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3)); 202 ASSERT64(UDS_SEL == U32CS_SEL + 8); 203 #endif 204 205 cpu_sep_enable(); 206 207 /* 208 * resume() sets this value to the base of the threads stack 209 * via a context handler. 210 */ 211 value = 0; 212 wrmsr(MSR_INTC_SEP_ESP, &value); 213 214 value = (uintptr_t)sys_sysenter; 215 wrmsr(MSR_INTC_SEP_EIP, &value); 216 } 217 218 kpreempt_enable(); 219 } 220 221 /* 222 * Multiprocessor initialization. 223 * 224 * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the 225 * startup and idle threads for the specified CPU. 226 */ 227 static void 228 mp_startup_init(int cpun) 229 { 230 #if defined(__amd64) 231 extern void *long_mode_64(void); 232 #endif /* __amd64 */ 233 234 struct cpu *cp; 235 struct tss *ntss; 236 kthread_id_t tp; 237 caddr_t sp; 238 int size; 239 proc_t *procp; 240 extern void idle(); 241 extern void init_intr_threads(struct cpu *); 242 243 struct cpu_tables *tablesp; 244 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 245 246 #ifdef TRAPTRACE 247 trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; 248 #endif 249 250 ASSERT(cpun < NCPU && cpu[cpun] == NULL); 251 252 if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) { 253 panic("mp_startup_init: cpu%d: " 254 "no memory for cpu structure", cpun); 255 /*NOTREACHED*/ 256 } 257 procp = curthread->t_procp; 258 259 mutex_enter(&cpu_lock); 260 /* 261 * Initialize the dispatcher first. 262 */ 263 disp_cpu_init(cp); 264 mutex_exit(&cpu_lock); 265 266 /* 267 * Allocate and initialize the startup thread for this CPU. 268 * Interrupt and process switch stacks get allocated later 269 * when the CPU starts running. 270 */ 271 tp = thread_create(NULL, 0, NULL, NULL, 0, procp, 272 TS_STOPPED, maxclsyspri); 273 274 /* 275 * Set state to TS_ONPROC since this thread will start running 276 * as soon as the CPU comes online. 277 * 278 * All the other fields of the thread structure are setup by 279 * thread_create(). 280 */ 281 THREAD_ONPROC(tp, cp); 282 tp->t_preempt = 1; 283 tp->t_bound_cpu = cp; 284 tp->t_affinitycnt = 1; 285 tp->t_cpu = cp; 286 tp->t_disp_queue = cp->cpu_disp; 287 288 /* 289 * Setup thread to start in mp_startup. 290 */ 291 sp = tp->t_stk; 292 tp->t_pc = (uintptr_t)mp_startup; 293 tp->t_sp = (uintptr_t)(sp - MINFRAME); 294 295 cp->cpu_id = cpun; 296 cp->cpu_self = cp; 297 cp->cpu_mask = 1 << cpun; 298 cp->cpu_thread = tp; 299 cp->cpu_lwp = NULL; 300 cp->cpu_dispthread = tp; 301 cp->cpu_dispatch_pri = DISP_PRIO(tp); 302 303 /* 304 * Now, initialize per-CPU idle thread for this CPU. 305 */ 306 tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); 307 308 cp->cpu_idle_thread = tp; 309 310 tp->t_preempt = 1; 311 tp->t_bound_cpu = cp; 312 tp->t_affinitycnt = 1; 313 tp->t_cpu = cp; 314 tp->t_disp_queue = cp->cpu_disp; 315 316 /* 317 * Bootstrap the CPU for CMT aware scheduling 318 * The rest of the initialization will happen from 319 * mp_startup() 320 */ 321 chip_bootstrap_cpu(cp); 322 323 /* 324 * Perform CPC intialization on the new CPU. 325 */ 326 kcpc_hw_init(cp); 327 328 /* 329 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 330 * for each CPU. 331 */ 332 333 setup_vaddr_for_ppcopy(cp); 334 335 /* 336 * Allocate space for page directory, stack, tss, gdt and idt. 337 * This assumes that kmem_alloc will return memory which is aligned 338 * to the next higher power of 2 or a page(if size > MAXABIG) 339 * If this assumption goes wrong at any time due to change in 340 * kmem alloc, things may not work as the page directory has to be 341 * page aligned 342 */ 343 if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL) 344 panic("mp_startup_init: cpu%d cannot allocate tables", cpun); 345 346 if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) { 347 kmem_free(tablesp, sizeof (struct cpu_tables)); 348 size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE; 349 tablesp = kmem_zalloc(size, KM_NOSLEEP); 350 tablesp = (struct cpu_tables *) 351 (((uintptr_t)tablesp + MMU_STD_PAGESIZE) & 352 MMU_STD_PAGEMASK); 353 } 354 355 ntss = cp->cpu_tss = &tablesp->ct_tss; 356 cp->cpu_gdt = tablesp->ct_gdt; 357 bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t))); 358 359 #if defined(__amd64) 360 361 /* 362 * #DF (double fault). 363 */ 364 ntss->tss_ist1 = 365 (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 366 367 #elif defined(__i386) 368 369 ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp = 370 (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 371 372 ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL; 373 374 ntss->tss_eip = (uint32_t)mp_startup; 375 376 ntss->tss_cs = KCS_SEL; 377 ntss->tss_fs = KFS_SEL; 378 ntss->tss_gs = KGS_SEL; 379 380 /* 381 * setup kernel %gs. 382 */ 383 set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA, 384 SEL_KPL, 0, 1); 385 386 #endif /* __i386 */ 387 388 /* 389 * Set I/O bit map offset equal to size of TSS segment limit 390 * for no I/O permission map. This will cause all user I/O 391 * instructions to generate #gp fault. 392 */ 393 ntss->tss_bitmapbase = sizeof (*ntss); 394 395 /* 396 * setup kernel tss. 397 */ 398 set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, 399 sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL); 400 401 /* 402 * If we have more than one node, each cpu gets a copy of IDT 403 * local to its node. If this is a Pentium box, we use cpu 0's 404 * IDT. cpu 0's IDT has been made read-only to workaround the 405 * cmpxchgl register bug 406 */ 407 cp->cpu_idt = CPU->cpu_idt; 408 if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) { 409 cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP); 410 bcopy(idt0, cp->cpu_idt, sizeof (idt0)); 411 } 412 413 /* 414 * Get interrupt priority data from cpu 0 415 */ 416 cp->cpu_pri_data = CPU->cpu_pri_data; 417 418 hat_cpu_online(cp); 419 420 /* Should remove all entries for the current process/thread here */ 421 422 /* 423 * Fill up the real mode platter to make it easy for real mode code to 424 * kick it off. This area should really be one passed by boot to kernel 425 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 426 * have identical physical and virtual address in paged mode. 427 */ 428 real_mode_platter->rm_idt_base = cp->cpu_idt; 429 real_mode_platter->rm_idt_lim = sizeof (idt0) - 1; 430 real_mode_platter->rm_gdt_base = cp->cpu_gdt; 431 real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1; 432 real_mode_platter->rm_pdbr = getcr3(); 433 real_mode_platter->rm_cpu = cpun; 434 real_mode_platter->rm_x86feature = x86_feature; 435 real_mode_platter->rm_cr4 = cr4_value; 436 437 #if defined(__amd64) 438 if (getcr3() > 0xffffffffUL) 439 panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 440 "located above 4G in physical memory (@ 0x%llx).", 441 (unsigned long long)getcr3()); 442 443 /* 444 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 445 * by code in real_mode_start(): 446 * 447 * GDT[0]: NULL selector 448 * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 449 * 450 * Clear the IDT as interrupts will be off and a limit of 0 will cause 451 * the CPU to triple fault and reset on an NMI, seemingly as reasonable 452 * a course of action as any other, though it may cause the entire 453 * platform to reset in some cases... 454 */ 455 real_mode_platter->rm_temp_gdt[0] = 0ULL; 456 real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 457 458 real_mode_platter->rm_temp_gdt_lim = (ushort_t) 459 (sizeof (real_mode_platter->rm_temp_gdt) - 1); 460 real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 461 (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt); 462 463 real_mode_platter->rm_temp_idt_lim = 0; 464 real_mode_platter->rm_temp_idt_base = 0; 465 466 /* 467 * Since the CPU needs to jump to protected mode using an identity 468 * mapped address, we need to calculate it here. 469 */ 470 real_mode_platter->rm_longmode64_addr = rm_platter_pa + 471 ((uint32_t)long_mode_64 - (uint32_t)real_mode_start); 472 #endif /* __amd64 */ 473 474 #ifdef TRAPTRACE 475 /* 476 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this 477 * CPU. 478 */ 479 ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); 480 ttc->ttc_next = ttc->ttc_first; 481 ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; 482 #endif 483 484 /* 485 * Record that we have another CPU. 486 */ 487 mutex_enter(&cpu_lock); 488 /* 489 * Initialize the interrupt threads for this CPU 490 */ 491 init_intr_threads(cp); 492 /* 493 * Add CPU to list of available CPUs. It'll be on the active list 494 * after mp_startup(). 495 */ 496 cpu_add_unit(cp); 497 mutex_exit(&cpu_lock); 498 } 499 500 /* 501 * Apply workarounds for known errata, and warn about those that are absent. 502 * 503 * System vendors occasionally create configurations which contain different 504 * revisions of the CPUs that are almost but not exactly the same. At the 505 * time of writing, this meant that their clock rates were the same, their 506 * feature sets were the same, but the required workaround were -not- 507 * necessarily the same. So, this routine is invoked on -every- CPU soon 508 * after starting to make sure that the resulting system contains the most 509 * pessimal set of workarounds needed to cope with *any* of the CPUs in the 510 * system. 511 * 512 * These workarounds are based on Rev 3.50 of the Revision Guide for 513 * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, May 2005. 514 */ 515 516 #if defined(OPTERON_ERRATUM_91) 517 int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ 518 #endif 519 520 #if defined(OPTERON_ERRATUM_93) 521 int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ 522 #endif 523 524 #if defined(OPTERON_ERRATUM_100) 525 int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ 526 #endif 527 528 #if defined(OPTERON_ERRATUM_109) 529 int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ 530 #endif 531 532 #if defined(OPTERON_ERRATUM_121) 533 int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ 534 #endif 535 536 #if defined(OPTERON_ERRATUM_122) 537 int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ 538 #endif 539 540 #if defined(OPTERON_ERRATUM_123) 541 int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ 542 #endif 543 544 545 #define WARNING(cpu, n) \ 546 cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d", \ 547 (cpu)->cpu_id, (n)) 548 549 uint_t 550 workaround_errata(struct cpu *cpu) 551 { 552 uint_t missing = 0; 553 554 ASSERT(cpu == CPU); 555 556 /*LINTED*/ 557 if (cpuid_opteron_erratum(cpu, 88) > 0) { 558 /* 559 * SWAPGS May Fail To Read Correct GS Base 560 */ 561 #if defined(OPTERON_ERRATUM_88) 562 /* 563 * The workaround is an mfence in the relevant assembler code 564 */ 565 #else 566 WARNING(cpu, 88); 567 missing++; 568 #endif 569 } 570 571 if (cpuid_opteron_erratum(cpu, 91) > 0) { 572 /* 573 * Software Prefetches May Report A Page Fault 574 */ 575 #if defined(OPTERON_ERRATUM_91) 576 /* 577 * fix is in trap.c 578 */ 579 opteron_erratum_91++; 580 #else 581 WARNING(cpu, 91); 582 missing++; 583 #endif 584 } 585 586 if (cpuid_opteron_erratum(cpu, 93) > 0) { 587 /* 588 * RSM Auto-Halt Restart Returns to Incorrect RIP 589 */ 590 #if defined(OPTERON_ERRATUM_93) 591 /* 592 * fix is in trap.c 593 */ 594 opteron_erratum_93++; 595 #else 596 WARNING(cpu, 93); 597 missing++; 598 #endif 599 } 600 601 /*LINTED*/ 602 if (cpuid_opteron_erratum(cpu, 95) > 0) { 603 /* 604 * RET Instruction May Return to Incorrect EIP 605 */ 606 #if defined(OPTERON_ERRATUM_95) 607 #if defined(_LP64) 608 /* 609 * Workaround this by ensuring that 32-bit user code and 610 * 64-bit kernel code never occupy the same address 611 * range mod 4G. 612 */ 613 if (_userlimit32 > 0xc0000000ul) 614 *(uintptr_t *)&_userlimit32 = 0xc0000000ul; 615 616 /*LINTED*/ 617 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); 618 #endif /* _LP64 */ 619 #else 620 WARNING(cpu, 95); 621 missing++; 622 #endif /* OPTERON_ERRATUM_95 */ 623 } 624 625 if (cpuid_opteron_erratum(cpu, 100) > 0) { 626 /* 627 * Compatibility Mode Branches Transfer to Illegal Address 628 */ 629 #if defined(OPTERON_ERRATUM_100) 630 /* 631 * fix is in trap.c 632 */ 633 opteron_erratum_100++; 634 #else 635 WARNING(cpu, 100); 636 missing++; 637 #endif 638 } 639 640 /*LINTED*/ 641 if (cpuid_opteron_erratum(cpu, 108) > 0) { 642 /* 643 * CPUID Instruction May Return Incorrect Model Number In 644 * Some Processors 645 */ 646 #if defined(OPTERON_ERRATUM_108) 647 /* 648 * (Our cpuid-handling code corrects the model number on 649 * those processors) 650 */ 651 #else 652 WARNING(cpu, 108); 653 missing++; 654 #endif 655 } 656 657 /*LINTED*/ 658 if (cpuid_opteron_erratum(cpu, 109) > 0) { 659 /* 660 * Certain Reverse REP MOVS May Produce Unpredictable Behaviour 661 */ 662 #if defined(OPTERON_ERRATUM_109) 663 uint64_t patchlevel; 664 665 (void) rdmsr(MSR_AMD_PATCHLEVEL, &patchlevel); 666 /* workaround is to print a warning to upgrade BIOS */ 667 if (patchlevel == 0) 668 opteron_erratum_109++; 669 #else 670 WARNING(cpu, 109); 671 missing++; 672 #endif 673 } 674 /*LINTED*/ 675 if (cpuid_opteron_erratum(cpu, 121) > 0) { 676 /* 677 * Sequential Execution Across Non_Canonical Boundary Caused 678 * Processor Hang 679 */ 680 #if defined(OPTERON_ERRATUM_121) 681 static int lma; 682 683 if (opteron_erratum_121) 684 opteron_erratum_121++; 685 686 /* 687 * Erratum 121 is only present in long (64 bit) mode. 688 * Workaround is to include the page immediately before the 689 * va hole to eliminate the possibility of system hangs due to 690 * sequential execution across the va hole boundary. 691 */ 692 if (lma == 0) { 693 uint64_t efer; 694 695 /* 696 * check LMA once: assume all cpus are in long mode 697 * or not. 698 */ 699 lma = 1; 700 701 (void) rdmsr(MSR_AMD_EFER, &efer); 702 if (efer & AMD_EFER_LMA) { 703 if (hole_start) { 704 hole_start -= PAGESIZE; 705 } else { 706 /* 707 * hole_start not yet initialized by 708 * mmu_init. Initialize hole_start 709 * with value to be subtracted. 710 */ 711 hole_start = PAGESIZE; 712 } 713 opteron_erratum_121++; 714 } 715 } 716 #else 717 WARNING(cpu, 121); 718 missing++; 719 #endif 720 } 721 722 /*LINTED*/ 723 if (cpuid_opteron_erratum(cpu, 122) > 0) { 724 /* 725 * TLB Flush Filter May Cause Cohenrency Problem in 726 * Multiprocessor Systems 727 */ 728 #if defined(OPTERON_ERRATUM_122) 729 /* 730 * Erratum 122 is only present in MP configurations (multi-core 731 * or multi-processor). 732 */ 733 734 if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 || 735 cpuid_get_ncpu_per_chip(cpu) > 1) { 736 uint64_t hwcrval; 737 738 /* disable TLB Flush Filter */ 739 (void) rdmsr(MSR_AMD_HWCR, &hwcrval); 740 hwcrval |= AMD_HWCR_FFDIS; 741 wrmsr(MSR_AMD_HWCR, &hwcrval); 742 opteron_erratum_122++; 743 } 744 745 #else 746 WARNING(cpu, 122); 747 missing++; 748 #endif 749 } 750 /*LINTED*/ 751 if (cpuid_opteron_erratum(cpu, 123) > 0) { 752 /* 753 * Bypassed Reads May Cause Data Corruption of System Hang in 754 * Dual Core Processors 755 */ 756 #if defined(OPTERON_ERRATUM_123) 757 /* 758 * Erratum 123 applies only to multi-core cpus. 759 */ 760 761 if (cpuid_get_ncpu_per_chip(cpu) > 1) { 762 uint64_t patchlevel; 763 764 (void) rdmsr(MSR_AMD_PATCHLEVEL, &patchlevel); 765 /* workaround is to print a warning to upgrade BIOS */ 766 if (patchlevel == 0) 767 opteron_erratum_123++; 768 } 769 #else 770 WARNING(cpu, 123); 771 missing++; 772 #endif 773 } 774 return (missing); 775 } 776 777 void 778 workaround_errata_end() 779 { 780 #if defined(OPTERON_ERRATUM_109) 781 if (opteron_erratum_109) { 782 cmn_err(CE_WARN, "!BIOS microcode patch for AMD Processor" 783 " Erratum 109 was not detected. Updating BIOS with the" 784 " microcode patch is highly recommended."); 785 } 786 #endif 787 #if defined(OPTERON_ERRATUM_123) 788 if (opteron_erratum_123) { 789 cmn_err(CE_WARN, "!BIOS microcode patch for AMD Processor" 790 " Erratum 123 was not detected. Updating BIOS with the" 791 " microcode patch is highly recommended."); 792 } 793 #endif 794 } 795 796 static ushort_t *mp_map_warm_reset_vector(); 797 static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector); 798 799 /*ARGSUSED*/ 800 void 801 start_other_cpus(int cprboot) 802 { 803 unsigned who; 804 int cpuid = getbootcpuid(); 805 int delays = 0; 806 int started_cpu; 807 ushort_t *warm_reset_vector = NULL; 808 extern int procset; 809 810 /* 811 * Initialize our own cpu_info. 812 */ 813 init_cpu_info(CPU); 814 815 /* 816 * Initialize our syscall handlers 817 */ 818 init_cpu_syscall(CPU); 819 820 /* 821 * if only 1 cpu or not using MP, skip the rest of this 822 */ 823 if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) { 824 if (use_mp == 0) 825 cmn_err(CE_CONT, "?***** Not in MP mode\n"); 826 goto done; 827 } 828 829 /* 830 * perform such initialization as is needed 831 * to be able to take CPUs on- and off-line. 832 */ 833 cpu_pause_init(); 834 835 xc_init(); /* initialize processor crosscalls */ 836 837 /* 838 * Copy the real mode code at "real_mode_start" to the 839 * page at rm_platter_va. 840 */ 841 warm_reset_vector = mp_map_warm_reset_vector(); 842 if (warm_reset_vector == NULL) 843 goto done; 844 845 bcopy((caddr_t)real_mode_start, 846 (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code, 847 (size_t)real_mode_end - (size_t)real_mode_start); 848 849 flushes_require_xcalls = 1; 850 851 affinity_set(CPU_CURRENT); 852 853 for (who = 0; who < NCPU; who++) { 854 if (who == cpuid) 855 continue; 856 857 if ((mp_cpus & (1 << who)) == 0) 858 continue; 859 860 mp_startup_init(who); 861 started_cpu = 1; 862 (*cpu_startf)(who, rm_platter_pa); 863 864 while ((procset & (1 << who)) == 0) { 865 866 delay(1); 867 if (++delays > (20 * hz)) { 868 869 cmn_err(CE_WARN, 870 "cpu%d failed to start", who); 871 872 mutex_enter(&cpu_lock); 873 cpu[who]->cpu_flags = 0; 874 cpu_del_unit(who); 875 mutex_exit(&cpu_lock); 876 877 started_cpu = 0; 878 break; 879 } 880 } 881 if (!started_cpu) 882 continue; 883 if (tsc_gethrtime_enable) 884 tsc_sync_master(who); 885 886 887 if (dtrace_cpu_init != NULL) { 888 /* 889 * DTrace CPU initialization expects cpu_lock 890 * to be held. 891 */ 892 mutex_enter(&cpu_lock); 893 (*dtrace_cpu_init)(who); 894 mutex_exit(&cpu_lock); 895 } 896 } 897 898 affinity_clear(); 899 900 for (who = 0; who < NCPU; who++) { 901 if (who == cpuid) 902 continue; 903 904 if (!(procset & (1 << who))) 905 continue; 906 907 while (!(cpu_ready_set & (1 << who))) 908 delay(1); 909 } 910 911 done: 912 workaround_errata_end(); 913 914 if (warm_reset_vector != NULL) 915 mp_unmap_warm_reset_vector(warm_reset_vector); 916 hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 917 HAT_UNLOAD); 918 } 919 920 /* 921 * Dummy functions - no i86pc platforms support dynamic cpu allocation. 922 */ 923 /*ARGSUSED*/ 924 int 925 mp_cpu_configure(int cpuid) 926 { 927 return (ENOTSUP); /* not supported */ 928 } 929 930 /*ARGSUSED*/ 931 int 932 mp_cpu_unconfigure(int cpuid) 933 { 934 return (ENOTSUP); /* not supported */ 935 } 936 937 /* 938 * Startup function for 'other' CPUs (besides boot cpu). 939 * Resumed from cpu_startup. 940 */ 941 void 942 mp_startup(void) 943 { 944 struct cpu *cp = CPU; 945 extern int procset; 946 uint_t new_x86_feature; 947 948 new_x86_feature = cpuid_pass1(cp); 949 950 /* 951 * We need to Sync MTRR with cpu0's MTRR. We have to do 952 * this with interrupts disabled. 953 */ 954 if (x86_feature & X86_MTRR) 955 mtrr_sync(); 956 /* 957 * Enable machine check architecture 958 */ 959 if (x86_feature & X86_MCA) 960 setup_mca(); 961 962 /* 963 * Initialize this CPU's syscall handlers 964 */ 965 init_cpu_syscall(cp); 966 967 /* 968 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the 969 * highest level at which a routine is permitted to block on 970 * an adaptive mutex (allows for cpu poke interrupt in case 971 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks 972 * device interrupts that may end up in the hat layer issuing cross 973 * calls before CPU_READY is set. 974 */ 975 (void) splx(ipltospl(LOCK_LEVEL)); 976 977 /* 978 * Do a sanity check to make sure this new CPU is a sane thing 979 * to add to the collection of processors running this system. 980 * 981 * XXX Clearly this needs to get more sophisticated, if x86 982 * systems start to get built out of heterogenous CPUs; as is 983 * likely to happen once the number of processors in a configuration 984 * gets large enough. 985 */ 986 if ((x86_feature & new_x86_feature) != x86_feature) { 987 cmn_err(CE_CONT, "?cpu%d: %b\n", 988 cp->cpu_id, new_x86_feature, FMT_X86_FEATURE); 989 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); 990 } 991 992 /* 993 * We could be more sophisticated here, and just mark the CPU 994 * as "faulted" but at this point we'll opt for the easier 995 * answer of dieing horribly. Provided the boot cpu is ok, 996 * the system can be recovered by booting with use_mp set to zero. 997 */ 998 if (workaround_errata(cp) != 0) 999 panic("critical workaround(s) missing for cpu%d", cp->cpu_id); 1000 1001 cpuid_pass2(cp); 1002 cpuid_pass3(cp); 1003 (void) cpuid_pass4(cp); 1004 1005 init_cpu_info(cp); 1006 1007 add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi); 1008 1009 mutex_enter(&cpu_lock); 1010 procset |= 1 << cp->cpu_id; 1011 mutex_exit(&cpu_lock); 1012 1013 if (tsc_gethrtime_enable) 1014 tsc_sync_slave(); 1015 1016 mutex_enter(&cpu_lock); 1017 /* 1018 * It's unfortunate that chip_cpu_init() has to be called here. 1019 * It really belongs in cpu_add_unit(), but unfortunately it is 1020 * dependent on the cpuid probing, which must be done in the 1021 * context of the current CPU. Care must be taken on x86 to ensure 1022 * that mp_startup can safely block even though chip_cpu_init() and 1023 * cpu_add_active() have not yet been called. 1024 */ 1025 chip_cpu_init(cp); 1026 chip_cpu_startup(cp); 1027 1028 cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS; 1029 cpu_add_active(cp); 1030 mutex_exit(&cpu_lock); 1031 1032 (void) spl0(); /* enable interrupts */ 1033 1034 if (boothowto & RB_DEBUG) 1035 kdi_dvec_cpu_init(cp); 1036 1037 /* 1038 * Setting the bit in cpu_ready_set must be the last operation in 1039 * processor initialization; the boot CPU will continue to boot once 1040 * it sees this bit set for all active CPUs. 1041 */ 1042 CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); 1043 1044 /* 1045 * Because mp_startup() gets fired off after init() starts, we 1046 * can't use the '?' trick to do 'boot -v' printing - so we 1047 * always direct the 'cpu .. online' messages to the log. 1048 */ 1049 cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 1050 cp->cpu_id); 1051 1052 /* 1053 * Now we are done with the startup thread, so free it up. 1054 */ 1055 thread_exit(); 1056 panic("mp_startup: cannot return"); 1057 /*NOTREACHED*/ 1058 } 1059 1060 1061 /* 1062 * Start CPU on user request. 1063 */ 1064 /* ARGSUSED */ 1065 int 1066 mp_cpu_start(struct cpu *cp) 1067 { 1068 ASSERT(MUTEX_HELD(&cpu_lock)); 1069 if (cp->cpu_id == getbootcpuid()) 1070 return (EBUSY); /* Cannot start boot CPU */ 1071 return (0); 1072 } 1073 1074 /* 1075 * Stop CPU on user request. 1076 */ 1077 /* ARGSUSED */ 1078 int 1079 mp_cpu_stop(struct cpu *cp) 1080 { 1081 ASSERT(MUTEX_HELD(&cpu_lock)); 1082 if (cp->cpu_id == getbootcpuid()) 1083 return (EBUSY); /* Cannot stop boot CPU */ 1084 1085 return (0); 1086 } 1087 1088 /* 1089 * Power on CPU. 1090 */ 1091 /* ARGSUSED */ 1092 int 1093 mp_cpu_poweron(struct cpu *cp) 1094 { 1095 ASSERT(MUTEX_HELD(&cpu_lock)); 1096 return (ENOTSUP); /* not supported */ 1097 } 1098 1099 /* 1100 * Power off CPU. 1101 */ 1102 /* ARGSUSED */ 1103 int 1104 mp_cpu_poweroff(struct cpu *cp) 1105 { 1106 ASSERT(MUTEX_HELD(&cpu_lock)); 1107 return (ENOTSUP); /* not supported */ 1108 } 1109 1110 1111 /* 1112 * Take the specified CPU out of participation in interrupts. 1113 */ 1114 int 1115 cpu_disable_intr(struct cpu *cp) 1116 { 1117 /* 1118 * cannot disable interrupts on boot cpu 1119 */ 1120 if (cp == cpu[getbootcpuid()]) 1121 return (EBUSY); 1122 1123 if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) 1124 return (EBUSY); 1125 1126 cp->cpu_flags &= ~CPU_ENABLE; 1127 return (0); 1128 } 1129 1130 /* 1131 * Allow the specified CPU to participate in interrupts. 1132 */ 1133 void 1134 cpu_enable_intr(struct cpu *cp) 1135 { 1136 ASSERT(MUTEX_HELD(&cpu_lock)); 1137 if (cp == cpu[getbootcpuid()]) 1138 return; 1139 1140 cp->cpu_flags |= CPU_ENABLE; 1141 psm_enable_intr(cp->cpu_id); 1142 } 1143 1144 1145 /* 1146 * return the cpu id of the initial startup cpu 1147 */ 1148 processorid_t 1149 getbootcpuid(void) 1150 { 1151 return (0); 1152 } 1153 1154 static ushort_t * 1155 mp_map_warm_reset_vector() 1156 { 1157 ushort_t *warm_reset_vector; 1158 1159 if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 1160 sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 1161 return (NULL); 1162 1163 /* 1164 * setup secondary cpu bios boot up vector 1165 */ 1166 *warm_reset_vector = (ushort_t)((caddr_t) 1167 ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 1168 + ((ulong_t)rm_platter_va & 0xf)); 1169 warm_reset_vector++; 1170 *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 1171 1172 --warm_reset_vector; 1173 return (warm_reset_vector); 1174 } 1175 1176 static void 1177 mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector) 1178 { 1179 psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 1180 } 1181 1182 /*ARGSUSED*/ 1183 void 1184 mp_cpu_faulted_enter(struct cpu *cp) 1185 {} 1186 1187 /*ARGSUSED*/ 1188 void 1189 mp_cpu_faulted_exit(struct cpu *cp) 1190 {} 1191 1192 /* 1193 * The following two routines are used as context operators on threads belonging 1194 * to processes with a private LDT (see sysi86). Due to the rarity of such 1195 * processes, these routines are currently written for best code readability and 1196 * organization rather than speed. We could avoid checking x86_feature at every 1197 * context switch by installing different context ops, depending on the 1198 * x86_feature flags, at LDT creation time -- one for each combination of fast 1199 * syscall feature flags. 1200 */ 1201 1202 /*ARGSUSED*/ 1203 void 1204 cpu_fast_syscall_disable(void *arg) 1205 { 1206 if (x86_feature & X86_SEP) 1207 cpu_sep_disable(); 1208 if (x86_feature & X86_ASYSC) 1209 cpu_asysc_disable(); 1210 } 1211 1212 /*ARGSUSED*/ 1213 void 1214 cpu_fast_syscall_enable(void *arg) 1215 { 1216 if (x86_feature & X86_SEP) 1217 cpu_sep_enable(); 1218 if (x86_feature & X86_ASYSC) 1219 cpu_asysc_enable(); 1220 } 1221 1222 static void 1223 cpu_sep_enable(void) 1224 { 1225 uint64_t value; 1226 1227 ASSERT(x86_feature & X86_SEP); 1228 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1229 1230 value = KCS_SEL; 1231 wrmsr(MSR_INTC_SEP_CS, &value); 1232 } 1233 1234 static void 1235 cpu_sep_disable(void) 1236 { 1237 uint64_t value; 1238 1239 ASSERT(x86_feature & X86_SEP); 1240 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1241 1242 /* 1243 * Setting the SYSENTER_CS_MSR register to 0 causes software executing 1244 * the sysenter or sysexit instruction to trigger a #gp fault. 1245 */ 1246 value = 0; 1247 wrmsr(MSR_INTC_SEP_CS, &value); 1248 } 1249 1250 static void 1251 cpu_asysc_enable(void) 1252 { 1253 uint64_t value; 1254 1255 ASSERT(x86_feature & X86_ASYSC); 1256 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1257 1258 (void) rdmsr(MSR_AMD_EFER, &value); 1259 value |= AMD_EFER_SCE; 1260 wrmsr(MSR_AMD_EFER, &value); 1261 } 1262 1263 static void 1264 cpu_asysc_disable(void) 1265 { 1266 uint64_t value; 1267 1268 ASSERT(x86_feature & X86_ASYSC); 1269 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1270 1271 /* 1272 * Turn off the SCE (syscall enable) bit in the EFER register. Software 1273 * executing syscall or sysret with this bit off will incur a #ud trap. 1274 */ 1275 (void) rdmsr(MSR_AMD_EFER, &value); 1276 value &= ~AMD_EFER_SCE; 1277 wrmsr(MSR_AMD_EFER, &value); 1278 } 1279