1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/thread.h> 31 #include <sys/cpuvar.h> 32 #include <sys/t_lock.h> 33 #include <sys/param.h> 34 #include <sys/proc.h> 35 #include <sys/disp.h> 36 #include <sys/mmu.h> 37 #include <sys/class.h> 38 #include <sys/cmn_err.h> 39 #include <sys/debug.h> 40 #include <sys/asm_linkage.h> 41 #include <sys/x_call.h> 42 #include <sys/systm.h> 43 #include <sys/var.h> 44 #include <sys/vtrace.h> 45 #include <vm/hat.h> 46 #include <sys/mmu.h> 47 #include <vm/as.h> 48 #include <vm/seg_kmem.h> 49 #include <sys/segments.h> 50 #include <sys/kmem.h> 51 #include <sys/stack.h> 52 #include <sys/smp_impldefs.h> 53 #include <sys/x86_archext.h> 54 #include <sys/machsystm.h> 55 #include <sys/traptrace.h> 56 #include <sys/clock.h> 57 #include <sys/cpc_impl.h> 58 #include <sys/chip.h> 59 #include <sys/dtrace.h> 60 #include <sys/archsystm.h> 61 #include <sys/fp.h> 62 #include <sys/reboot.h> 63 #include <sys/kdi.h> 64 #include <vm/hat_i86.h> 65 #include <sys/memnode.h> 66 67 struct cpu cpus[1]; /* CPU data */ 68 struct cpu *cpu[NCPU] = {&cpus[0]}; /* pointers to all CPUs */ 69 cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 70 71 /* 72 * Useful for disabling MP bring-up for an MP capable kernel 73 * (a kernel that was built with MP defined) 74 */ 75 int use_mp = 1; 76 77 int mp_cpus = 0x1; /* to be set by platform specific module */ 78 79 /* 80 * This variable is used by the hat layer to decide whether or not 81 * critical sections are needed to prevent race conditions. For sun4m, 82 * this variable is set once enough MP initialization has been done in 83 * order to allow cross calls. 84 */ 85 int flushes_require_xcalls = 0; 86 ulong_t cpu_ready_set = 1; 87 88 extern void real_mode_start(void); 89 extern void real_mode_end(void); 90 static void mp_startup(void); 91 92 static void cpu_sep_enable(void); 93 static void cpu_sep_disable(void); 94 static void cpu_asysc_enable(void); 95 static void cpu_asysc_disable(void); 96 97 extern int tsc_gethrtime_enable; 98 99 /* 100 * Init CPU info - get CPU type info for processor_info system call. 101 */ 102 void 103 init_cpu_info(struct cpu *cp) 104 { 105 processor_info_t *pi = &cp->cpu_type_info; 106 char buf[CPU_IDSTRLEN]; 107 108 /* 109 * Get clock-frequency property for the CPU. 110 */ 111 pi->pi_clock = cpu_freq; 112 113 (void) strcpy(pi->pi_processor_type, "i386"); 114 if (fpu_exists) 115 (void) strcpy(pi->pi_fputypes, "i387 compatible"); 116 117 (void) cpuid_getidstr(cp, buf, sizeof (buf)); 118 119 cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 120 (void) strcpy(cp->cpu_idstr, buf); 121 122 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); 123 124 (void) cpuid_getbrandstr(cp, buf, sizeof (buf)); 125 cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 126 (void) strcpy(cp->cpu_brandstr, buf); 127 128 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); 129 } 130 131 /* 132 * Configure syscall support on this CPU. 133 */ 134 /*ARGSUSED*/ 135 static void 136 init_cpu_syscall(struct cpu *cp) 137 { 138 uint64_t value; 139 140 kpreempt_disable(); 141 142 #if defined(__amd64) 143 if (x86_feature & X86_ASYSC) { 144 145 #if !defined(__lint) 146 /* 147 * The syscall instruction imposes a certain ordering on 148 * segment selectors, so we double-check that ordering 149 * here. 150 */ 151 ASSERT(KDS_SEL == KCS_SEL + 8); 152 ASSERT(UDS_SEL == U32CS_SEL + 8); 153 ASSERT(UCS_SEL == U32CS_SEL + 16); 154 #endif 155 /* 156 * Turn syscall/sysret extensions on. 157 */ 158 cpu_asysc_enable(); 159 160 /* 161 * Program the magic registers .. 162 */ 163 value = ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32; 164 wrmsr(MSR_AMD_STAR, &value); 165 value = (uintptr_t)sys_syscall; 166 wrmsr(MSR_AMD_LSTAR, &value); 167 value = (uintptr_t)sys_syscall32; 168 wrmsr(MSR_AMD_CSTAR, &value); 169 170 /* 171 * This list of flags is masked off the incoming 172 * %rfl when we enter the kernel. 173 */ 174 value = PS_IE | PS_T; 175 wrmsr(MSR_AMD_SFMASK, &value); 176 } 177 #endif 178 179 /* 180 * On 32-bit kernels, we use sysenter/sysexit because it's too 181 * hard to use syscall/sysret, and it is more portable anyway. 182 * 183 * On 64-bit kernels on Nocona machines, the 32-bit syscall 184 * variant isn't available to 32-bit applications, but sysenter is. 185 */ 186 if (x86_feature & X86_SEP) { 187 188 #if !defined(__lint) 189 /* 190 * The sysenter instruction imposes a certain ordering on 191 * segment selectors, so we double-check that ordering 192 * here. See "sysenter" in Intel document 245471-012, "IA-32 193 * Intel Architecture Software Developer's Manual Volume 2: 194 * Instruction Set Reference" 195 */ 196 ASSERT(KDS_SEL == KCS_SEL + 8); 197 198 ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3)); 199 ASSERT32(UDS_SEL == UCS_SEL + 8); 200 201 ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3)); 202 ASSERT64(UDS_SEL == U32CS_SEL + 8); 203 #endif 204 205 cpu_sep_enable(); 206 207 /* 208 * resume() sets this value to the base of the threads stack 209 * via a context handler. 210 */ 211 value = 0; 212 wrmsr(MSR_INTC_SEP_ESP, &value); 213 214 value = (uintptr_t)sys_sysenter; 215 wrmsr(MSR_INTC_SEP_EIP, &value); 216 } 217 218 kpreempt_enable(); 219 } 220 221 /* 222 * Multiprocessor initialization. 223 * 224 * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the 225 * startup and idle threads for the specified CPU. 226 */ 227 static void 228 mp_startup_init(int cpun) 229 { 230 #if defined(__amd64) 231 extern void *long_mode_64(void); 232 #endif /* __amd64 */ 233 234 struct cpu *cp; 235 struct tss *ntss; 236 kthread_id_t tp; 237 caddr_t sp; 238 int size; 239 proc_t *procp; 240 extern void idle(); 241 extern void init_intr_threads(struct cpu *); 242 243 struct cpu_tables *tablesp; 244 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 245 246 #ifdef TRAPTRACE 247 trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; 248 #endif 249 250 ASSERT(cpun < NCPU && cpu[cpun] == NULL); 251 252 if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) { 253 panic("mp_startup_init: cpu%d: " 254 "no memory for cpu structure", cpun); 255 /*NOTREACHED*/ 256 } 257 procp = curthread->t_procp; 258 259 mutex_enter(&cpu_lock); 260 /* 261 * Initialize the dispatcher first. 262 */ 263 disp_cpu_init(cp); 264 mutex_exit(&cpu_lock); 265 266 /* 267 * Allocate and initialize the startup thread for this CPU. 268 * Interrupt and process switch stacks get allocated later 269 * when the CPU starts running. 270 */ 271 tp = thread_create(NULL, 0, NULL, NULL, 0, procp, 272 TS_STOPPED, maxclsyspri); 273 274 /* 275 * Set state to TS_ONPROC since this thread will start running 276 * as soon as the CPU comes online. 277 * 278 * All the other fields of the thread structure are setup by 279 * thread_create(). 280 */ 281 THREAD_ONPROC(tp, cp); 282 tp->t_preempt = 1; 283 tp->t_bound_cpu = cp; 284 tp->t_affinitycnt = 1; 285 tp->t_cpu = cp; 286 tp->t_disp_queue = cp->cpu_disp; 287 288 /* 289 * Setup thread to start in mp_startup. 290 */ 291 sp = tp->t_stk; 292 tp->t_pc = (uintptr_t)mp_startup; 293 tp->t_sp = (uintptr_t)(sp - MINFRAME); 294 295 cp->cpu_id = cpun; 296 cp->cpu_self = cp; 297 cp->cpu_mask = 1 << cpun; 298 cp->cpu_thread = tp; 299 cp->cpu_lwp = NULL; 300 cp->cpu_dispthread = tp; 301 cp->cpu_dispatch_pri = DISP_PRIO(tp); 302 303 /* 304 * Now, initialize per-CPU idle thread for this CPU. 305 */ 306 tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); 307 308 cp->cpu_idle_thread = tp; 309 310 tp->t_preempt = 1; 311 tp->t_bound_cpu = cp; 312 tp->t_affinitycnt = 1; 313 tp->t_cpu = cp; 314 tp->t_disp_queue = cp->cpu_disp; 315 316 /* 317 * Bootstrap the CPU for CMT aware scheduling 318 * The rest of the initialization will happen from 319 * mp_startup() 320 */ 321 chip_bootstrap_cpu(cp); 322 323 /* 324 * Perform CPC intialization on the new CPU. 325 */ 326 kcpc_hw_init(cp); 327 328 /* 329 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 330 * for each CPU. 331 */ 332 333 setup_vaddr_for_ppcopy(cp); 334 335 /* 336 * Allocate space for page directory, stack, tss, gdt and idt. 337 * This assumes that kmem_alloc will return memory which is aligned 338 * to the next higher power of 2 or a page(if size > MAXABIG) 339 * If this assumption goes wrong at any time due to change in 340 * kmem alloc, things may not work as the page directory has to be 341 * page aligned 342 */ 343 if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL) 344 panic("mp_startup_init: cpu%d cannot allocate tables", cpun); 345 346 if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) { 347 kmem_free(tablesp, sizeof (struct cpu_tables)); 348 size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE; 349 tablesp = kmem_zalloc(size, KM_NOSLEEP); 350 tablesp = (struct cpu_tables *) 351 (((uintptr_t)tablesp + MMU_STD_PAGESIZE) & 352 MMU_STD_PAGEMASK); 353 } 354 355 ntss = cp->cpu_tss = &tablesp->ct_tss; 356 cp->cpu_gdt = tablesp->ct_gdt; 357 bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t))); 358 359 #if defined(__amd64) 360 361 /* 362 * #DF (double fault). 363 */ 364 ntss->tss_ist1 = 365 (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 366 367 #elif defined(__i386) 368 369 ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp = 370 (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 371 372 ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL; 373 374 ntss->tss_eip = (uint32_t)mp_startup; 375 376 ntss->tss_cs = KCS_SEL; 377 ntss->tss_fs = KFS_SEL; 378 ntss->tss_gs = KGS_SEL; 379 380 /* 381 * setup kernel %gs. 382 */ 383 set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA, 384 SEL_KPL, 0, 1); 385 386 #endif /* __i386 */ 387 388 /* 389 * Set I/O bit map offset equal to size of TSS segment limit 390 * for no I/O permission map. This will cause all user I/O 391 * instructions to generate #gp fault. 392 */ 393 ntss->tss_bitmapbase = sizeof (*ntss); 394 395 /* 396 * setup kernel tss. 397 */ 398 set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, 399 sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL); 400 401 /* 402 * If we have more than one node, each cpu gets a copy of IDT 403 * local to its node. If this is a Pentium box, we use cpu 0's 404 * IDT. cpu 0's IDT has been made read-only to workaround the 405 * cmpxchgl register bug 406 */ 407 cp->cpu_idt = CPU->cpu_idt; 408 if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) { 409 cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP); 410 bcopy(idt0, cp->cpu_idt, sizeof (idt0)); 411 } 412 413 /* 414 * Get interrupt priority data from cpu 0 415 */ 416 cp->cpu_pri_data = CPU->cpu_pri_data; 417 418 hat_cpu_online(cp); 419 420 /* Should remove all entries for the current process/thread here */ 421 422 /* 423 * Fill up the real mode platter to make it easy for real mode code to 424 * kick it off. This area should really be one passed by boot to kernel 425 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 426 * have identical physical and virtual address in paged mode. 427 */ 428 real_mode_platter->rm_idt_base = cp->cpu_idt; 429 real_mode_platter->rm_idt_lim = sizeof (idt0) - 1; 430 real_mode_platter->rm_gdt_base = cp->cpu_gdt; 431 real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1; 432 real_mode_platter->rm_pdbr = getcr3(); 433 real_mode_platter->rm_cpu = cpun; 434 real_mode_platter->rm_x86feature = x86_feature; 435 real_mode_platter->rm_cr4 = cr4_value; 436 437 #if defined(__amd64) 438 if (getcr3() > 0xffffffffUL) 439 panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 440 "located above 4G in physical memory (@ 0x%llx).", 441 (unsigned long long)getcr3()); 442 443 /* 444 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 445 * by code in real_mode_start(): 446 * 447 * GDT[0]: NULL selector 448 * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 449 * 450 * Clear the IDT as interrupts will be off and a limit of 0 will cause 451 * the CPU to triple fault and reset on an NMI, seemingly as reasonable 452 * a course of action as any other, though it may cause the entire 453 * platform to reset in some cases... 454 */ 455 real_mode_platter->rm_temp_gdt[0] = 0ULL; 456 real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 457 458 real_mode_platter->rm_temp_gdt_lim = (ushort_t) 459 (sizeof (real_mode_platter->rm_temp_gdt) - 1); 460 real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 461 (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt); 462 463 real_mode_platter->rm_temp_idt_lim = 0; 464 real_mode_platter->rm_temp_idt_base = 0; 465 466 /* 467 * Since the CPU needs to jump to protected mode using an identity 468 * mapped address, we need to calculate it here. 469 */ 470 real_mode_platter->rm_longmode64_addr = rm_platter_pa + 471 ((uint32_t)long_mode_64 - (uint32_t)real_mode_start); 472 #endif /* __amd64 */ 473 474 #ifdef TRAPTRACE 475 /* 476 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this 477 * CPU. 478 */ 479 ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); 480 ttc->ttc_next = ttc->ttc_first; 481 ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; 482 #endif 483 484 /* 485 * Record that we have another CPU. 486 */ 487 mutex_enter(&cpu_lock); 488 /* 489 * Initialize the interrupt threads for this CPU 490 */ 491 init_intr_threads(cp); 492 /* 493 * Add CPU to list of available CPUs. It'll be on the active list 494 * after mp_startup(). 495 */ 496 cpu_add_unit(cp); 497 mutex_exit(&cpu_lock); 498 } 499 500 /* 501 * Apply workarounds for known errata, and warn about those that are absent. 502 * 503 * System vendors occasionally create configurations which contain different 504 * revisions of the CPUs that are almost but not exactly the same. At the 505 * time of writing, this meant that their clock rates were the same, their 506 * feature sets were the same, but the required workaround were -not- 507 * necessarily the same. So, this routine is invoked on -every- CPU soon 508 * after starting to make sure that the resulting system contains the most 509 * pessimal set of workarounds needed to cope with *any* of the CPUs in the 510 * system. 511 * 512 * These workarounds are based on Rev 3.50 of the Revision Guide for 513 * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, May 2005. 514 */ 515 516 #if defined(OPTERON_ERRATUM_91) 517 int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ 518 #endif 519 520 #if defined(OPTERON_ERRATUM_93) 521 int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ 522 #endif 523 524 #if defined(OPTERON_ERRATUM_100) 525 int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ 526 #endif 527 528 #if defined(OPTERON_ERRATUM_109) 529 int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ 530 #endif 531 532 #if defined(OPTERON_ERRATUM_121) 533 int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ 534 #endif 535 536 #if defined(OPTERON_ERRATUM_122) 537 int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ 538 #endif 539 540 #if defined(OPTERON_ERRATUM_123) 541 int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ 542 #endif 543 544 545 #define WARNING(cpu, n) \ 546 cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d", \ 547 (cpu)->cpu_id, (n)) 548 549 uint_t 550 workaround_errata(struct cpu *cpu) 551 { 552 uint_t missing = 0; 553 554 ASSERT(cpu == CPU); 555 556 /*LINTED*/ 557 if (cpuid_opteron_erratum(cpu, 88) > 0) { 558 /* 559 * SWAPGS May Fail To Read Correct GS Base 560 */ 561 #if defined(OPTERON_ERRATUM_88) 562 /* 563 * The workaround is an mfence in the relevant assembler code 564 */ 565 #else 566 WARNING(cpu, 88); 567 missing++; 568 #endif 569 } 570 571 if (cpuid_opteron_erratum(cpu, 91) > 0) { 572 /* 573 * Software Prefetches May Report A Page Fault 574 */ 575 #if defined(OPTERON_ERRATUM_91) 576 /* 577 * fix is in trap.c 578 */ 579 opteron_erratum_91++; 580 #else 581 WARNING(cpu, 91); 582 missing++; 583 #endif 584 } 585 586 if (cpuid_opteron_erratum(cpu, 93) > 0) { 587 /* 588 * RSM Auto-Halt Restart Returns to Incorrect RIP 589 */ 590 #if defined(OPTERON_ERRATUM_93) 591 /* 592 * fix is in trap.c 593 */ 594 opteron_erratum_93++; 595 #else 596 WARNING(cpu, 93); 597 missing++; 598 #endif 599 } 600 601 /*LINTED*/ 602 if (cpuid_opteron_erratum(cpu, 95) > 0) { 603 /* 604 * RET Instruction May Return to Incorrect EIP 605 */ 606 #if defined(OPTERON_ERRATUM_95) 607 #if defined(_LP64) 608 /* 609 * Workaround this by ensuring that 32-bit user code and 610 * 64-bit kernel code never occupy the same address 611 * range mod 4G. 612 */ 613 if (_userlimit32 > 0xc0000000ul) 614 *(uintptr_t *)&_userlimit32 = 0xc0000000ul; 615 616 /*LINTED*/ 617 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); 618 #endif /* _LP64 */ 619 #else 620 WARNING(cpu, 95); 621 missing++; 622 #endif /* OPTERON_ERRATUM_95 */ 623 } 624 625 if (cpuid_opteron_erratum(cpu, 100) > 0) { 626 /* 627 * Compatibility Mode Branches Transfer to Illegal Address 628 */ 629 #if defined(OPTERON_ERRATUM_100) 630 /* 631 * fix is in trap.c 632 */ 633 opteron_erratum_100++; 634 #else 635 WARNING(cpu, 100); 636 missing++; 637 #endif 638 } 639 640 /*LINTED*/ 641 if (cpuid_opteron_erratum(cpu, 108) > 0) { 642 /* 643 * CPUID Instruction May Return Incorrect Model Number In 644 * Some Processors 645 */ 646 #if defined(OPTERON_ERRATUM_108) 647 /* 648 * (Our cpuid-handling code corrects the model number on 649 * those processors) 650 */ 651 #else 652 WARNING(cpu, 108); 653 missing++; 654 #endif 655 } 656 657 /*LINTED*/ 658 if (cpuid_opteron_erratum(cpu, 109) > 0) { 659 /* 660 * Certain Reverse REP MOVS May Produce Unpredictable Behaviour 661 */ 662 #if defined(OPTERON_ERRATUM_109) 663 uint64_t patchlevel; 664 665 (void) rdmsr(MSR_AMD_PATCHLEVEL, &patchlevel); 666 /* workaround is to print a warning to upgrade BIOS */ 667 if (patchlevel == 0) 668 opteron_erratum_109++; 669 #else 670 WARNING(cpu, 109); 671 missing++; 672 #endif 673 } 674 /*LINTED*/ 675 if (cpuid_opteron_erratum(cpu, 121) > 0) { 676 /* 677 * Sequential Execution Across Non_Canonical Boundary Caused 678 * Processor Hang 679 */ 680 #if defined(OPTERON_ERRATUM_121) 681 static int lma; 682 683 if (opteron_erratum_121) 684 opteron_erratum_121++; 685 686 /* 687 * Erratum 121 is only present in long (64 bit) mode. 688 * Workaround is to include the page immediately before the 689 * va hole to eliminate the possibility of system hangs due to 690 * sequential execution across the va hole boundary. 691 */ 692 if (lma == 0) { 693 uint64_t efer; 694 695 /* 696 * check LMA once: assume all cpus are in long mode 697 * or not. 698 */ 699 lma = 1; 700 701 (void) rdmsr(MSR_AMD_EFER, &efer); 702 if (efer & AMD_EFER_LMA) { 703 if (hole_start) { 704 hole_start -= PAGESIZE; 705 } else { 706 /* 707 * hole_start not yet initialized by 708 * mmu_init. Initialize hole_start 709 * with value to be subtracted. 710 */ 711 hole_start = PAGESIZE; 712 } 713 opteron_erratum_121++; 714 } 715 } 716 #else 717 WARNING(cpu, 121); 718 missing++; 719 #endif 720 } 721 722 /*LINTED*/ 723 if (cpuid_opteron_erratum(cpu, 122) > 0) { 724 /* 725 * TLB Flush Filter May Cause Cohenrency Problem in 726 * Multiprocessor Systems 727 */ 728 #if defined(OPTERON_ERRATUM_122) 729 /* 730 * Erratum 122 is only present in MP configurations (multi-core 731 * or multi-processor). 732 */ 733 734 if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 || 735 cpuid_get_ncpu_per_chip(cpu) > 1) { 736 uint64_t hwcrval; 737 738 /* disable TLB Flush Filter */ 739 (void) rdmsr(MSR_AMD_HWCR, &hwcrval); 740 hwcrval |= AMD_HWCR_FFDIS; 741 wrmsr(MSR_AMD_HWCR, &hwcrval); 742 opteron_erratum_122++; 743 } 744 745 #else 746 WARNING(cpu, 122); 747 missing++; 748 #endif 749 } 750 751 #if defined(OPTERON_ERRATUM_123) 752 /*LINTED*/ 753 if (cpuid_opteron_erratum(cpu, 123) > 0) { 754 /* 755 * Bypassed Reads May Cause Data Corruption of System Hang in 756 * Dual Core Processors 757 */ 758 /* 759 * Erratum 123 applies only to multi-core cpus. 760 */ 761 762 if (cpuid_get_ncpu_per_chip(cpu) > 1) { 763 uint64_t patchlevel; 764 765 (void) rdmsr(MSR_AMD_PATCHLEVEL, &patchlevel); 766 /* workaround is to print a warning to upgrade BIOS */ 767 if (patchlevel == 0) 768 opteron_erratum_123++; 769 } 770 } 771 #endif 772 return (missing); 773 } 774 775 void 776 workaround_errata_end() 777 { 778 #if defined(OPTERON_ERRATUM_109) 779 if (opteron_erratum_109) { 780 cmn_err(CE_WARN, "!BIOS microcode patch for AMD Processor" 781 " Erratum 109 was not detected. Updating BIOS with the" 782 " microcode patch is highly recommended."); 783 } 784 #endif 785 #if defined(OPTERON_ERRATUM_123) 786 if (opteron_erratum_123) { 787 cmn_err(CE_WARN, "!BIOS microcode patch for AMD Processor" 788 " Erratum 123 was not detected. Updating BIOS with the" 789 " microcode patch is highly recommended."); 790 } 791 #endif 792 } 793 794 static ushort_t *mp_map_warm_reset_vector(); 795 static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector); 796 797 /*ARGSUSED*/ 798 void 799 start_other_cpus(int cprboot) 800 { 801 unsigned who; 802 int cpuid = getbootcpuid(); 803 int delays = 0; 804 int started_cpu; 805 ushort_t *warm_reset_vector = NULL; 806 extern int procset; 807 808 /* 809 * Initialize our own cpu_info. 810 */ 811 init_cpu_info(CPU); 812 813 /* 814 * Initialize our syscall handlers 815 */ 816 init_cpu_syscall(CPU); 817 818 /* 819 * if only 1 cpu or not using MP, skip the rest of this 820 */ 821 if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) { 822 if (use_mp == 0) 823 cmn_err(CE_CONT, "?***** Not in MP mode\n"); 824 goto done; 825 } 826 827 /* 828 * perform such initialization as is needed 829 * to be able to take CPUs on- and off-line. 830 */ 831 cpu_pause_init(); 832 833 xc_init(); /* initialize processor crosscalls */ 834 835 /* 836 * Copy the real mode code at "real_mode_start" to the 837 * page at rm_platter_va. 838 */ 839 warm_reset_vector = mp_map_warm_reset_vector(); 840 if (warm_reset_vector == NULL) 841 goto done; 842 843 bcopy((caddr_t)real_mode_start, 844 (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code, 845 (size_t)real_mode_end - (size_t)real_mode_start); 846 847 flushes_require_xcalls = 1; 848 849 affinity_set(CPU_CURRENT); 850 851 for (who = 0; who < NCPU; who++) { 852 if (who == cpuid) 853 continue; 854 855 if ((mp_cpus & (1 << who)) == 0) 856 continue; 857 858 mp_startup_init(who); 859 started_cpu = 1; 860 (*cpu_startf)(who, rm_platter_pa); 861 862 while ((procset & (1 << who)) == 0) { 863 864 delay(1); 865 if (++delays > (20 * hz)) { 866 867 cmn_err(CE_WARN, 868 "cpu%d failed to start", who); 869 870 mutex_enter(&cpu_lock); 871 cpu[who]->cpu_flags = 0; 872 cpu_del_unit(who); 873 mutex_exit(&cpu_lock); 874 875 started_cpu = 0; 876 break; 877 } 878 } 879 if (!started_cpu) 880 continue; 881 if (tsc_gethrtime_enable) 882 tsc_sync_master(who); 883 884 885 if (dtrace_cpu_init != NULL) { 886 /* 887 * DTrace CPU initialization expects cpu_lock 888 * to be held. 889 */ 890 mutex_enter(&cpu_lock); 891 (*dtrace_cpu_init)(who); 892 mutex_exit(&cpu_lock); 893 } 894 } 895 896 affinity_clear(); 897 898 for (who = 0; who < NCPU; who++) { 899 if (who == cpuid) 900 continue; 901 902 if (!(procset & (1 << who))) 903 continue; 904 905 while (!(cpu_ready_set & (1 << who))) 906 delay(1); 907 } 908 909 done: 910 workaround_errata_end(); 911 912 if (warm_reset_vector != NULL) 913 mp_unmap_warm_reset_vector(warm_reset_vector); 914 hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 915 HAT_UNLOAD); 916 } 917 918 /* 919 * Dummy functions - no i86pc platforms support dynamic cpu allocation. 920 */ 921 /*ARGSUSED*/ 922 int 923 mp_cpu_configure(int cpuid) 924 { 925 return (ENOTSUP); /* not supported */ 926 } 927 928 /*ARGSUSED*/ 929 int 930 mp_cpu_unconfigure(int cpuid) 931 { 932 return (ENOTSUP); /* not supported */ 933 } 934 935 /* 936 * Startup function for 'other' CPUs (besides boot cpu). 937 * Resumed from cpu_startup. 938 */ 939 void 940 mp_startup(void) 941 { 942 struct cpu *cp = CPU; 943 extern int procset; 944 uint_t new_x86_feature; 945 946 new_x86_feature = cpuid_pass1(cp); 947 948 /* 949 * We need to Sync MTRR with cpu0's MTRR. We have to do 950 * this with interrupts disabled. 951 */ 952 if (x86_feature & X86_MTRR) 953 mtrr_sync(); 954 /* 955 * Enable machine check architecture 956 */ 957 if (x86_feature & X86_MCA) 958 setup_mca(); 959 960 /* 961 * Initialize this CPU's syscall handlers 962 */ 963 init_cpu_syscall(cp); 964 965 /* 966 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the 967 * highest level at which a routine is permitted to block on 968 * an adaptive mutex (allows for cpu poke interrupt in case 969 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks 970 * device interrupts that may end up in the hat layer issuing cross 971 * calls before CPU_READY is set. 972 */ 973 (void) splx(ipltospl(LOCK_LEVEL)); 974 975 /* 976 * Do a sanity check to make sure this new CPU is a sane thing 977 * to add to the collection of processors running this system. 978 * 979 * XXX Clearly this needs to get more sophisticated, if x86 980 * systems start to get built out of heterogenous CPUs; as is 981 * likely to happen once the number of processors in a configuration 982 * gets large enough. 983 */ 984 if ((x86_feature & new_x86_feature) != x86_feature) { 985 cmn_err(CE_CONT, "?cpu%d: %b\n", 986 cp->cpu_id, new_x86_feature, FMT_X86_FEATURE); 987 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); 988 } 989 990 /* 991 * We could be more sophisticated here, and just mark the CPU 992 * as "faulted" but at this point we'll opt for the easier 993 * answer of dieing horribly. Provided the boot cpu is ok, 994 * the system can be recovered by booting with use_mp set to zero. 995 */ 996 if (workaround_errata(cp) != 0) 997 panic("critical workaround(s) missing for cpu%d", cp->cpu_id); 998 999 cpuid_pass2(cp); 1000 cpuid_pass3(cp); 1001 (void) cpuid_pass4(cp); 1002 1003 init_cpu_info(cp); 1004 1005 add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi); 1006 1007 mutex_enter(&cpu_lock); 1008 procset |= 1 << cp->cpu_id; 1009 mutex_exit(&cpu_lock); 1010 1011 if (tsc_gethrtime_enable) 1012 tsc_sync_slave(); 1013 1014 mutex_enter(&cpu_lock); 1015 /* 1016 * It's unfortunate that chip_cpu_init() has to be called here. 1017 * It really belongs in cpu_add_unit(), but unfortunately it is 1018 * dependent on the cpuid probing, which must be done in the 1019 * context of the current CPU. Care must be taken on x86 to ensure 1020 * that mp_startup can safely block even though chip_cpu_init() and 1021 * cpu_add_active() have not yet been called. 1022 */ 1023 chip_cpu_init(cp); 1024 chip_cpu_startup(cp); 1025 1026 cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS; 1027 cpu_add_active(cp); 1028 mutex_exit(&cpu_lock); 1029 1030 (void) spl0(); /* enable interrupts */ 1031 1032 if (boothowto & RB_DEBUG) 1033 kdi_dvec_cpu_init(cp); 1034 1035 /* 1036 * Setting the bit in cpu_ready_set must be the last operation in 1037 * processor initialization; the boot CPU will continue to boot once 1038 * it sees this bit set for all active CPUs. 1039 */ 1040 CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); 1041 1042 /* 1043 * Because mp_startup() gets fired off after init() starts, we 1044 * can't use the '?' trick to do 'boot -v' printing - so we 1045 * always direct the 'cpu .. online' messages to the log. 1046 */ 1047 cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 1048 cp->cpu_id); 1049 1050 /* 1051 * Now we are done with the startup thread, so free it up. 1052 */ 1053 thread_exit(); 1054 panic("mp_startup: cannot return"); 1055 /*NOTREACHED*/ 1056 } 1057 1058 1059 /* 1060 * Start CPU on user request. 1061 */ 1062 /* ARGSUSED */ 1063 int 1064 mp_cpu_start(struct cpu *cp) 1065 { 1066 ASSERT(MUTEX_HELD(&cpu_lock)); 1067 if (cp->cpu_id == getbootcpuid()) 1068 return (EBUSY); /* Cannot start boot CPU */ 1069 return (0); 1070 } 1071 1072 /* 1073 * Stop CPU on user request. 1074 */ 1075 /* ARGSUSED */ 1076 int 1077 mp_cpu_stop(struct cpu *cp) 1078 { 1079 ASSERT(MUTEX_HELD(&cpu_lock)); 1080 if (cp->cpu_id == getbootcpuid()) 1081 return (EBUSY); /* Cannot stop boot CPU */ 1082 1083 return (0); 1084 } 1085 1086 /* 1087 * Power on CPU. 1088 */ 1089 /* ARGSUSED */ 1090 int 1091 mp_cpu_poweron(struct cpu *cp) 1092 { 1093 ASSERT(MUTEX_HELD(&cpu_lock)); 1094 return (ENOTSUP); /* not supported */ 1095 } 1096 1097 /* 1098 * Power off CPU. 1099 */ 1100 /* ARGSUSED */ 1101 int 1102 mp_cpu_poweroff(struct cpu *cp) 1103 { 1104 ASSERT(MUTEX_HELD(&cpu_lock)); 1105 return (ENOTSUP); /* not supported */ 1106 } 1107 1108 1109 /* 1110 * Take the specified CPU out of participation in interrupts. 1111 */ 1112 int 1113 cpu_disable_intr(struct cpu *cp) 1114 { 1115 /* 1116 * cannot disable interrupts on boot cpu 1117 */ 1118 if (cp == cpu[getbootcpuid()]) 1119 return (EBUSY); 1120 1121 if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) 1122 return (EBUSY); 1123 1124 cp->cpu_flags &= ~CPU_ENABLE; 1125 return (0); 1126 } 1127 1128 /* 1129 * Allow the specified CPU to participate in interrupts. 1130 */ 1131 void 1132 cpu_enable_intr(struct cpu *cp) 1133 { 1134 ASSERT(MUTEX_HELD(&cpu_lock)); 1135 if (cp == cpu[getbootcpuid()]) 1136 return; 1137 1138 cp->cpu_flags |= CPU_ENABLE; 1139 psm_enable_intr(cp->cpu_id); 1140 } 1141 1142 1143 /* 1144 * return the cpu id of the initial startup cpu 1145 */ 1146 processorid_t 1147 getbootcpuid(void) 1148 { 1149 return (0); 1150 } 1151 1152 static ushort_t * 1153 mp_map_warm_reset_vector() 1154 { 1155 ushort_t *warm_reset_vector; 1156 1157 if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 1158 sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 1159 return (NULL); 1160 1161 /* 1162 * setup secondary cpu bios boot up vector 1163 */ 1164 *warm_reset_vector = (ushort_t)((caddr_t) 1165 ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 1166 + ((ulong_t)rm_platter_va & 0xf)); 1167 warm_reset_vector++; 1168 *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 1169 1170 --warm_reset_vector; 1171 return (warm_reset_vector); 1172 } 1173 1174 static void 1175 mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector) 1176 { 1177 psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 1178 } 1179 1180 /*ARGSUSED*/ 1181 void 1182 mp_cpu_faulted_enter(struct cpu *cp) 1183 {} 1184 1185 /*ARGSUSED*/ 1186 void 1187 mp_cpu_faulted_exit(struct cpu *cp) 1188 {} 1189 1190 /* 1191 * The following two routines are used as context operators on threads belonging 1192 * to processes with a private LDT (see sysi86). Due to the rarity of such 1193 * processes, these routines are currently written for best code readability and 1194 * organization rather than speed. We could avoid checking x86_feature at every 1195 * context switch by installing different context ops, depending on the 1196 * x86_feature flags, at LDT creation time -- one for each combination of fast 1197 * syscall feature flags. 1198 */ 1199 1200 /*ARGSUSED*/ 1201 void 1202 cpu_fast_syscall_disable(void *arg) 1203 { 1204 if (x86_feature & X86_SEP) 1205 cpu_sep_disable(); 1206 if (x86_feature & X86_ASYSC) 1207 cpu_asysc_disable(); 1208 } 1209 1210 /*ARGSUSED*/ 1211 void 1212 cpu_fast_syscall_enable(void *arg) 1213 { 1214 if (x86_feature & X86_SEP) 1215 cpu_sep_enable(); 1216 if (x86_feature & X86_ASYSC) 1217 cpu_asysc_enable(); 1218 } 1219 1220 static void 1221 cpu_sep_enable(void) 1222 { 1223 uint64_t value; 1224 1225 ASSERT(x86_feature & X86_SEP); 1226 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1227 1228 value = KCS_SEL; 1229 wrmsr(MSR_INTC_SEP_CS, &value); 1230 } 1231 1232 static void 1233 cpu_sep_disable(void) 1234 { 1235 uint64_t value; 1236 1237 ASSERT(x86_feature & X86_SEP); 1238 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1239 1240 /* 1241 * Setting the SYSENTER_CS_MSR register to 0 causes software executing 1242 * the sysenter or sysexit instruction to trigger a #gp fault. 1243 */ 1244 value = 0; 1245 wrmsr(MSR_INTC_SEP_CS, &value); 1246 } 1247 1248 static void 1249 cpu_asysc_enable(void) 1250 { 1251 uint64_t value; 1252 1253 ASSERT(x86_feature & X86_ASYSC); 1254 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1255 1256 (void) rdmsr(MSR_AMD_EFER, &value); 1257 value |= AMD_EFER_SCE; 1258 wrmsr(MSR_AMD_EFER, &value); 1259 } 1260 1261 static void 1262 cpu_asysc_disable(void) 1263 { 1264 uint64_t value; 1265 1266 ASSERT(x86_feature & X86_ASYSC); 1267 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1268 1269 /* 1270 * Turn off the SCE (syscall enable) bit in the EFER register. Software 1271 * executing syscall or sysret with this bit off will incur a #ud trap. 1272 */ 1273 (void) rdmsr(MSR_AMD_EFER, &value); 1274 value &= ~AMD_EFER_SCE; 1275 wrmsr(MSR_AMD_EFER, &value); 1276 } 1277