1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/thread.h> 30 #include <sys/cpuvar.h> 31 #include <sys/t_lock.h> 32 #include <sys/param.h> 33 #include <sys/proc.h> 34 #include <sys/disp.h> 35 #include <sys/mmu.h> 36 #include <sys/class.h> 37 #include <sys/cmn_err.h> 38 #include <sys/debug.h> 39 #include <sys/asm_linkage.h> 40 #include <sys/x_call.h> 41 #include <sys/systm.h> 42 #include <sys/var.h> 43 #include <sys/vtrace.h> 44 #include <vm/hat.h> 45 #include <sys/mmu.h> 46 #include <vm/as.h> 47 #include <vm/seg_kmem.h> 48 #include <sys/segments.h> 49 #include <sys/kmem.h> 50 #include <sys/stack.h> 51 #include <sys/smp_impldefs.h> 52 #include <sys/x86_archext.h> 53 #include <sys/machsystm.h> 54 #include <sys/traptrace.h> 55 #include <sys/clock.h> 56 #include <sys/cpc_impl.h> 57 #include <sys/chip.h> 58 #include <sys/dtrace.h> 59 #include <sys/archsystm.h> 60 #include <sys/fp.h> 61 #include <sys/reboot.h> 62 #include <sys/kdi.h> 63 #include <vm/hat_i86.h> 64 #include <sys/memnode.h> 65 #include <sys/pci_cfgspace.h> 66 #include <sys/cpu_module.h> 67 68 struct cpu cpus[1]; /* CPU data */ 69 struct cpu *cpu[NCPU] = {&cpus[0]}; /* pointers to all CPUs */ 70 cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 71 72 /* 73 * Useful for disabling MP bring-up for an MP capable kernel 74 * (a kernel that was built with MP defined) 75 */ 76 int use_mp = 1; 77 78 int mp_cpus = 0x1; /* to be set by platform specific module */ 79 80 /* 81 * This variable is used by the hat layer to decide whether or not 82 * critical sections are needed to prevent race conditions. For sun4m, 83 * this variable is set once enough MP initialization has been done in 84 * order to allow cross calls. 85 */ 86 int flushes_require_xcalls = 0; 87 ulong_t cpu_ready_set = 1; 88 89 extern void real_mode_start(void); 90 extern void real_mode_end(void); 91 static void mp_startup(void); 92 93 static void cpu_sep_enable(void); 94 static void cpu_sep_disable(void); 95 static void cpu_asysc_enable(void); 96 static void cpu_asysc_disable(void); 97 98 extern int tsc_gethrtime_enable; 99 100 /* 101 * Init CPU info - get CPU type info for processor_info system call. 102 */ 103 void 104 init_cpu_info(struct cpu *cp) 105 { 106 processor_info_t *pi = &cp->cpu_type_info; 107 char buf[CPU_IDSTRLEN]; 108 109 /* 110 * Get clock-frequency property for the CPU. 111 */ 112 pi->pi_clock = cpu_freq; 113 114 (void) strcpy(pi->pi_processor_type, "i386"); 115 if (fpu_exists) 116 (void) strcpy(pi->pi_fputypes, "i387 compatible"); 117 118 (void) cpuid_getidstr(cp, buf, sizeof (buf)); 119 120 cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 121 (void) strcpy(cp->cpu_idstr, buf); 122 123 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); 124 125 (void) cpuid_getbrandstr(cp, buf, sizeof (buf)); 126 cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 127 (void) strcpy(cp->cpu_brandstr, buf); 128 129 cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); 130 } 131 132 /* 133 * Configure syscall support on this CPU. 134 */ 135 /*ARGSUSED*/ 136 static void 137 init_cpu_syscall(struct cpu *cp) 138 { 139 kpreempt_disable(); 140 141 #if defined(__amd64) 142 if (x86_feature & X86_ASYSC) { 143 144 #if !defined(__lint) 145 /* 146 * The syscall instruction imposes a certain ordering on 147 * segment selectors, so we double-check that ordering 148 * here. 149 */ 150 ASSERT(KDS_SEL == KCS_SEL + 8); 151 ASSERT(UDS_SEL == U32CS_SEL + 8); 152 ASSERT(UCS_SEL == U32CS_SEL + 16); 153 #endif 154 /* 155 * Turn syscall/sysret extensions on. 156 */ 157 cpu_asysc_enable(); 158 159 /* 160 * Program the magic registers .. 161 */ 162 wrmsr(MSR_AMD_STAR, ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 163 32); 164 wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall); 165 wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32); 166 167 /* 168 * This list of flags is masked off the incoming 169 * %rfl when we enter the kernel. 170 */ 171 wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T)); 172 } 173 #endif 174 175 /* 176 * On 32-bit kernels, we use sysenter/sysexit because it's too 177 * hard to use syscall/sysret, and it is more portable anyway. 178 * 179 * On 64-bit kernels on Nocona machines, the 32-bit syscall 180 * variant isn't available to 32-bit applications, but sysenter is. 181 */ 182 if (x86_feature & X86_SEP) { 183 184 #if !defined(__lint) 185 /* 186 * The sysenter instruction imposes a certain ordering on 187 * segment selectors, so we double-check that ordering 188 * here. See "sysenter" in Intel document 245471-012, "IA-32 189 * Intel Architecture Software Developer's Manual Volume 2: 190 * Instruction Set Reference" 191 */ 192 ASSERT(KDS_SEL == KCS_SEL + 8); 193 194 ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3)); 195 ASSERT32(UDS_SEL == UCS_SEL + 8); 196 197 ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3)); 198 ASSERT64(UDS_SEL == U32CS_SEL + 8); 199 #endif 200 201 cpu_sep_enable(); 202 203 /* 204 * resume() sets this value to the base of the threads stack 205 * via a context handler. 206 */ 207 wrmsr(MSR_INTC_SEP_ESP, 0ULL); 208 wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter); 209 } 210 211 kpreempt_enable(); 212 } 213 214 /* 215 * Multiprocessor initialization. 216 * 217 * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the 218 * startup and idle threads for the specified CPU. 219 */ 220 static void 221 mp_startup_init(int cpun) 222 { 223 #if defined(__amd64) 224 extern void *long_mode_64(void); 225 #endif /* __amd64 */ 226 227 struct cpu *cp; 228 struct tss *ntss; 229 kthread_id_t tp; 230 caddr_t sp; 231 int size; 232 proc_t *procp; 233 extern void idle(); 234 235 struct cpu_tables *tablesp; 236 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 237 238 #ifdef TRAPTRACE 239 trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; 240 #endif 241 242 ASSERT(cpun < NCPU && cpu[cpun] == NULL); 243 244 if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) { 245 panic("mp_startup_init: cpu%d: " 246 "no memory for cpu structure", cpun); 247 /*NOTREACHED*/ 248 } 249 procp = curthread->t_procp; 250 251 mutex_enter(&cpu_lock); 252 /* 253 * Initialize the dispatcher first. 254 */ 255 disp_cpu_init(cp); 256 mutex_exit(&cpu_lock); 257 258 cpu_vm_data_init(cp); 259 260 /* 261 * Allocate and initialize the startup thread for this CPU. 262 * Interrupt and process switch stacks get allocated later 263 * when the CPU starts running. 264 */ 265 tp = thread_create(NULL, 0, NULL, NULL, 0, procp, 266 TS_STOPPED, maxclsyspri); 267 268 /* 269 * Set state to TS_ONPROC since this thread will start running 270 * as soon as the CPU comes online. 271 * 272 * All the other fields of the thread structure are setup by 273 * thread_create(). 274 */ 275 THREAD_ONPROC(tp, cp); 276 tp->t_preempt = 1; 277 tp->t_bound_cpu = cp; 278 tp->t_affinitycnt = 1; 279 tp->t_cpu = cp; 280 tp->t_disp_queue = cp->cpu_disp; 281 282 /* 283 * Setup thread to start in mp_startup. 284 */ 285 sp = tp->t_stk; 286 tp->t_pc = (uintptr_t)mp_startup; 287 tp->t_sp = (uintptr_t)(sp - MINFRAME); 288 289 cp->cpu_id = cpun; 290 cp->cpu_self = cp; 291 cp->cpu_mask = 1 << cpun; 292 cp->cpu_thread = tp; 293 cp->cpu_lwp = NULL; 294 cp->cpu_dispthread = tp; 295 cp->cpu_dispatch_pri = DISP_PRIO(tp); 296 297 /* 298 * cpu_base_spl must be set explicitly here to prevent any blocking 299 * operations in mp_startup from causing the spl of the cpu to drop 300 * to 0 (allowing device interrupts before we're ready) in resume(). 301 * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY. 302 * As an extra bit of security on DEBUG kernels, this is enforced with 303 * an assertion in mp_startup() -- before cpu_base_spl is set to its 304 * proper value. 305 */ 306 cp->cpu_base_spl = ipltospl(LOCK_LEVEL); 307 308 /* 309 * Now, initialize per-CPU idle thread for this CPU. 310 */ 311 tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); 312 313 cp->cpu_idle_thread = tp; 314 315 tp->t_preempt = 1; 316 tp->t_bound_cpu = cp; 317 tp->t_affinitycnt = 1; 318 tp->t_cpu = cp; 319 tp->t_disp_queue = cp->cpu_disp; 320 321 /* 322 * Bootstrap the CPU for CMT aware scheduling 323 * The rest of the initialization will happen from 324 * mp_startup() 325 */ 326 chip_bootstrap_cpu(cp); 327 328 /* 329 * Perform CPC intialization on the new CPU. 330 */ 331 kcpc_hw_init(cp); 332 333 /* 334 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 335 * for each CPU. 336 */ 337 338 setup_vaddr_for_ppcopy(cp); 339 340 /* 341 * Allocate space for page directory, stack, tss, gdt and idt. 342 * This assumes that kmem_alloc will return memory which is aligned 343 * to the next higher power of 2 or a page(if size > MAXABIG) 344 * If this assumption goes wrong at any time due to change in 345 * kmem alloc, things may not work as the page directory has to be 346 * page aligned 347 */ 348 if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL) 349 panic("mp_startup_init: cpu%d cannot allocate tables", cpun); 350 351 if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) { 352 kmem_free(tablesp, sizeof (struct cpu_tables)); 353 size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE; 354 tablesp = kmem_zalloc(size, KM_NOSLEEP); 355 tablesp = (struct cpu_tables *) 356 (((uintptr_t)tablesp + MMU_STD_PAGESIZE) & 357 MMU_STD_PAGEMASK); 358 } 359 360 ntss = cp->cpu_tss = &tablesp->ct_tss; 361 cp->cpu_gdt = tablesp->ct_gdt; 362 bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t))); 363 364 #if defined(__amd64) 365 366 /* 367 * #DF (double fault). 368 */ 369 ntss->tss_ist1 = 370 (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 371 372 #elif defined(__i386) 373 374 ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp = 375 (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 376 377 ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL; 378 379 ntss->tss_eip = (uint32_t)mp_startup; 380 381 ntss->tss_cs = KCS_SEL; 382 ntss->tss_fs = KFS_SEL; 383 ntss->tss_gs = KGS_SEL; 384 385 /* 386 * setup kernel %gs. 387 */ 388 set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA, 389 SEL_KPL, 0, 1); 390 391 #endif /* __i386 */ 392 393 /* 394 * Set I/O bit map offset equal to size of TSS segment limit 395 * for no I/O permission map. This will cause all user I/O 396 * instructions to generate #gp fault. 397 */ 398 ntss->tss_bitmapbase = sizeof (*ntss); 399 400 /* 401 * setup kernel tss. 402 */ 403 set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, 404 sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL); 405 406 /* 407 * If we have more than one node, each cpu gets a copy of IDT 408 * local to its node. If this is a Pentium box, we use cpu 0's 409 * IDT. cpu 0's IDT has been made read-only to workaround the 410 * cmpxchgl register bug 411 */ 412 cp->cpu_idt = CPU->cpu_idt; 413 if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) { 414 cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP); 415 bcopy(idt0, cp->cpu_idt, sizeof (idt0)); 416 } 417 418 /* 419 * Get interrupt priority data from cpu 0 420 */ 421 cp->cpu_pri_data = CPU->cpu_pri_data; 422 423 hat_cpu_online(cp); 424 425 /* Should remove all entries for the current process/thread here */ 426 427 /* 428 * Fill up the real mode platter to make it easy for real mode code to 429 * kick it off. This area should really be one passed by boot to kernel 430 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 431 * have identical physical and virtual address in paged mode. 432 */ 433 real_mode_platter->rm_idt_base = cp->cpu_idt; 434 real_mode_platter->rm_idt_lim = sizeof (idt0) - 1; 435 real_mode_platter->rm_gdt_base = cp->cpu_gdt; 436 real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1; 437 real_mode_platter->rm_pdbr = getcr3(); 438 real_mode_platter->rm_cpu = cpun; 439 real_mode_platter->rm_x86feature = x86_feature; 440 real_mode_platter->rm_cr4 = cr4_value; 441 442 #if defined(__amd64) 443 if (getcr3() > 0xffffffffUL) 444 panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 445 "located above 4G in physical memory (@ 0x%llx).", 446 (unsigned long long)getcr3()); 447 448 /* 449 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 450 * by code in real_mode_start(): 451 * 452 * GDT[0]: NULL selector 453 * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 454 * 455 * Clear the IDT as interrupts will be off and a limit of 0 will cause 456 * the CPU to triple fault and reset on an NMI, seemingly as reasonable 457 * a course of action as any other, though it may cause the entire 458 * platform to reset in some cases... 459 */ 460 real_mode_platter->rm_temp_gdt[0] = 0ULL; 461 real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 462 463 real_mode_platter->rm_temp_gdt_lim = (ushort_t) 464 (sizeof (real_mode_platter->rm_temp_gdt) - 1); 465 real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 466 (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt); 467 468 real_mode_platter->rm_temp_idt_lim = 0; 469 real_mode_platter->rm_temp_idt_base = 0; 470 471 /* 472 * Since the CPU needs to jump to protected mode using an identity 473 * mapped address, we need to calculate it here. 474 */ 475 real_mode_platter->rm_longmode64_addr = rm_platter_pa + 476 ((uint32_t)long_mode_64 - (uint32_t)real_mode_start); 477 #endif /* __amd64 */ 478 479 #ifdef TRAPTRACE 480 /* 481 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this 482 * CPU. 483 */ 484 ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); 485 ttc->ttc_next = ttc->ttc_first; 486 ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; 487 #endif 488 489 /* 490 * Record that we have another CPU. 491 */ 492 mutex_enter(&cpu_lock); 493 /* 494 * Initialize the interrupt threads for this CPU 495 */ 496 cpu_intr_alloc(cp, NINTR_THREADS); 497 /* 498 * Add CPU to list of available CPUs. It'll be on the active list 499 * after mp_startup(). 500 */ 501 cpu_add_unit(cp); 502 mutex_exit(&cpu_lock); 503 } 504 505 /* 506 * Apply workarounds for known errata, and warn about those that are absent. 507 * 508 * System vendors occasionally create configurations which contain different 509 * revisions of the CPUs that are almost but not exactly the same. At the 510 * time of writing, this meant that their clock rates were the same, their 511 * feature sets were the same, but the required workaround were -not- 512 * necessarily the same. So, this routine is invoked on -every- CPU soon 513 * after starting to make sure that the resulting system contains the most 514 * pessimal set of workarounds needed to cope with *any* of the CPUs in the 515 * system. 516 * 517 * workaround_errata is invoked early in mlsetup() for CPU 0, and in 518 * mp_startup() for all slave CPUs. Slaves process workaround_errata prior 519 * to acknowledging their readiness to the master, so this routine will 520 * never be executed by multiple CPUs in parallel, thus making updates to 521 * global data safe. 522 * 523 * These workarounds are based on Rev 3.57 of the Revision Guide for 524 * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005. 525 */ 526 527 #if defined(OPTERON_ERRATUM_91) 528 int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ 529 #endif 530 531 #if defined(OPTERON_ERRATUM_93) 532 int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ 533 #endif 534 535 #if defined(OPTERON_ERRATUM_100) 536 int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ 537 #endif 538 539 #if defined(OPTERON_ERRATUM_109) 540 int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ 541 #endif 542 543 #if defined(OPTERON_ERRATUM_121) 544 int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ 545 #endif 546 547 #if defined(OPTERON_ERRATUM_122) 548 int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ 549 #endif 550 551 #if defined(OPTERON_ERRATUM_123) 552 int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ 553 #endif 554 555 #if defined(OPTERON_ERRATUM_131) 556 int opteron_erratum_131; /* if non-zero -> at least one cpu has it */ 557 #endif 558 559 #if defined(OPTERON_WORKAROUND_6336786) 560 int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */ 561 int opteron_workaround_6336786_UP = 0; /* Not needed for UP */ 562 #endif 563 564 #if defined(OPTERON_WORKAROUND_6323525) 565 int opteron_workaround_6323525; /* if non-zero -> at least one cpu has it */ 566 #endif 567 568 #define WARNING(cpu, n) \ 569 cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d", \ 570 (cpu)->cpu_id, (n)) 571 572 uint_t 573 workaround_errata(struct cpu *cpu) 574 { 575 uint_t missing = 0; 576 577 ASSERT(cpu == CPU); 578 579 /*LINTED*/ 580 if (cpuid_opteron_erratum(cpu, 88) > 0) { 581 /* 582 * SWAPGS May Fail To Read Correct GS Base 583 */ 584 #if defined(OPTERON_ERRATUM_88) 585 /* 586 * The workaround is an mfence in the relevant assembler code 587 */ 588 #else 589 WARNING(cpu, 88); 590 missing++; 591 #endif 592 } 593 594 if (cpuid_opteron_erratum(cpu, 91) > 0) { 595 /* 596 * Software Prefetches May Report A Page Fault 597 */ 598 #if defined(OPTERON_ERRATUM_91) 599 /* 600 * fix is in trap.c 601 */ 602 opteron_erratum_91++; 603 #else 604 WARNING(cpu, 91); 605 missing++; 606 #endif 607 } 608 609 if (cpuid_opteron_erratum(cpu, 93) > 0) { 610 /* 611 * RSM Auto-Halt Restart Returns to Incorrect RIP 612 */ 613 #if defined(OPTERON_ERRATUM_93) 614 /* 615 * fix is in trap.c 616 */ 617 opteron_erratum_93++; 618 #else 619 WARNING(cpu, 93); 620 missing++; 621 #endif 622 } 623 624 /*LINTED*/ 625 if (cpuid_opteron_erratum(cpu, 95) > 0) { 626 /* 627 * RET Instruction May Return to Incorrect EIP 628 */ 629 #if defined(OPTERON_ERRATUM_95) 630 #if defined(_LP64) 631 /* 632 * Workaround this by ensuring that 32-bit user code and 633 * 64-bit kernel code never occupy the same address 634 * range mod 4G. 635 */ 636 if (_userlimit32 > 0xc0000000ul) 637 *(uintptr_t *)&_userlimit32 = 0xc0000000ul; 638 639 /*LINTED*/ 640 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); 641 #endif /* _LP64 */ 642 #else 643 WARNING(cpu, 95); 644 missing++; 645 #endif /* OPTERON_ERRATUM_95 */ 646 } 647 648 if (cpuid_opteron_erratum(cpu, 100) > 0) { 649 /* 650 * Compatibility Mode Branches Transfer to Illegal Address 651 */ 652 #if defined(OPTERON_ERRATUM_100) 653 /* 654 * fix is in trap.c 655 */ 656 opteron_erratum_100++; 657 #else 658 WARNING(cpu, 100); 659 missing++; 660 #endif 661 } 662 663 /*LINTED*/ 664 if (cpuid_opteron_erratum(cpu, 108) > 0) { 665 /* 666 * CPUID Instruction May Return Incorrect Model Number In 667 * Some Processors 668 */ 669 #if defined(OPTERON_ERRATUM_108) 670 /* 671 * (Our cpuid-handling code corrects the model number on 672 * those processors) 673 */ 674 #else 675 WARNING(cpu, 108); 676 missing++; 677 #endif 678 } 679 680 /*LINTED*/ 681 if (cpuid_opteron_erratum(cpu, 109) > 0) { 682 /* 683 * Certain Reverse REP MOVS May Produce Unpredictable Behaviour 684 */ 685 #if defined(OPTERON_ERRATUM_109) 686 687 /* workaround is to print a warning to upgrade BIOS */ 688 if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 689 opteron_erratum_109++; 690 #else 691 WARNING(cpu, 109); 692 missing++; 693 #endif 694 } 695 /*LINTED*/ 696 if (cpuid_opteron_erratum(cpu, 121) > 0) { 697 /* 698 * Sequential Execution Across Non_Canonical Boundary Caused 699 * Processor Hang 700 */ 701 #if defined(OPTERON_ERRATUM_121) 702 static int lma; 703 704 if (opteron_erratum_121) 705 opteron_erratum_121++; 706 707 /* 708 * Erratum 121 is only present in long (64 bit) mode. 709 * Workaround is to include the page immediately before the 710 * va hole to eliminate the possibility of system hangs due to 711 * sequential execution across the va hole boundary. 712 */ 713 if (lma == 0) { 714 /* 715 * check LMA once: assume all cpus are in long mode 716 * or not. 717 */ 718 lma = 1; 719 720 if (rdmsr(MSR_AMD_EFER) & AMD_EFER_LMA) { 721 if (hole_start) { 722 hole_start -= PAGESIZE; 723 } else { 724 /* 725 * hole_start not yet initialized by 726 * mmu_init. Initialize hole_start 727 * with value to be subtracted. 728 */ 729 hole_start = PAGESIZE; 730 } 731 opteron_erratum_121++; 732 } 733 } 734 #else 735 WARNING(cpu, 121); 736 missing++; 737 #endif 738 } 739 740 /*LINTED*/ 741 if (cpuid_opteron_erratum(cpu, 122) > 0) { 742 /* 743 * TLB Flush Filter May Cause Cohenrency Problem in 744 * Multiprocessor Systems 745 */ 746 #if defined(OPTERON_ERRATUM_122) 747 /* 748 * Erratum 122 is only present in MP configurations (multi-core 749 * or multi-processor). 750 */ 751 752 if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 || 753 cpuid_get_ncpu_per_chip(cpu) > 1) { 754 /* disable TLB Flush Filter */ 755 wrmsr(MSR_AMD_HWCR, rdmsr(MSR_AMD_HWCR) | 756 (uint64_t)(uintptr_t)AMD_HWCR_FFDIS); 757 opteron_erratum_122++; 758 } 759 760 #else 761 WARNING(cpu, 122); 762 missing++; 763 #endif 764 } 765 766 #if defined(OPTERON_ERRATUM_123) 767 /*LINTED*/ 768 if (cpuid_opteron_erratum(cpu, 123) > 0) { 769 /* 770 * Bypassed Reads May Cause Data Corruption of System Hang in 771 * Dual Core Processors 772 */ 773 /* 774 * Erratum 123 applies only to multi-core cpus. 775 */ 776 777 if (cpuid_get_ncpu_per_chip(cpu) > 1) { 778 /* workaround is to print a warning to upgrade BIOS */ 779 if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 780 opteron_erratum_123++; 781 } 782 } 783 #endif 784 785 #if defined(OPTERON_ERRATUM_131) 786 /*LINTED*/ 787 if (cpuid_opteron_erratum(cpu, 131) > 0) { 788 /* 789 * Multiprocessor Systems with Four or More Cores May Deadlock 790 * Waiting for a Probe Response 791 */ 792 /* 793 * Erratum 131 applies to any system with four or more cores. 794 */ 795 if ((opteron_erratum_131 == 0) && ((lgrp_plat_node_cnt * 796 cpuid_get_ncpu_per_chip(cpu)) >= 4)) { 797 /* 798 * Workaround is to print a warning to upgrade 799 * the BIOS 800 */ 801 if (!(rdmsr(MSR_AMD_NB_CFG) & AMD_NB_CFG_SRQ_HEARTBEAT)) 802 opteron_erratum_131++; 803 } 804 } 805 #endif 806 807 #if defined(OPTERON_WORKAROUND_6336786) 808 /* 809 * This isn't really erratum, but for convenience the 810 * detection/workaround code lives here and in cpuid_opteron_erratum. 811 */ 812 if (cpuid_opteron_erratum(cpu, 6336786) > 0) { 813 int node; 814 uint8_t data; 815 816 /* 817 * Disable C1-Clock ramping on multi-core/multi-processor 818 * K8 platforms to guard against TSC drift. 819 */ 820 if (opteron_workaround_6336786) { 821 opteron_workaround_6336786++; 822 } else if ((lgrp_plat_node_cnt * 823 cpuid_get_ncpu_per_chip(cpu) >= 2) || 824 opteron_workaround_6336786_UP) { 825 for (node = 0; node < lgrp_plat_node_cnt; node++) { 826 /* 827 * Clear PMM7[1:0] (function 3, offset 0x87) 828 * Northbridge device is the node id + 24. 829 */ 830 data = pci_getb_func(0, node + 24, 3, 0x87); 831 data &= 0xFC; 832 pci_putb_func(0, node + 24, 3, 0x87, data); 833 } 834 opteron_workaround_6336786++; 835 } 836 } 837 #endif 838 839 #if defined(OPTERON_WORKAROUND_6323525) 840 /*LINTED*/ 841 /* 842 * Mutex primitives don't work as expected. 843 */ 844 if (cpuid_opteron_erratum(cpu, 6323525) > 0) { 845 846 /* 847 * problem only occurs with 2 or more cores. If bit in 848 * MSR_BU_CFG set, then not applicable. The workaround 849 * is to patch the semaphone routines with the lfence 850 * instruction to provide necessary load memory barrier with 851 * possible subsequent read-modify-write ops. 852 * 853 * It is too early in boot to call the patch routine so 854 * set erratum variable to be done in startup_end(). 855 */ 856 if (opteron_workaround_6323525) { 857 opteron_workaround_6323525++; 858 } else if ((x86_feature & X86_SSE2) && ((lgrp_plat_node_cnt * 859 cpuid_get_ncpu_per_chip(cpu)) >= 2)) { 860 if ((xrdmsr(MSR_BU_CFG) & 0x02) == 0) 861 opteron_workaround_6323525++; 862 } 863 } 864 #endif 865 return (missing); 866 } 867 868 void 869 workaround_errata_end() 870 { 871 #if defined(OPTERON_ERRATUM_109) 872 if (opteron_erratum_109) { 873 cmn_err(CE_WARN, 874 "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 875 " processor\nerratum 109 was not detected; updating your" 876 " system's BIOS to a version\ncontaining this" 877 " microcode patch is HIGHLY recommended or erroneous" 878 " system\noperation may occur.\n"); 879 } 880 #endif /* OPTERON_ERRATUM_109 */ 881 #if defined(OPTERON_ERRATUM_123) 882 if (opteron_erratum_123) { 883 cmn_err(CE_WARN, 884 "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 885 " processor\nerratum 123 was not detected; updating your" 886 " system's BIOS to a version\ncontaining this" 887 " microcode patch is HIGHLY recommended or erroneous" 888 " system\noperation may occur.\n"); 889 } 890 #endif /* OPTERON_ERRATUM_123 */ 891 #if defined(OPTERON_ERRATUM_131) 892 if (opteron_erratum_131) { 893 cmn_err(CE_WARN, 894 "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 895 " processor\nerratum 131 was not detected; updating your" 896 " system's BIOS to a version\ncontaining this" 897 " microcode patch is HIGHLY recommended or erroneous" 898 " system\noperation may occur.\n"); 899 } 900 #endif /* OPTERON_ERRATUM_131 */ 901 } 902 903 static ushort_t *mp_map_warm_reset_vector(); 904 static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector); 905 906 /*ARGSUSED*/ 907 void 908 start_other_cpus(int cprboot) 909 { 910 unsigned who; 911 int cpuid = 0; 912 int delays = 0; 913 int started_cpu; 914 ushort_t *warm_reset_vector = NULL; 915 extern int procset; 916 917 /* 918 * Initialize our own cpu_info. 919 */ 920 init_cpu_info(CPU); 921 922 /* 923 * Initialize our syscall handlers 924 */ 925 init_cpu_syscall(CPU); 926 927 /* 928 * if only 1 cpu or not using MP, skip the rest of this 929 */ 930 if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) { 931 if (use_mp == 0) 932 cmn_err(CE_CONT, "?***** Not in MP mode\n"); 933 goto done; 934 } 935 936 /* 937 * perform such initialization as is needed 938 * to be able to take CPUs on- and off-line. 939 */ 940 cpu_pause_init(); 941 942 xc_init(); /* initialize processor crosscalls */ 943 944 /* 945 * Copy the real mode code at "real_mode_start" to the 946 * page at rm_platter_va. 947 */ 948 warm_reset_vector = mp_map_warm_reset_vector(); 949 if (warm_reset_vector == NULL) 950 goto done; 951 952 bcopy((caddr_t)real_mode_start, 953 (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code, 954 (size_t)real_mode_end - (size_t)real_mode_start); 955 956 flushes_require_xcalls = 1; 957 958 affinity_set(CPU_CURRENT); 959 960 for (who = 0; who < NCPU; who++) { 961 if (who == cpuid) 962 continue; 963 964 if ((mp_cpus & (1 << who)) == 0) 965 continue; 966 967 mp_startup_init(who); 968 started_cpu = 1; 969 (*cpu_startf)(who, rm_platter_pa); 970 971 while ((procset & (1 << who)) == 0) { 972 973 delay(1); 974 if (++delays > (20 * hz)) { 975 976 cmn_err(CE_WARN, 977 "cpu%d failed to start", who); 978 979 mutex_enter(&cpu_lock); 980 cpu[who]->cpu_flags = 0; 981 cpu_vm_data_destroy(cpu[who]); 982 cpu_del_unit(who); 983 mutex_exit(&cpu_lock); 984 985 started_cpu = 0; 986 break; 987 } 988 } 989 if (!started_cpu) 990 continue; 991 if (tsc_gethrtime_enable) 992 tsc_sync_master(who); 993 994 995 if (dtrace_cpu_init != NULL) { 996 /* 997 * DTrace CPU initialization expects cpu_lock 998 * to be held. 999 */ 1000 mutex_enter(&cpu_lock); 1001 (*dtrace_cpu_init)(who); 1002 mutex_exit(&cpu_lock); 1003 } 1004 } 1005 1006 affinity_clear(); 1007 1008 for (who = 0; who < NCPU; who++) { 1009 if (who == cpuid) 1010 continue; 1011 1012 if (!(procset & (1 << who))) 1013 continue; 1014 1015 while (!(cpu_ready_set & (1 << who))) 1016 delay(1); 1017 } 1018 1019 done: 1020 workaround_errata_end(); 1021 1022 if (warm_reset_vector != NULL) 1023 mp_unmap_warm_reset_vector(warm_reset_vector); 1024 hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 1025 HAT_UNLOAD); 1026 } 1027 1028 /* 1029 * Dummy functions - no i86pc platforms support dynamic cpu allocation. 1030 */ 1031 /*ARGSUSED*/ 1032 int 1033 mp_cpu_configure(int cpuid) 1034 { 1035 return (ENOTSUP); /* not supported */ 1036 } 1037 1038 /*ARGSUSED*/ 1039 int 1040 mp_cpu_unconfigure(int cpuid) 1041 { 1042 return (ENOTSUP); /* not supported */ 1043 } 1044 1045 /* 1046 * Startup function for 'other' CPUs (besides boot cpu). 1047 * Called from real_mode_start (after *ap_mlsetup). 1048 * 1049 * WARNING: until CPU_READY is set, mp_startup and routines called by 1050 * mp_startup should not call routines (e.g. kmem_free) that could call 1051 * hat_unload which requires CPU_READY to be set. 1052 */ 1053 void 1054 mp_startup(void) 1055 { 1056 struct cpu *cp = CPU; 1057 extern int procset; 1058 uint_t new_x86_feature; 1059 1060 new_x86_feature = cpuid_pass1(cp); 1061 1062 /* 1063 * We need to Sync MTRR with cpu0's MTRR. We have to do 1064 * this with interrupts disabled. 1065 */ 1066 if (x86_feature & X86_MTRR) 1067 mtrr_sync(); 1068 1069 /* 1070 * Initialize this CPU's syscall handlers 1071 */ 1072 init_cpu_syscall(cp); 1073 1074 /* 1075 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the 1076 * highest level at which a routine is permitted to block on 1077 * an adaptive mutex (allows for cpu poke interrupt in case 1078 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks 1079 * device interrupts that may end up in the hat layer issuing cross 1080 * calls before CPU_READY is set. 1081 */ 1082 (void) splx(ipltospl(LOCK_LEVEL)); 1083 1084 /* 1085 * Do a sanity check to make sure this new CPU is a sane thing 1086 * to add to the collection of processors running this system. 1087 * 1088 * XXX Clearly this needs to get more sophisticated, if x86 1089 * systems start to get built out of heterogenous CPUs; as is 1090 * likely to happen once the number of processors in a configuration 1091 * gets large enough. 1092 */ 1093 if ((x86_feature & new_x86_feature) != x86_feature) { 1094 cmn_err(CE_CONT, "?cpu%d: %b\n", 1095 cp->cpu_id, new_x86_feature, FMT_X86_FEATURE); 1096 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); 1097 } 1098 1099 /* 1100 * We could be more sophisticated here, and just mark the CPU 1101 * as "faulted" but at this point we'll opt for the easier 1102 * answer of dieing horribly. Provided the boot cpu is ok, 1103 * the system can be recovered by booting with use_mp set to zero. 1104 */ 1105 if (workaround_errata(cp) != 0) 1106 panic("critical workaround(s) missing for cpu%d", cp->cpu_id); 1107 1108 cpuid_pass2(cp); 1109 cpuid_pass3(cp); 1110 (void) cpuid_pass4(cp); 1111 1112 init_cpu_info(cp); 1113 1114 mutex_enter(&cpu_lock); 1115 procset |= 1 << cp->cpu_id; 1116 mutex_exit(&cpu_lock); 1117 1118 if (tsc_gethrtime_enable) 1119 tsc_sync_slave(); 1120 1121 mutex_enter(&cpu_lock); 1122 /* 1123 * It's unfortunate that chip_cpu_init() has to be called here. 1124 * It really belongs in cpu_add_unit(), but unfortunately it is 1125 * dependent on the cpuid probing, which must be done in the 1126 * context of the current CPU. Care must be taken on x86 to ensure 1127 * that mp_startup can safely block even though chip_cpu_init() and 1128 * cpu_add_active() have not yet been called. 1129 */ 1130 chip_cpu_init(cp); 1131 chip_cpu_startup(cp); 1132 1133 cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS; 1134 cpu_add_active(cp); 1135 mutex_exit(&cpu_lock); 1136 1137 add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi); 1138 1139 /* The base spl should still be at LOCK LEVEL here */ 1140 ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL)); 1141 set_base_spl(); /* Restore the spl to its proper value */ 1142 1143 (void) spl0(); /* enable interrupts */ 1144 1145 /* 1146 * Set up the CPU module for this CPU. This can't be done before 1147 * this CPU is made CPU_READY, because we may (in heterogeneous systems) 1148 * need to go load another CPU module. The act of attempting to load 1149 * a module may trigger a cross-call, which will ASSERT unless this 1150 * cpu is CPU_READY. 1151 */ 1152 cmi_init(); 1153 1154 if (x86_feature & X86_MCA) 1155 cmi_mca_init(); 1156 1157 if (boothowto & RB_DEBUG) 1158 kdi_dvec_cpu_init(cp); 1159 1160 /* 1161 * Setting the bit in cpu_ready_set must be the last operation in 1162 * processor initialization; the boot CPU will continue to boot once 1163 * it sees this bit set for all active CPUs. 1164 */ 1165 CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); 1166 1167 /* 1168 * Because mp_startup() gets fired off after init() starts, we 1169 * can't use the '?' trick to do 'boot -v' printing - so we 1170 * always direct the 'cpu .. online' messages to the log. 1171 */ 1172 cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 1173 cp->cpu_id); 1174 1175 /* 1176 * Now we are done with the startup thread, so free it up. 1177 */ 1178 thread_exit(); 1179 panic("mp_startup: cannot return"); 1180 /*NOTREACHED*/ 1181 } 1182 1183 1184 /* 1185 * Start CPU on user request. 1186 */ 1187 /* ARGSUSED */ 1188 int 1189 mp_cpu_start(struct cpu *cp) 1190 { 1191 ASSERT(MUTEX_HELD(&cpu_lock)); 1192 return (0); 1193 } 1194 1195 /* 1196 * Stop CPU on user request. 1197 */ 1198 /* ARGSUSED */ 1199 int 1200 mp_cpu_stop(struct cpu *cp) 1201 { 1202 extern int cbe_psm_timer_mode; 1203 ASSERT(MUTEX_HELD(&cpu_lock)); 1204 1205 /* 1206 * If TIMER_PERIODIC mode is used, CPU0 is the one running it; 1207 * can't stop it. (This is true only for machines with no TSC.) 1208 */ 1209 1210 if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0)) 1211 return (1); 1212 1213 return (0); 1214 } 1215 1216 /* 1217 * Power on CPU. 1218 */ 1219 /* ARGSUSED */ 1220 int 1221 mp_cpu_poweron(struct cpu *cp) 1222 { 1223 ASSERT(MUTEX_HELD(&cpu_lock)); 1224 return (ENOTSUP); /* not supported */ 1225 } 1226 1227 /* 1228 * Power off CPU. 1229 */ 1230 /* ARGSUSED */ 1231 int 1232 mp_cpu_poweroff(struct cpu *cp) 1233 { 1234 ASSERT(MUTEX_HELD(&cpu_lock)); 1235 return (ENOTSUP); /* not supported */ 1236 } 1237 1238 1239 /* 1240 * Take the specified CPU out of participation in interrupts. 1241 */ 1242 int 1243 cpu_disable_intr(struct cpu *cp) 1244 { 1245 if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) 1246 return (EBUSY); 1247 1248 cp->cpu_flags &= ~CPU_ENABLE; 1249 return (0); 1250 } 1251 1252 /* 1253 * Allow the specified CPU to participate in interrupts. 1254 */ 1255 void 1256 cpu_enable_intr(struct cpu *cp) 1257 { 1258 ASSERT(MUTEX_HELD(&cpu_lock)); 1259 cp->cpu_flags |= CPU_ENABLE; 1260 psm_enable_intr(cp->cpu_id); 1261 } 1262 1263 1264 1265 static ushort_t * 1266 mp_map_warm_reset_vector() 1267 { 1268 ushort_t *warm_reset_vector; 1269 1270 if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 1271 sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 1272 return (NULL); 1273 1274 /* 1275 * setup secondary cpu bios boot up vector 1276 */ 1277 *warm_reset_vector = (ushort_t)((caddr_t) 1278 ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 1279 + ((ulong_t)rm_platter_va & 0xf)); 1280 warm_reset_vector++; 1281 *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 1282 1283 --warm_reset_vector; 1284 return (warm_reset_vector); 1285 } 1286 1287 static void 1288 mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector) 1289 { 1290 psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 1291 } 1292 1293 void 1294 mp_cpu_faulted_enter(struct cpu *cp) 1295 { 1296 cmi_faulted_enter(cp); 1297 } 1298 1299 void 1300 mp_cpu_faulted_exit(struct cpu *cp) 1301 { 1302 cmi_faulted_exit(cp); 1303 } 1304 1305 /* 1306 * The following two routines are used as context operators on threads belonging 1307 * to processes with a private LDT (see sysi86). Due to the rarity of such 1308 * processes, these routines are currently written for best code readability and 1309 * organization rather than speed. We could avoid checking x86_feature at every 1310 * context switch by installing different context ops, depending on the 1311 * x86_feature flags, at LDT creation time -- one for each combination of fast 1312 * syscall feature flags. 1313 */ 1314 1315 /*ARGSUSED*/ 1316 void 1317 cpu_fast_syscall_disable(void *arg) 1318 { 1319 if (x86_feature & X86_SEP) 1320 cpu_sep_disable(); 1321 if (x86_feature & X86_ASYSC) 1322 cpu_asysc_disable(); 1323 } 1324 1325 /*ARGSUSED*/ 1326 void 1327 cpu_fast_syscall_enable(void *arg) 1328 { 1329 if (x86_feature & X86_SEP) 1330 cpu_sep_enable(); 1331 if (x86_feature & X86_ASYSC) 1332 cpu_asysc_enable(); 1333 } 1334 1335 static void 1336 cpu_sep_enable(void) 1337 { 1338 ASSERT(x86_feature & X86_SEP); 1339 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1340 1341 wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL); 1342 } 1343 1344 static void 1345 cpu_sep_disable(void) 1346 { 1347 ASSERT(x86_feature & X86_SEP); 1348 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1349 1350 /* 1351 * Setting the SYSENTER_CS_MSR register to 0 causes software executing 1352 * the sysenter or sysexit instruction to trigger a #gp fault. 1353 */ 1354 wrmsr(MSR_INTC_SEP_CS, 0ULL); 1355 } 1356 1357 static void 1358 cpu_asysc_enable(void) 1359 { 1360 ASSERT(x86_feature & X86_ASYSC); 1361 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1362 1363 wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) | 1364 (uint64_t)(uintptr_t)AMD_EFER_SCE); 1365 } 1366 1367 static void 1368 cpu_asysc_disable(void) 1369 { 1370 ASSERT(x86_feature & X86_ASYSC); 1371 ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 1372 1373 /* 1374 * Turn off the SCE (syscall enable) bit in the EFER register. Software 1375 * executing syscall or sysret with this bit off will incur a #ud trap. 1376 */ 1377 wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) & 1378 ~((uint64_t)(uintptr_t)AMD_EFER_SCE)); 1379 } 1380