1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24/* 25 * Copyright (c) 2010, Intel Corporation. 26 * All rights reserved. 27 */ 28 29#include <sys/asm_linkage.h> 30#include <sys/asm_misc.h> 31#include <sys/regset.h> 32#include <sys/privregs.h> 33#include <sys/x86_archext.h> 34 35#if !defined(__lint) 36#include <sys/segments.h> 37#include "assym.h" 38#endif 39 40/* 41 * Our assumptions: 42 * - We are running in real mode. 43 * - Interrupts are disabled. 44 * - Selectors are equal (cs == ds == ss) for all real mode code 45 * - The GDT, IDT, ktss and page directory has been built for us 46 * 47 * Our actions: 48 * Start CPU: 49 * - We start using our GDT by loading correct values in the 50 * selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL, 51 * gs=KGS_SEL). 52 * - We change over to using our IDT. 53 * - We load the default LDT into the hardware LDT register. 54 * - We load the default TSS into the hardware task register. 55 * - call mp_startup(void) indirectly through the T_PC 56 * Stop CPU: 57 * - Put CPU into halted state with interrupts disabled 58 * 59 */ 60 61#if defined(__lint) 62 63void 64real_mode_start_cpu(void) 65{} 66 67void 68real_mode_stop_cpu_stage1(void) 69{} 70 71void 72real_mode_stop_cpu_stage2(void) 73{} 74 75#else /* __lint */ 76 77#if defined(__amd64) 78 79 ENTRY_NP(real_mode_start_cpu) 80 81 /* 82 * NOTE: The GNU assembler automatically does the right thing to 83 * generate data size operand prefixes based on the code size 84 * generation mode (e.g. .code16, .code32, .code64) and as such 85 * prefixes need not be used on instructions EXCEPT in the case 86 * of address prefixes for code for which the reference is not 87 * automatically of the default operand size. 88 */ 89 .code16 90 cli 91 movw %cs, %ax 92 movw %ax, %ds /* load cs into ds */ 93 movw %ax, %ss /* and into ss */ 94 95 /* 96 * Helps in debugging by giving us the fault address. 97 * 98 * Remember to patch a hlt (0xf4) at cmntrap to get a good stack. 99 */ 100 movl $0xffc, %esp 101 movl %cr0, %eax 102 103 /* 104 * Enable protected-mode, write protect, and alignment mask 105 */ 106 orl $(CR0_PE|CR0_WP|CR0_AM), %eax 107 movl %eax, %cr0 108 109 /* 110 * Do a jmp immediately after writing to cr0 when enabling protected 111 * mode to clear the real mode prefetch queue (per Intel's docs) 112 */ 113 jmp pestart 114 115pestart: 116 /* 117 * 16-bit protected mode is now active, so prepare to turn on long 118 * mode. 119 * 120 * Note that we currently assume that if we're attempting to run a 121 * kernel compiled with (__amd64) #defined, the target CPU has long 122 * mode support. 123 */ 124 125#if 0 126 /* 127 * If there's a chance this might not be true, the following test should 128 * be done, with the no_long_mode branch then doing something 129 * appropriate: 130 */ 131 132 movl $0x80000000, %eax /* get largest extended CPUID */ 133 cpuid 134 cmpl $0x80000000, %eax /* check if > 0x80000000 */ 135 jbe no_long_mode /* nope, no long mode */ 136 movl $0x80000001, %eax 137 cpuid /* get extended feature flags */ 138 btl $29, %edx /* check for long mode */ 139 jnc no_long_mode /* long mode not supported */ 140#endif 141 142 /* 143 * Add any initial cr4 bits 144 */ 145 movl %cr4, %eax 146 addr32 orl CR4OFF, %eax 147 148 /* 149 * Enable PAE mode (CR4.PAE) 150 */ 151 orl $CR4_PAE, %eax 152 movl %eax, %cr4 153 154 /* 155 * Point cr3 to the 64-bit long mode page tables. 156 * 157 * Note that these MUST exist in 32-bit space, as we don't have 158 * a way to load %cr3 with a 64-bit base address for the page tables 159 * until the CPU is actually executing in 64-bit long mode. 160 */ 161 addr32 movl CR3OFF, %eax 162 movl %eax, %cr3 163 164 /* 165 * Set long mode enable in EFER (EFER.LME = 1) 166 */ 167 movl $MSR_AMD_EFER, %ecx 168 rdmsr 169 orl $AMD_EFER_LME, %eax 170 wrmsr 171 172 /* 173 * Finally, turn on paging (CR0.PG = 1) to activate long mode. 174 */ 175 movl %cr0, %eax 176 orl $CR0_PG, %eax 177 movl %eax, %cr0 178 179 /* 180 * The instruction after enabling paging in CR0 MUST be a branch. 181 */ 182 jmp long_mode_active 183 184long_mode_active: 185 /* 186 * Long mode is now active but since we're still running with the 187 * original 16-bit CS we're actually in 16-bit compatability mode. 188 * 189 * We have to load an intermediate GDT and IDT here that we know are 190 * in 32-bit space before we can use the kernel's GDT and IDT, which 191 * may be in the 64-bit address space, and since we're in compatability 192 * mode, we only have access to 16 and 32-bit instructions at the 193 * moment. 194 */ 195 addr32 lgdtl TEMPGDTOFF /* load temporary GDT */ 196 addr32 lidtl TEMPIDTOFF /* load temporary IDT */ 197 198 /* 199 * Do a far transfer to 64-bit mode. Set the CS selector to a 64-bit 200 * long mode selector (CS.L=1) in the temporary 32-bit GDT and jump 201 * to the real mode platter address of long_mode 64 as until the 64-bit 202 * CS is in place we don't have access to 64-bit instructions and thus 203 * can't reference a 64-bit %rip. 204 */ 205 pushl $TEMP_CS64_SEL 206 addr32 pushl LM64OFF 207 lretl 208 209 .globl long_mode_64 210long_mode_64: 211 .code64 212 /* 213 * We are now running in long mode with a 64-bit CS (EFER.LMA=1, 214 * CS.L=1) so we now have access to 64-bit instructions. 215 * 216 * First, set the 64-bit GDT base. 217 */ 218 .globl rm_platter_pa 219 movl rm_platter_pa, %eax 220 lgdtq GDTROFF(%rax) /* load 64-bit GDT */ 221 222 /* 223 * Save the CPU number in %r11; get the value here since it's saved in 224 * the real mode platter. 225 */ 226 movl CPUNOFF(%rax), %r11d 227 228 /* 229 * Add rm_platter_pa to %rsp to point it to the same location as seen 230 * from 64-bit mode. 231 */ 232 addq %rax, %rsp 233 234 /* 235 * Now do an lretq to load CS with the appropriate selector for the 236 * kernel's 64-bit GDT and to start executing 64-bit setup code at the 237 * virtual address where boot originally loaded this code rather than 238 * the copy in the real mode platter's rm_code array as we've been 239 * doing so far. 240 */ 241 pushq $KCS_SEL 242 pushq $kernel_cs_code 243 lretq 244 .globl real_mode_start_cpu_end 245real_mode_start_cpu_end: 246 nop 247 248kernel_cs_code: 249 /* 250 * Complete the balance of the setup we need to before executing 251 * 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS). 252 */ 253 .globl rm_platter_va 254 movq rm_platter_va, %rax 255 lidtq IDTROFF(%rax) 256 257 movw $KDS_SEL, %ax 258 movw %ax, %ds 259 movw %ax, %es 260 movw %ax, %ss 261 262 movw $KTSS_SEL, %ax /* setup kernel TSS */ 263 ltr %ax 264 265 xorw %ax, %ax /* clear LDTR */ 266 lldt %ax 267 268 /* 269 * Set GS to the address of the per-cpu structure as contained in 270 * cpu[cpu_number]. 271 * 272 * Unfortunately there's no way to set the 64-bit gsbase with a mov, 273 * so we have to stuff the low 32 bits in %eax and the high 32 bits in 274 * %edx, then call wrmsr. 275 */ 276 leaq cpu(%rip), %rdi 277 movl (%rdi, %r11, 8), %eax 278 movl 4(%rdi, %r11, 8), %edx 279 movl $MSR_AMD_GSBASE, %ecx 280 wrmsr 281 282 /* 283 * Init FS and KernelGSBase. 284 * 285 * Based on code in mlsetup(), set them both to 8G (which shouldn't be 286 * valid until some 64-bit processes run); this will then cause an 287 * exception in any code that tries to index off them before they are 288 * properly setup. 289 */ 290 xorl %eax, %eax /* low 32 bits = 0 */ 291 movl $2, %edx /* high 32 bits = 2 */ 292 movl $MSR_AMD_FSBASE, %ecx 293 wrmsr 294 295 movl $MSR_AMD_KGSBASE, %ecx 296 wrmsr 297 298 /* 299 * Init %rsp to the exception stack set in tss_ist1 and create a legal 300 * AMD64 ABI stack frame 301 */ 302 movq %gs:CPU_TSS, %rax 303 movq TSS_IST1(%rax), %rsp 304 pushq $0 /* null return address */ 305 pushq $0 /* null frame pointer terminates stack trace */ 306 movq %rsp, %rbp /* stack aligned on 16-byte boundary */ 307 308 movq %cr0, %rax 309 andq $~(CR0_TS|CR0_EM), %rax /* clear emulate math chip bit */ 310 orq $(CR0_MP|CR0_NE), %rax 311 movq %rax, %cr0 /* set machine status word */ 312 313 /* 314 * Before going any further, enable usage of page table NX bit if 315 * that's how our page tables are set up. 316 */ 317 bt $X86FSET_NX, x86_featureset(%rip) 318 jnc 1f 319 movl $MSR_AMD_EFER, %ecx 320 rdmsr 321 orl $AMD_EFER_NXE, %eax 322 wrmsr 3231: 324 325 /* 326 * Complete the rest of the setup and call mp_startup(). 327 */ 328 movq %gs:CPU_THREAD, %rax /* get thread ptr */ 329 call *T_PC(%rax) /* call mp_startup */ 330 /* not reached */ 331 int $20 /* whoops, returned somehow! */ 332 333 SET_SIZE(real_mode_start_cpu) 334 335#elif defined(__i386) 336 337 ENTRY_NP(real_mode_start_cpu) 338 339#if !defined(__GNUC_AS__) 340 341 cli 342 D16 movw %cs, %eax 343 movw %eax, %ds /* load cs into ds */ 344 movw %eax, %ss /* and into ss */ 345 346 /* 347 * Helps in debugging by giving us the fault address. 348 * 349 * Remember to patch a hlt (0xf4) at cmntrap to get a good stack. 350 */ 351 D16 movl $0xffc, %esp 352 353 D16 A16 lgdt %cs:GDTROFF 354 D16 A16 lidt %cs:IDTROFF 355 D16 A16 movl %cs:CR4OFF, %eax /* set up CR4, if desired */ 356 D16 andl %eax, %eax 357 D16 A16 je no_cr4 358 359 D16 movl %eax, %ecx 360 D16 movl %cr4, %eax 361 D16 orl %ecx, %eax 362 D16 movl %eax, %cr4 363no_cr4: 364 D16 A16 movl %cs:CR3OFF, %eax 365 A16 movl %eax, %cr3 366 movl %cr0, %eax 367 368 /* 369 * Enable protected-mode, paging, write protect, and alignment mask 370 */ 371 D16 orl $[CR0_PG|CR0_PE|CR0_WP|CR0_AM], %eax 372 movl %eax, %cr0 373 jmp pestart 374 375pestart: 376 D16 pushl $KCS_SEL 377 D16 pushl $kernel_cs_code 378 D16 lret 379 .globl real_mode_start_cpu_end 380real_mode_start_cpu_end: 381 nop 382 383 .globl kernel_cs_code 384kernel_cs_code: 385 /* 386 * At this point we are with kernel's cs and proper eip. 387 * 388 * We will be executing not from the copy in real mode platter, 389 * but from the original code where boot loaded us. 390 * 391 * By this time GDT and IDT are loaded as is cr3. 392 */ 393 movw $KFS_SEL,%eax 394 movw %eax,%fs 395 movw $KGS_SEL,%eax 396 movw %eax,%gs 397 movw $KDS_SEL,%eax 398 movw %eax,%ds 399 movw %eax,%es 400 movl %gs:CPU_TSS,%esi 401 movw %eax,%ss 402 movl TSS_ESP0(%esi),%esp 403 movw $KTSS_SEL,%ax 404 ltr %ax 405 xorw %ax, %ax /* clear LDTR */ 406 lldt %ax 407 movl %cr0,%edx 408 andl $-1![CR0_TS|CR0_EM],%edx /* clear emulate math chip bit */ 409 orl $[CR0_MP|CR0_NE],%edx 410 movl %edx,%cr0 /* set machine status word */ 411 412 /* 413 * Before going any further, enable usage of page table NX bit if 414 * that's how our page tables are set up. 415 */ 416 bt $X86FSET_NX, x86_featureset 417 jnc 1f 418 movl %cr4, %ecx 419 andl $CR4_PAE, %ecx 420 jz 1f 421 movl $MSR_AMD_EFER, %ecx 422 rdmsr 423 orl $AMD_EFER_NXE, %eax 424 wrmsr 4251: 426 movl %gs:CPU_THREAD, %eax /* get thread ptr */ 427 call *T_PC(%eax) /* call mp_startup */ 428 /* not reached */ 429 int $20 /* whoops, returned somehow! */ 430 431#else 432 433 cli 434 mov %cs, %ax 435 mov %eax, %ds /* load cs into ds */ 436 mov %eax, %ss /* and into ss */ 437 438 /* 439 * Helps in debugging by giving us the fault address. 440 * 441 * Remember to patch a hlt (0xf4) at cmntrap to get a good stack. 442 */ 443 D16 mov $0xffc, %esp 444 445 D16 A16 lgdtl %cs:GDTROFF 446 D16 A16 lidtl %cs:IDTROFF 447 D16 A16 mov %cs:CR4OFF, %eax /* set up CR4, if desired */ 448 D16 and %eax, %eax 449 D16 A16 je no_cr4 450 451 D16 mov %eax, %ecx 452 D16 mov %cr4, %eax 453 D16 or %ecx, %eax 454 D16 mov %eax, %cr4 455no_cr4: 456 D16 A16 mov %cs:CR3OFF, %eax 457 A16 mov %eax, %cr3 458 mov %cr0, %eax 459 460 /* 461 * Enable protected-mode, paging, write protect, and alignment mask 462 */ 463 D16 or $(CR0_PG|CR0_PE|CR0_WP|CR0_AM), %eax 464 mov %eax, %cr0 465 jmp pestart 466 467pestart: 468 D16 pushl $KCS_SEL 469 D16 pushl $kernel_cs_code 470 D16 lret 471 .globl real_mode_start_cpu_end 472real_mode_start_cpu_end: 473 nop 474 .globl kernel_cs_code 475kernel_cs_code: 476 /* 477 * At this point we are with kernel's cs and proper eip. 478 * 479 * We will be executing not from the copy in real mode platter, 480 * but from the original code where boot loaded us. 481 * 482 * By this time GDT and IDT are loaded as is cr3. 483 */ 484 mov $KFS_SEL, %ax 485 mov %eax, %fs 486 mov $KGS_SEL, %ax 487 mov %eax, %gs 488 mov $KDS_SEL, %ax 489 mov %eax, %ds 490 mov %eax, %es 491 mov %gs:CPU_TSS, %esi 492 mov %eax, %ss 493 mov TSS_ESP0(%esi), %esp 494 mov $(KTSS_SEL), %ax 495 ltr %ax 496 xorw %ax, %ax /* clear LDTR */ 497 lldt %ax 498 mov %cr0, %edx 499 and $~(CR0_TS|CR0_EM), %edx /* clear emulate math chip bit */ 500 or $(CR0_MP|CR0_NE), %edx 501 mov %edx, %cr0 /* set machine status word */ 502 503 /* 504 * Before going any farther, enable usage of page table NX bit if 505 * that's how our page tables are set up. 506 */ 507 bt $X86FSET_NX, x86_featureset 508 jnc 1f 509 movl %cr4, %ecx 510 andl $CR4_PAE, %ecx 511 jz 1f 512 movl $MSR_AMD_EFER, %ecx 513 rdmsr 514 orl $AMD_EFER_NXE, %eax 515 wrmsr 5161: 517 mov %gs:CPU_THREAD, %eax /* get thread ptr */ 518 call *T_PC(%eax) /* call mp_startup */ 519 /* not reached */ 520 int $20 /* whoops, returned somehow! */ 521#endif 522 523 SET_SIZE(real_mode_start_cpu) 524 525#endif /* __amd64 */ 526 527#if defined(__amd64) 528 529 ENTRY_NP(real_mode_stop_cpu_stage1) 530 531#if !defined(__GNUC_AS__) 532 533 /* 534 * For vulcan as we need to do a .code32 and mentally invert the 535 * meaning of the addr16 and data16 prefixes to get 32-bit access when 536 * generating code to be executed in 16-bit mode (sigh...) 537 */ 538 .code32 539 cli 540 movw %cs, %ax 541 movw %ax, %ds /* load cs into ds */ 542 movw %ax, %ss /* and into ss */ 543 544 /* 545 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code 546 */ 547 movw $CPUHALTCODEOFF, %ax 548 .byte 0xff, 0xe0 /* jmp *%ax */ 549 550#else /* __GNUC_AS__ */ 551 552 /* 553 * NOTE: The GNU assembler automatically does the right thing to 554 * generate data size operand prefixes based on the code size 555 * generation mode (e.g. .code16, .code32, .code64) and as such 556 * prefixes need not be used on instructions EXCEPT in the case 557 * of address prefixes for code for which the reference is not 558 * automatically of the default operand size. 559 */ 560 .code16 561 cli 562 movw %cs, %ax 563 movw %ax, %ds /* load cs into ds */ 564 movw %ax, %ss /* and into ss */ 565 566 /* 567 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code 568 */ 569 movw $CPUHALTCODEOFF, %ax 570 jmp *%ax 571 572#endif /* !__GNUC_AS__ */ 573 574 .globl real_mode_stop_cpu_stage1_end 575real_mode_stop_cpu_stage1_end: 576 nop 577 578 SET_SIZE(real_mode_stop_cpu_stage1) 579 580#elif defined(__i386) 581 582 ENTRY_NP(real_mode_stop_cpu_stage1) 583 584#if !defined(__GNUC_AS__) 585 586 cli 587 D16 movw %cs, %eax 588 movw %eax, %ds /* load cs into ds */ 589 movw %eax, %ss /* and into ss */ 590 591 /* 592 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code 593 */ 594 movw $CPUHALTCODEOFF, %ax 595 .byte 0xff, 0xe0 /* jmp *%ax */ 596 597#else /* __GNUC_AS__ */ 598 599 cli 600 mov %cs, %ax 601 mov %eax, %ds /* load cs into ds */ 602 mov %eax, %ss /* and into ss */ 603 604 /* 605 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code 606 */ 607 movw $CPUHALTCODEOFF, %ax 608 jmp *%ax 609 610#endif /* !__GNUC_AS__ */ 611 612 .globl real_mode_stop_cpu_stage1_end 613real_mode_stop_cpu_stage1_end: 614 nop 615 616 SET_SIZE(real_mode_stop_cpu_stage1) 617 618#endif /* __amd64 */ 619 620 ENTRY_NP(real_mode_stop_cpu_stage2) 621 622 movw $0xdead, %ax 623 movw %ax, CPUHALTEDOFF 624 625real_mode_stop_cpu_loop: 626 /* 627 * Put CPU into halted state. 628 * Only INIT, SMI, NMI could break the loop. 629 */ 630 hlt 631 jmp real_mode_stop_cpu_loop 632 633 .globl real_mode_stop_cpu_stage2_end 634real_mode_stop_cpu_stage2_end: 635 nop 636 637 SET_SIZE(real_mode_stop_cpu_stage2) 638 639#endif /* __lint */ 640