1/* 2 * 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * 5 * Enhanced CPU detection and feature setting code by Mike Jagdis 6 * and Martin Mares, November 1997. 7 */ 8 9.text 10#include <linux/threads.h> 11#include <linux/init.h> 12#include <linux/linkage.h> 13#include <asm/segment.h> 14#include <asm/page_types.h> 15#include <asm/pgtable_types.h> 16#include <asm/cache.h> 17#include <asm/thread_info.h> 18#include <asm/asm-offsets.h> 19#include <asm/setup.h> 20#include <asm/processor-flags.h> 21#include <asm/msr-index.h> 22#include <asm/cpufeatures.h> 23#include <asm/percpu.h> 24#include <asm/nops.h> 25#include <asm/bootparam.h> 26#include <asm/export.h> 27#include <asm/pgtable_32.h> 28 29/* Physical address */ 30#define pa(X) ((X) - __PAGE_OFFSET) 31 32/* 33 * References to members of the new_cpu_data structure. 34 */ 35 36#define X86 new_cpu_data+CPUINFO_x86 37#define X86_VENDOR new_cpu_data+CPUINFO_x86_vendor 38#define X86_MODEL new_cpu_data+CPUINFO_x86_model 39#define X86_MASK new_cpu_data+CPUINFO_x86_mask 40#define X86_HARD_MATH new_cpu_data+CPUINFO_hard_math 41#define X86_CPUID new_cpu_data+CPUINFO_cpuid_level 42#define X86_CAPABILITY new_cpu_data+CPUINFO_x86_capability 43#define X86_VENDOR_ID new_cpu_data+CPUINFO_x86_vendor_id 44 45 46#define SIZEOF_PTREGS 17*4 47 48/* 49 * Worst-case size of the kernel mapping we need to make: 50 * a relocatable kernel can live anywhere in lowmem, so we need to be able 51 * to map all of lowmem. 52 */ 53KERNEL_PAGES = LOWMEM_PAGES 54 55INIT_MAP_SIZE = PAGE_TABLE_SIZE(KERNEL_PAGES) * PAGE_SIZE 56RESERVE_BRK(pagetables, INIT_MAP_SIZE) 57 58/* 59 * 32-bit kernel entrypoint; only used by the boot CPU. On entry, 60 * %esi points to the real-mode code as a 32-bit pointer. 61 * CS and DS must be 4 GB flat segments, but we don't depend on 62 * any particular GDT layout, because we load our own as soon as we 63 * can. 64 */ 65__HEAD 66ENTRY(startup_32) 67 movl pa(initial_stack),%ecx 68 69 /* test KEEP_SEGMENTS flag to see if the bootloader is asking 70 us to not reload segments */ 71 testb $KEEP_SEGMENTS, BP_loadflags(%esi) 72 jnz 2f 73 74/* 75 * Set segments to known values. 76 */ 77 lgdt pa(boot_gdt_descr) 78 movl $(__BOOT_DS),%eax 79 movl %eax,%ds 80 movl %eax,%es 81 movl %eax,%fs 82 movl %eax,%gs 83 movl %eax,%ss 842: 85 leal -__PAGE_OFFSET(%ecx),%esp 86 87/* 88 * Clear BSS first so that there are no surprises... 89 */ 90 cld 91 xorl %eax,%eax 92 movl $pa(__bss_start),%edi 93 movl $pa(__bss_stop),%ecx 94 subl %edi,%ecx 95 shrl $2,%ecx 96 rep ; stosl 97/* 98 * Copy bootup parameters out of the way. 99 * Note: %esi still has the pointer to the real-mode data. 100 * With the kexec as boot loader, parameter segment might be loaded beyond 101 * kernel image and might not even be addressable by early boot page tables. 102 * (kexec on panic case). Hence copy out the parameters before initializing 103 * page tables. 104 */ 105 movl $pa(boot_params),%edi 106 movl $(PARAM_SIZE/4),%ecx 107 cld 108 rep 109 movsl 110 movl pa(boot_params) + NEW_CL_POINTER,%esi 111 andl %esi,%esi 112 jz 1f # No command line 113 movl $pa(boot_command_line),%edi 114 movl $(COMMAND_LINE_SIZE/4),%ecx 115 rep 116 movsl 1171: 118 119#ifdef CONFIG_OLPC 120 /* save OFW's pgdir table for later use when calling into OFW */ 121 movl %cr3, %eax 122 movl %eax, pa(olpc_ofw_pgd) 123#endif 124 125#ifdef CONFIG_MICROCODE 126 /* Early load ucode on BSP. */ 127 call load_ucode_bsp 128#endif 129 130 /* Create early pagetables. */ 131 call mk_early_pgtbl_32 132 133 /* Do early initialization of the fixmap area */ 134 movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax 135#ifdef CONFIG_X86_PAE 136#define KPMDS (((-__PAGE_OFFSET) >> 30) & 3) /* Number of kernel PMDs */ 137 movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8) 138#else 139 movl %eax,pa(initial_page_table+0xffc) 140#endif 141 142#ifdef CONFIG_PARAVIRT 143 /* This is can only trip for a broken bootloader... */ 144 cmpw $0x207, pa(boot_params + BP_version) 145 jb .Ldefault_entry 146 147 /* Paravirt-compatible boot parameters. Look to see what architecture 148 we're booting under. */ 149 movl pa(boot_params + BP_hardware_subarch), %eax 150 cmpl $num_subarch_entries, %eax 151 jae .Lbad_subarch 152 153 movl pa(subarch_entries)(,%eax,4), %eax 154 subl $__PAGE_OFFSET, %eax 155 jmp *%eax 156 157.Lbad_subarch: 158WEAK(lguest_entry) 159WEAK(xen_entry) 160 /* Unknown implementation; there's really 161 nothing we can do at this point. */ 162 ud2a 163 164 __INITDATA 165 166subarch_entries: 167 .long .Ldefault_entry /* normal x86/PC */ 168 .long lguest_entry /* lguest hypervisor */ 169 .long xen_entry /* Xen hypervisor */ 170 .long .Ldefault_entry /* Moorestown MID */ 171num_subarch_entries = (. - subarch_entries) / 4 172.previous 173#else 174 jmp .Ldefault_entry 175#endif /* CONFIG_PARAVIRT */ 176 177#ifdef CONFIG_HOTPLUG_CPU 178/* 179 * Boot CPU0 entry point. It's called from play_dead(). Everything has been set 180 * up already except stack. We just set up stack here. Then call 181 * start_secondary(). 182 */ 183ENTRY(start_cpu0) 184 movl initial_stack, %ecx 185 movl %ecx, %esp 186 call *(initial_code) 1871: jmp 1b 188ENDPROC(start_cpu0) 189#endif 190 191/* 192 * Non-boot CPU entry point; entered from trampoline.S 193 * We can't lgdt here, because lgdt itself uses a data segment, but 194 * we know the trampoline has already loaded the boot_gdt for us. 195 * 196 * If cpu hotplug is not supported then this code can go in init section 197 * which will be freed later 198 */ 199ENTRY(startup_32_smp) 200 cld 201 movl $(__BOOT_DS),%eax 202 movl %eax,%ds 203 movl %eax,%es 204 movl %eax,%fs 205 movl %eax,%gs 206 movl pa(initial_stack),%ecx 207 movl %eax,%ss 208 leal -__PAGE_OFFSET(%ecx),%esp 209 210#ifdef CONFIG_MICROCODE 211 /* Early load ucode on AP. */ 212 call load_ucode_ap 213#endif 214 215.Ldefault_entry: 216#define CR0_STATE (X86_CR0_PE | X86_CR0_MP | X86_CR0_ET | \ 217 X86_CR0_NE | X86_CR0_WP | X86_CR0_AM | \ 218 X86_CR0_PG) 219 movl $(CR0_STATE & ~X86_CR0_PG),%eax 220 movl %eax,%cr0 221 222/* 223 * We want to start out with EFLAGS unambiguously cleared. Some BIOSes leave 224 * bits like NT set. This would confuse the debugger if this code is traced. So 225 * initialize them properly now before switching to protected mode. That means 226 * DF in particular (even though we have cleared it earlier after copying the 227 * command line) because GCC expects it. 228 */ 229 pushl $0 230 popfl 231 232/* 233 * New page tables may be in 4Mbyte page mode and may be using the global pages. 234 * 235 * NOTE! If we are on a 486 we may have no cr4 at all! Specifically, cr4 exists 236 * if and only if CPUID exists and has flags other than the FPU flag set. 237 */ 238 movl $-1,pa(X86_CPUID) # preset CPUID level 239 movl $X86_EFLAGS_ID,%ecx 240 pushl %ecx 241 popfl # set EFLAGS=ID 242 pushfl 243 popl %eax # get EFLAGS 244 testl $X86_EFLAGS_ID,%eax # did EFLAGS.ID remained set? 245 jz .Lenable_paging # hw disallowed setting of ID bit 246 # which means no CPUID and no CR4 247 248 xorl %eax,%eax 249 cpuid 250 movl %eax,pa(X86_CPUID) # save largest std CPUID function 251 252 movl $1,%eax 253 cpuid 254 andl $~1,%edx # Ignore CPUID.FPU 255 jz .Lenable_paging # No flags or only CPUID.FPU = no CR4 256 257 movl pa(mmu_cr4_features),%eax 258 movl %eax,%cr4 259 260 testb $X86_CR4_PAE, %al # check if PAE is enabled 261 jz .Lenable_paging 262 263 /* Check if extended functions are implemented */ 264 movl $0x80000000, %eax 265 cpuid 266 /* Value must be in the range 0x80000001 to 0x8000ffff */ 267 subl $0x80000001, %eax 268 cmpl $(0x8000ffff-0x80000001), %eax 269 ja .Lenable_paging 270 271 /* Clear bogus XD_DISABLE bits */ 272 call verify_cpu 273 274 mov $0x80000001, %eax 275 cpuid 276 /* Execute Disable bit supported? */ 277 btl $(X86_FEATURE_NX & 31), %edx 278 jnc .Lenable_paging 279 280 /* Setup EFER (Extended Feature Enable Register) */ 281 movl $MSR_EFER, %ecx 282 rdmsr 283 284 btsl $_EFER_NX, %eax 285 /* Make changes effective */ 286 wrmsr 287 288.Lenable_paging: 289 290/* 291 * Enable paging 292 */ 293 movl $pa(initial_page_table), %eax 294 movl %eax,%cr3 /* set the page table pointer.. */ 295 movl $CR0_STATE,%eax 296 movl %eax,%cr0 /* ..and set paging (PG) bit */ 297 ljmp $__BOOT_CS,$1f /* Clear prefetch and normalize %eip */ 2981: 299 /* Shift the stack pointer to a virtual address */ 300 addl $__PAGE_OFFSET, %esp 301 302/* 303 * start system 32-bit setup. We need to re-do some of the things done 304 * in 16-bit mode for the "real" operations. 305 */ 306 movl setup_once_ref,%eax 307 andl %eax,%eax 308 jz 1f # Did we do this already? 309 call *%eax 3101: 311 312/* 313 * Check if it is 486 314 */ 315 movb $4,X86 # at least 486 316 cmpl $-1,X86_CPUID 317 je .Lis486 318 319 /* get vendor info */ 320 xorl %eax,%eax # call CPUID with 0 -> return vendor ID 321 cpuid 322 movl %eax,X86_CPUID # save CPUID level 323 movl %ebx,X86_VENDOR_ID # lo 4 chars 324 movl %edx,X86_VENDOR_ID+4 # next 4 chars 325 movl %ecx,X86_VENDOR_ID+8 # last 4 chars 326 327 orl %eax,%eax # do we have processor info as well? 328 je .Lis486 329 330 movl $1,%eax # Use the CPUID instruction to get CPU type 331 cpuid 332 movb %al,%cl # save reg for future use 333 andb $0x0f,%ah # mask processor family 334 movb %ah,X86 335 andb $0xf0,%al # mask model 336 shrb $4,%al 337 movb %al,X86_MODEL 338 andb $0x0f,%cl # mask mask revision 339 movb %cl,X86_MASK 340 movl %edx,X86_CAPABILITY 341 342.Lis486: 343 movl $0x50022,%ecx # set AM, WP, NE and MP 344 movl %cr0,%eax 345 andl $0x80000011,%eax # Save PG,PE,ET 346 orl %ecx,%eax 347 movl %eax,%cr0 348 349 lgdt early_gdt_descr 350 lidt idt_descr 351 ljmp $(__KERNEL_CS),$1f 3521: movl $(__KERNEL_DS),%eax # reload all the segment registers 353 movl %eax,%ss # after changing gdt. 354 355 movl $(__USER_DS),%eax # DS/ES contains default USER segment 356 movl %eax,%ds 357 movl %eax,%es 358 359 movl $(__KERNEL_PERCPU), %eax 360 movl %eax,%fs # set this cpu's percpu 361 362 movl $(__KERNEL_STACK_CANARY),%eax 363 movl %eax,%gs 364 365 xorl %eax,%eax # Clear LDT 366 lldt %ax 367 368 call *(initial_code) 3691: jmp 1b 370ENDPROC(startup_32_smp) 371 372#include "verify_cpu.S" 373 374/* 375 * setup_once 376 * 377 * The setup work we only want to run on the BSP. 378 * 379 * Warning: %esi is live across this function. 380 */ 381__INIT 382setup_once: 383 /* 384 * Set up a idt with 256 interrupt gates that push zero if there 385 * is no error code and then jump to early_idt_handler_common. 386 * It doesn't actually load the idt - that needs to be done on 387 * each CPU. Interrupts are enabled elsewhere, when we can be 388 * relatively sure everything is ok. 389 */ 390 391 movl $idt_table,%edi 392 movl $early_idt_handler_array,%eax 393 movl $NUM_EXCEPTION_VECTORS,%ecx 3941: 395 movl %eax,(%edi) 396 movl %eax,4(%edi) 397 /* interrupt gate, dpl=0, present */ 398 movl $(0x8E000000 + __KERNEL_CS),2(%edi) 399 addl $EARLY_IDT_HANDLER_SIZE,%eax 400 addl $8,%edi 401 loop 1b 402 403 movl $256 - NUM_EXCEPTION_VECTORS,%ecx 404 movl $ignore_int,%edx 405 movl $(__KERNEL_CS << 16),%eax 406 movw %dx,%ax /* selector = 0x0010 = cs */ 407 movw $0x8E00,%dx /* interrupt gate - dpl=0, present */ 4082: 409 movl %eax,(%edi) 410 movl %edx,4(%edi) 411 addl $8,%edi 412 loop 2b 413 414#ifdef CONFIG_CC_STACKPROTECTOR 415 /* 416 * Configure the stack canary. The linker can't handle this by 417 * relocation. Manually set base address in stack canary 418 * segment descriptor. 419 */ 420 movl $gdt_page,%eax 421 movl $stack_canary,%ecx 422 movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax) 423 shrl $16, %ecx 424 movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax) 425 movb %ch, 8 * GDT_ENTRY_STACK_CANARY + 7(%eax) 426#endif 427 428 andl $0,setup_once_ref /* Once is enough, thanks */ 429 ret 430 431ENTRY(early_idt_handler_array) 432 # 36(%esp) %eflags 433 # 32(%esp) %cs 434 # 28(%esp) %eip 435 # 24(%rsp) error code 436 i = 0 437 .rept NUM_EXCEPTION_VECTORS 438 .ifeq (EXCEPTION_ERRCODE_MASK >> i) & 1 439 pushl $0 # Dummy error code, to make stack frame uniform 440 .endif 441 pushl $i # 20(%esp) Vector number 442 jmp early_idt_handler_common 443 i = i + 1 444 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc 445 .endr 446ENDPROC(early_idt_handler_array) 447 448early_idt_handler_common: 449 /* 450 * The stack is the hardware frame, an error code or zero, and the 451 * vector number. 452 */ 453 cld 454 455 incl %ss:early_recursion_flag 456 457 /* The vector number is in pt_regs->gs */ 458 459 cld 460 pushl %fs /* pt_regs->fs */ 461 movw $0, 2(%esp) /* clear high bits (some CPUs leave garbage) */ 462 pushl %es /* pt_regs->es */ 463 movw $0, 2(%esp) /* clear high bits (some CPUs leave garbage) */ 464 pushl %ds /* pt_regs->ds */ 465 movw $0, 2(%esp) /* clear high bits (some CPUs leave garbage) */ 466 pushl %eax /* pt_regs->ax */ 467 pushl %ebp /* pt_regs->bp */ 468 pushl %edi /* pt_regs->di */ 469 pushl %esi /* pt_regs->si */ 470 pushl %edx /* pt_regs->dx */ 471 pushl %ecx /* pt_regs->cx */ 472 pushl %ebx /* pt_regs->bx */ 473 474 /* Fix up DS and ES */ 475 movl $(__KERNEL_DS), %ecx 476 movl %ecx, %ds 477 movl %ecx, %es 478 479 /* Load the vector number into EDX */ 480 movl PT_GS(%esp), %edx 481 482 /* Load GS into pt_regs->gs and clear high bits */ 483 movw %gs, PT_GS(%esp) 484 movw $0, PT_GS+2(%esp) 485 486 movl %esp, %eax /* args are pt_regs (EAX), trapnr (EDX) */ 487 call early_fixup_exception 488 489 popl %ebx /* pt_regs->bx */ 490 popl %ecx /* pt_regs->cx */ 491 popl %edx /* pt_regs->dx */ 492 popl %esi /* pt_regs->si */ 493 popl %edi /* pt_regs->di */ 494 popl %ebp /* pt_regs->bp */ 495 popl %eax /* pt_regs->ax */ 496 popl %ds /* pt_regs->ds */ 497 popl %es /* pt_regs->es */ 498 popl %fs /* pt_regs->fs */ 499 popl %gs /* pt_regs->gs */ 500 decl %ss:early_recursion_flag 501 addl $4, %esp /* pop pt_regs->orig_ax */ 502 iret 503ENDPROC(early_idt_handler_common) 504 505/* This is the default interrupt "handler" :-) */ 506 ALIGN 507ignore_int: 508 cld 509#ifdef CONFIG_PRINTK 510 pushl %eax 511 pushl %ecx 512 pushl %edx 513 pushl %es 514 pushl %ds 515 movl $(__KERNEL_DS),%eax 516 movl %eax,%ds 517 movl %eax,%es 518 cmpl $2,early_recursion_flag 519 je hlt_loop 520 incl early_recursion_flag 521 pushl 16(%esp) 522 pushl 24(%esp) 523 pushl 32(%esp) 524 pushl 40(%esp) 525 pushl $int_msg 526 call printk 527 528 call dump_stack 529 530 addl $(5*4),%esp 531 popl %ds 532 popl %es 533 popl %edx 534 popl %ecx 535 popl %eax 536#endif 537 iret 538 539hlt_loop: 540 hlt 541 jmp hlt_loop 542ENDPROC(ignore_int) 543__INITDATA 544 .align 4 545GLOBAL(early_recursion_flag) 546 .long 0 547 548__REFDATA 549 .align 4 550ENTRY(initial_code) 551 .long i386_start_kernel 552ENTRY(setup_once_ref) 553 .long setup_once 554 555/* 556 * BSS section 557 */ 558__PAGE_ALIGNED_BSS 559 .align PAGE_SIZE 560#ifdef CONFIG_X86_PAE 561.globl initial_pg_pmd 562initial_pg_pmd: 563 .fill 1024*KPMDS,4,0 564#else 565.globl initial_page_table 566initial_page_table: 567 .fill 1024,4,0 568#endif 569initial_pg_fixmap: 570 .fill 1024,4,0 571.globl empty_zero_page 572empty_zero_page: 573 .fill 4096,1,0 574.globl swapper_pg_dir 575swapper_pg_dir: 576 .fill 1024,4,0 577EXPORT_SYMBOL(empty_zero_page) 578 579/* 580 * This starts the data section. 581 */ 582#ifdef CONFIG_X86_PAE 583__PAGE_ALIGNED_DATA 584 /* Page-aligned for the benefit of paravirt? */ 585 .align PAGE_SIZE 586ENTRY(initial_page_table) 587 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */ 588# if KPMDS == 3 589 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 590 .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0 591 .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x2000),0 592# elif KPMDS == 2 593 .long 0,0 594 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 595 .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0 596# elif KPMDS == 1 597 .long 0,0 598 .long 0,0 599 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 600# else 601# error "Kernel PMDs should be 1, 2 or 3" 602# endif 603 .align PAGE_SIZE /* needs to be page-sized too */ 604#endif 605 606.data 607.balign 4 608ENTRY(initial_stack) 609 /* 610 * The SIZEOF_PTREGS gap is a convention which helps the in-kernel 611 * unwinder reliably detect the end of the stack. 612 */ 613 .long init_thread_union + THREAD_SIZE - SIZEOF_PTREGS - \ 614 TOP_OF_KERNEL_STACK_PADDING; 615 616__INITRODATA 617int_msg: 618 .asciz "Unknown interrupt or fault at: %p %p %p\n" 619 620#include "../../x86/xen/xen-head.S" 621 622/* 623 * The IDT and GDT 'descriptors' are a strange 48-bit object 624 * only used by the lidt and lgdt instructions. They are not 625 * like usual segment descriptors - they consist of a 16-bit 626 * segment size, and 32-bit linear address value: 627 */ 628 629 .data 630.globl boot_gdt_descr 631.globl idt_descr 632 633 ALIGN 634# early boot GDT descriptor (must use 1:1 address mapping) 635 .word 0 # 32 bit align gdt_desc.address 636boot_gdt_descr: 637 .word __BOOT_DS+7 638 .long boot_gdt - __PAGE_OFFSET 639 640 .word 0 # 32-bit align idt_desc.address 641idt_descr: 642 .word IDT_ENTRIES*8-1 # idt contains 256 entries 643 .long idt_table 644 645# boot GDT descriptor (later on used by CPU#0): 646 .word 0 # 32 bit align gdt_desc.address 647ENTRY(early_gdt_descr) 648 .word GDT_ENTRIES*8-1 649 .long gdt_page /* Overwritten for secondary CPUs */ 650 651/* 652 * The boot_gdt must mirror the equivalent in setup.S and is 653 * used only for booting. 654 */ 655 .align L1_CACHE_BYTES 656ENTRY(boot_gdt) 657 .fill GDT_ENTRY_BOOT_CS,8,0 658 .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */ 659 .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */ 660