1/* 2 * linux/arch/arm/kernel/head.S 3 * 4 * Copyright (C) 1994-2002 Russell King 5 * Copyright (c) 2003 ARM Limited 6 * All Rights Reserved 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * Kernel startup code for all 32-bit CPUs 13 */ 14#include <linux/linkage.h> 15#include <linux/init.h> 16 17#include <asm/assembler.h> 18#include <asm/domain.h> 19#include <asm/ptrace.h> 20#include <asm/asm-offsets.h> 21#include <asm/memory.h> 22#include <asm/thread_info.h> 23#include <asm/system.h> 24#include <asm/pgtable.h> 25 26#ifdef CONFIG_DEBUG_LL 27#include <mach/debug-macro.S> 28#endif 29 30/* 31 * swapper_pg_dir is the virtual address of the initial page table. 32 * We place the page tables 16K below KERNEL_RAM_VADDR. Therefore, we must 33 * make sure that KERNEL_RAM_VADDR is correctly set. Currently, we expect 34 * the least significant 16 bits to be 0x8000, but we could probably 35 * relax this restriction to KERNEL_RAM_VADDR >= PAGE_OFFSET + 0x4000. 36 */ 37#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 38#if (KERNEL_RAM_VADDR & 0xffff) != 0x8000 39#error KERNEL_RAM_VADDR must start at 0xXXXX8000 40#endif 41 42#ifdef CONFIG_ARM_LPAE 43 /* LPAE requires an additional page for the PGD */ 44#define PG_DIR_SIZE 0x5000 45#define PMD_ORDER 3 46#else 47#define PG_DIR_SIZE 0x4000 48#define PMD_ORDER 2 49#endif 50 51 .globl swapper_pg_dir 52 .equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE 53 54 .macro pgtbl, rd, phys 55 add \rd, \phys, #TEXT_OFFSET - PG_DIR_SIZE 56 .endm 57 58#ifdef CONFIG_XIP_KERNEL 59#define KERNEL_START XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) 60#define KERNEL_END _edata_loc 61#else 62#define KERNEL_START KERNEL_RAM_VADDR 63#define KERNEL_END _end 64#endif 65 66/* 67 * Kernel startup entry point. 68 * --------------------------- 69 * 70 * This is normally called from the decompressor code. The requirements 71 * are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0, 72 * r1 = machine nr, r2 = atags or dtb pointer. 73 * 74 * This code is mostly position independent, so if you link the kernel at 75 * 0xc0008000, you call this at __pa(0xc0008000). 76 * 77 * See linux/arch/arm/tools/mach-types for the complete list of machine 78 * numbers for r1. 79 * 80 * We're trying to keep crap to a minimum; DO NOT add any machine specific 81 * crap here - that's what the boot loader (or in extreme, well justified 82 * circumstances, zImage) is for. 83 */ 84 .arm 85 86 __HEAD 87ENTRY(stext) 88 89 THUMB( adr r9, BSYM(1f) ) @ Kernel is always entered in ARM. 90 THUMB( bx r9 ) @ If this is a Thumb-2 kernel, 91 THUMB( .thumb ) @ switch to Thumb now. 92 THUMB(1: ) 93 94 setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode 95 @ and irqs disabled 96 mrc p15, 0, r9, c0, c0 @ get processor id 97 bl __lookup_processor_type @ r5=procinfo r9=cpuid 98 movs r10, r5 @ invalid processor (r5=0)? 99 THUMB( it eq ) @ force fixup-able long branch encoding 100 beq __error_p @ yes, error 'p' 101 102#ifdef CONFIG_ARM_LPAE 103 mrc p15, 0, r3, c0, c1, 4 @ read ID_MMFR0 104 and r3, r3, #0xf @ extract VMSA support 105 cmp r3, #5 @ long-descriptor translation table format? 106 THUMB( it lo ) @ force fixup-able long branch encoding 107 blo __error_p @ only classic page table format 108#endif 109 110#ifndef CONFIG_XIP_KERNEL 111 adr r3, 2f 112 ldmia r3, {r4, r8} 113 sub r4, r3, r4 @ (PHYS_OFFSET - PAGE_OFFSET) 114 add r8, r8, r4 @ PHYS_OFFSET 115#else 116 ldr r8, =PHYS_OFFSET @ always constant in this case 117#endif 118 119 /* 120 * r1 = machine no, r2 = atags or dtb, 121 * r8 = phys_offset, r9 = cpuid, r10 = procinfo 122 */ 123 bl __vet_atags 124#ifdef CONFIG_SMP_ON_UP 125 bl __fixup_smp 126#endif 127#ifdef CONFIG_ARM_PATCH_PHYS_VIRT 128 bl __fixup_pv_table 129#endif 130 bl __create_page_tables 131 132 /* 133 * The following calls CPU specific code in a position independent 134 * manner. See arch/arm/mm/proc-*.S for details. r10 = base of 135 * xxx_proc_info structure selected by __lookup_processor_type 136 * above. On return, the CPU will be ready for the MMU to be 137 * turned on, and r0 will hold the CPU control register value. 138 */ 139 ldr r13, =__mmap_switched @ address to jump to after 140 @ mmu has been enabled 141 adr lr, BSYM(1f) @ return (PIC) address 142 mov r8, r4 @ set TTBR1 to swapper_pg_dir 143 ARM( add pc, r10, #PROCINFO_INITFUNC ) 144 THUMB( add r12, r10, #PROCINFO_INITFUNC ) 145 THUMB( mov pc, r12 ) 1461: b __enable_mmu 147ENDPROC(stext) 148 .ltorg 149#ifndef CONFIG_XIP_KERNEL 1502: .long . 151 .long PAGE_OFFSET 152#endif 153 154/* 155 * Setup the initial page tables. We only setup the barest 156 * amount which are required to get the kernel running, which 157 * generally means mapping in the kernel code. 158 * 159 * r8 = phys_offset, r9 = cpuid, r10 = procinfo 160 * 161 * Returns: 162 * r0, r3, r5-r7 corrupted 163 * r4 = physical page table address 164 */ 165__create_page_tables: 166 pgtbl r4, r8 @ page table address 167 168 /* 169 * Clear the swapper page table 170 */ 171 mov r0, r4 172 mov r3, #0 173 add r6, r0, #PG_DIR_SIZE 1741: str r3, [r0], #4 175 str r3, [r0], #4 176 str r3, [r0], #4 177 str r3, [r0], #4 178 teq r0, r6 179 bne 1b 180 181#ifdef CONFIG_ARM_LPAE 182 /* 183 * Build the PGD table (first level) to point to the PMD table. A PGD 184 * entry is 64-bit wide. 185 */ 186 mov r0, r4 187 add r3, r4, #0x1000 @ first PMD table address 188 orr r3, r3, #3 @ PGD block type 189 mov r6, #4 @ PTRS_PER_PGD 190 mov r7, #1 << (55 - 32) @ L_PGD_SWAPPER 1911: str r3, [r0], #4 @ set bottom PGD entry bits 192 str r7, [r0], #4 @ set top PGD entry bits 193 add r3, r3, #0x1000 @ next PMD table 194 subs r6, r6, #1 195 bne 1b 196 197 add r4, r4, #0x1000 @ point to the PMD tables 198#endif 199 200 ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags 201 202 /* 203 * Create identity mapping to cater for __enable_mmu. 204 * This identity mapping will be removed by paging_init(). 205 */ 206 adr r0, __turn_mmu_on_loc 207 ldmia r0, {r3, r5, r6} 208 sub r0, r0, r3 @ virt->phys offset 209 add r5, r5, r0 @ phys __turn_mmu_on 210 add r6, r6, r0 @ phys __turn_mmu_on_end 211 mov r5, r5, lsr #SECTION_SHIFT 212 mov r6, r6, lsr #SECTION_SHIFT 213 2141: orr r3, r7, r5, lsl #SECTION_SHIFT @ flags + kernel base 215 str r3, [r4, r5, lsl #PMD_ORDER] @ identity mapping 216 cmp r5, r6 217 addlo r5, r5, #1 @ next section 218 blo 1b 219 220 /* 221 * Now setup the pagetables for our kernel direct 222 * mapped region. 223 */ 224 mov r3, pc 225 mov r3, r3, lsr #SECTION_SHIFT 226 orr r3, r7, r3, lsl #SECTION_SHIFT 227 add r0, r4, #(KERNEL_START & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER) 228 str r3, [r0, #((KERNEL_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ORDER]! 229 ldr r6, =(KERNEL_END - 1) 230 add r0, r0, #1 << PMD_ORDER 231 add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) 2321: cmp r0, r6 233 add r3, r3, #1 << SECTION_SHIFT 234 strls r3, [r0], #1 << PMD_ORDER 235 bls 1b 236 237#ifdef CONFIG_XIP_KERNEL 238 /* 239 * Map some ram to cover our .data and .bss areas. 240 */ 241 add r3, r8, #TEXT_OFFSET 242 orr r3, r3, r7 243 add r0, r4, #(KERNEL_RAM_VADDR & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER) 244 str r3, [r0, #(KERNEL_RAM_VADDR & 0x00f00000) >> (SECTION_SHIFT - PMD_ORDER)]! 245 ldr r6, =(_end - 1) 246 add r0, r0, #4 247 add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) 2481: cmp r0, r6 249 add r3, r3, #1 << 20 250 strls r3, [r0], #4 251 bls 1b 252#endif 253 254 /* 255 * Then map boot params address in r2 or the first 1MB (2MB with LPAE) 256 * of ram if boot params address is not specified. 257 */ 258 mov r0, r2, lsr #SECTION_SHIFT 259 movs r0, r0, lsl #SECTION_SHIFT 260 moveq r0, r8 261 sub r3, r0, r8 262 add r3, r3, #PAGE_OFFSET 263 add r3, r4, r3, lsr #(SECTION_SHIFT - PMD_ORDER) 264 orr r6, r7, r0 265 str r6, [r3] 266 267#ifdef CONFIG_DEBUG_LL 268#ifndef CONFIG_DEBUG_ICEDCC 269 /* 270 * Map in IO space for serial debugging. 271 * This allows debug messages to be output 272 * via a serial console before paging_init. 273 */ 274 addruart r7, r3, r0 275 276 mov r3, r3, lsr #SECTION_SHIFT 277 mov r3, r3, lsl #PMD_ORDER 278 279 add r0, r4, r3 280 rsb r3, r3, #0x4000 @ PTRS_PER_PGD*sizeof(long) 281 cmp r3, #0x0800 @ limit to 512MB 282 movhi r3, #0x0800 283 add r6, r0, r3 284 mov r3, r7, lsr #SECTION_SHIFT 285 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags 286 orr r3, r7, r3, lsl #SECTION_SHIFT 287#ifdef CONFIG_ARM_LPAE 288 mov r7, #1 << (54 - 32) @ XN 289#else 290 orr r3, r3, #PMD_SECT_XN 291#endif 2921: str r3, [r0], #4 293#ifdef CONFIG_ARM_LPAE 294 str r7, [r0], #4 295#endif 296 add r3, r3, #1 << SECTION_SHIFT 297 cmp r0, r6 298 blo 1b 299 300#else /* CONFIG_DEBUG_ICEDCC */ 301 /* we don't need any serial debugging mappings for ICEDCC */ 302 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags 303#endif /* !CONFIG_DEBUG_ICEDCC */ 304 305#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS) 306 /* 307 * If we're using the NetWinder or CATS, we also need to map 308 * in the 16550-type serial port for the debug messages 309 */ 310 add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ORDER) 311 orr r3, r7, #0x7c000000 312 str r3, [r0] 313#endif 314#ifdef CONFIG_ARCH_RPC 315 /* 316 * Map in screen at 0x02000000 & SCREEN2_BASE 317 * Similar reasons here - for debug. This is 318 * only for Acorn RiscPC architectures. 319 */ 320 add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ORDER) 321 orr r3, r7, #0x02000000 322 str r3, [r0] 323 add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ORDER) 324 str r3, [r0] 325#endif 326#endif 327#ifdef CONFIG_ARM_LPAE 328 sub r4, r4, #0x1000 @ point to the PGD table 329#endif 330 mov pc, lr 331ENDPROC(__create_page_tables) 332 .ltorg 333 .align 334__turn_mmu_on_loc: 335 .long . 336 .long __turn_mmu_on 337 .long __turn_mmu_on_end 338 339#if defined(CONFIG_SMP) 340 __CPUINIT 341ENTRY(secondary_startup) 342 /* 343 * Common entry point for secondary CPUs. 344 * 345 * Ensure that we're in SVC mode, and IRQs are disabled. Lookup 346 * the processor type - there is no need to check the machine type 347 * as it has already been validated by the primary processor. 348 */ 349 setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 350 mrc p15, 0, r9, c0, c0 @ get processor id 351 bl __lookup_processor_type 352 movs r10, r5 @ invalid processor? 353 moveq r0, #'p' @ yes, error 'p' 354 THUMB( it eq ) @ force fixup-able long branch encoding 355 beq __error_p 356 357 /* 358 * Use the page tables supplied from __cpu_up. 359 */ 360 adr r4, __secondary_data 361 ldmia r4, {r5, r7, r12} @ address to jump to after 362 sub lr, r4, r5 @ mmu has been enabled 363 ldr r4, [r7, lr] @ get secondary_data.pgdir 364 add r7, r7, #4 365 ldr r8, [r7, lr] @ get secondary_data.swapper_pg_dir 366 adr lr, BSYM(__enable_mmu) @ return address 367 mov r13, r12 @ __secondary_switched address 368 ARM( add pc, r10, #PROCINFO_INITFUNC ) @ initialise processor 369 @ (return control reg) 370 THUMB( add r12, r10, #PROCINFO_INITFUNC ) 371 THUMB( mov pc, r12 ) 372ENDPROC(secondary_startup) 373 374 /* 375 * r6 = &secondary_data 376 */ 377ENTRY(__secondary_switched) 378 ldr sp, [r7, #4] @ get secondary_data.stack 379 mov fp, #0 380 b secondary_start_kernel 381ENDPROC(__secondary_switched) 382 383 .align 384 385 .type __secondary_data, %object 386__secondary_data: 387 .long . 388 .long secondary_data 389 .long __secondary_switched 390#endif /* defined(CONFIG_SMP) */ 391 392 393 394/* 395 * Setup common bits before finally enabling the MMU. Essentially 396 * this is just loading the page table pointer and domain access 397 * registers. 398 * 399 * r0 = cp#15 control register 400 * r1 = machine ID 401 * r2 = atags or dtb pointer 402 * r4 = page table pointer 403 * r9 = processor ID 404 * r13 = *virtual* address to jump to upon completion 405 */ 406__enable_mmu: 407#if defined(CONFIG_ALIGNMENT_TRAP) && __LINUX_ARM_ARCH__ < 6 408 orr r0, r0, #CR_A 409#else 410 bic r0, r0, #CR_A 411#endif 412#ifdef CONFIG_CPU_DCACHE_DISABLE 413 bic r0, r0, #CR_C 414#endif 415#ifdef CONFIG_CPU_BPREDICT_DISABLE 416 bic r0, r0, #CR_Z 417#endif 418#ifdef CONFIG_CPU_ICACHE_DISABLE 419 bic r0, r0, #CR_I 420#endif 421#ifdef CONFIG_ARM_LPAE 422 mov r5, #0 423 mcrr p15, 0, r4, r5, c2 @ load TTBR0 424#else 425 mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ 426 domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ 427 domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ 428 domain_val(DOMAIN_IO, DOMAIN_CLIENT)) 429 mcr p15, 0, r5, c3, c0, 0 @ load domain access register 430 mcr p15, 0, r4, c2, c0, 0 @ load page table pointer 431#endif 432 b __turn_mmu_on 433ENDPROC(__enable_mmu) 434 435/* 436 * Enable the MMU. This completely changes the structure of the visible 437 * memory space. You will not be able to trace execution through this. 438 * If you have an enquiry about this, *please* check the linux-arm-kernel 439 * mailing list archives BEFORE sending another post to the list. 440 * 441 * r0 = cp#15 control register 442 * r1 = machine ID 443 * r2 = atags or dtb pointer 444 * r9 = processor ID 445 * r13 = *virtual* address to jump to upon completion 446 * 447 * other registers depend on the function called upon completion 448 */ 449 .align 5 450 .pushsection .idmap.text, "ax" 451ENTRY(__turn_mmu_on) 452 mov r0, r0 453 instr_sync 454 mcr p15, 0, r0, c1, c0, 0 @ write control reg 455 mrc p15, 0, r3, c0, c0, 0 @ read id reg 456 instr_sync 457 mov r3, r3 458 mov r3, r13 459 mov pc, r3 460__turn_mmu_on_end: 461ENDPROC(__turn_mmu_on) 462 .popsection 463 464 465#ifdef CONFIG_SMP_ON_UP 466 __INIT 467__fixup_smp: 468 and r3, r9, #0x000f0000 @ architecture version 469 teq r3, #0x000f0000 @ CPU ID supported? 470 bne __fixup_smp_on_up @ no, assume UP 471 472 bic r3, r9, #0x00ff0000 473 bic r3, r3, #0x0000000f @ mask 0xff00fff0 474 mov r4, #0x41000000 475 orr r4, r4, #0x0000b000 476 orr r4, r4, #0x00000020 @ val 0x4100b020 477 teq r3, r4 @ ARM 11MPCore? 478 moveq pc, lr @ yes, assume SMP 479 480 mrc p15, 0, r0, c0, c0, 5 @ read MPIDR 481 and r0, r0, #0xc0000000 @ multiprocessing extensions and 482 teq r0, #0x80000000 @ not part of a uniprocessor system? 483 moveq pc, lr @ yes, assume SMP 484 485__fixup_smp_on_up: 486 adr r0, 1f 487 ldmia r0, {r3 - r5} 488 sub r3, r0, r3 489 add r4, r4, r3 490 add r5, r5, r3 491 b __do_fixup_smp_on_up 492ENDPROC(__fixup_smp) 493 494 .align 4951: .word . 496 .word __smpalt_begin 497 .word __smpalt_end 498 499 .pushsection .data 500 .globl smp_on_up 501smp_on_up: 502 ALT_SMP(.long 1) 503 ALT_UP(.long 0) 504 .popsection 505#endif 506 507 .text 508__do_fixup_smp_on_up: 509 cmp r4, r5 510 movhs pc, lr 511 ldmia r4!, {r0, r6} 512 ARM( str r6, [r0, r3] ) 513 THUMB( add r0, r0, r3 ) 514#ifdef __ARMEB__ 515 THUMB( mov r6, r6, ror #16 ) @ Convert word order for big-endian. 516#endif 517 THUMB( strh r6, [r0], #2 ) @ For Thumb-2, store as two halfwords 518 THUMB( mov r6, r6, lsr #16 ) @ to be robust against misaligned r3. 519 THUMB( strh r6, [r0] ) 520 b __do_fixup_smp_on_up 521ENDPROC(__do_fixup_smp_on_up) 522 523ENTRY(fixup_smp) 524 stmfd sp!, {r4 - r6, lr} 525 mov r4, r0 526 add r5, r0, r1 527 mov r3, #0 528 bl __do_fixup_smp_on_up 529 ldmfd sp!, {r4 - r6, pc} 530ENDPROC(fixup_smp) 531 532#ifdef CONFIG_ARM_PATCH_PHYS_VIRT 533 534/* __fixup_pv_table - patch the stub instructions with the delta between 535 * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be 16MiB aligned and 536 * can be expressed by an immediate shifter operand. The stub instruction 537 * has a form of '(add|sub) rd, rn, #imm'. 538 */ 539 __HEAD 540__fixup_pv_table: 541 adr r0, 1f 542 ldmia r0, {r3-r5, r7} 543 sub r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET 544 add r4, r4, r3 @ adjust table start address 545 add r5, r5, r3 @ adjust table end address 546 add r7, r7, r3 @ adjust __pv_phys_offset address 547 str r8, [r7] @ save computed PHYS_OFFSET to __pv_phys_offset 548 mov r6, r3, lsr #24 @ constant for add/sub instructions 549 teq r3, r6, lsl #24 @ must be 16MiB aligned 550THUMB( it ne @ cross section branch ) 551 bne __error 552 str r6, [r7, #4] @ save to __pv_offset 553 b __fixup_a_pv_table 554ENDPROC(__fixup_pv_table) 555 556 .align 5571: .long . 558 .long __pv_table_begin 559 .long __pv_table_end 5602: .long __pv_phys_offset 561 562 .text 563__fixup_a_pv_table: 564#ifdef CONFIG_THUMB2_KERNEL 565 lsls r6, #24 566 beq 2f 567 clz r7, r6 568 lsr r6, #24 569 lsl r6, r7 570 bic r6, #0x0080 571 lsrs r7, #1 572 orrcs r6, #0x0080 573 orr r6, r6, r7, lsl #12 574 orr r6, #0x4000 575 b 2f 5761: add r7, r3 577 ldrh ip, [r7, #2] 578 and ip, 0x8f00 579 orr ip, r6 @ mask in offset bits 31-24 580 strh ip, [r7, #2] 5812: cmp r4, r5 582 ldrcc r7, [r4], #4 @ use branch for delay slot 583 bcc 1b 584 bx lr 585#else 586 b 2f 5871: ldr ip, [r7, r3] 588 bic ip, ip, #0x000000ff 589 orr ip, ip, r6 @ mask in offset bits 31-24 590 str ip, [r7, r3] 5912: cmp r4, r5 592 ldrcc r7, [r4], #4 @ use branch for delay slot 593 bcc 1b 594 mov pc, lr 595#endif 596ENDPROC(__fixup_a_pv_table) 597 598ENTRY(fixup_pv_table) 599 stmfd sp!, {r4 - r7, lr} 600 ldr r2, 2f @ get address of __pv_phys_offset 601 mov r3, #0 @ no offset 602 mov r4, r0 @ r0 = table start 603 add r5, r0, r1 @ r1 = table size 604 ldr r6, [r2, #4] @ get __pv_offset 605 bl __fixup_a_pv_table 606 ldmfd sp!, {r4 - r7, pc} 607ENDPROC(fixup_pv_table) 608 609 .align 6102: .long __pv_phys_offset 611 612 .data 613 .globl __pv_phys_offset 614 .type __pv_phys_offset, %object 615__pv_phys_offset: 616 .long 0 617 .size __pv_phys_offset, . - __pv_phys_offset 618__pv_offset: 619 .long 0 620#endif 621 622#include "head-common.S" 623