1/* 2 * linux/arch/arm/kernel/head.S 3 * 4 * Copyright (C) 1994-2002 Russell King 5 * Copyright (c) 2003 ARM Limited 6 * All Rights Reserved 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * Kernel startup code for all 32-bit CPUs 13 */ 14#include <linux/linkage.h> 15#include <linux/init.h> 16 17#include <asm/assembler.h> 18#include <asm/cp15.h> 19#include <asm/domain.h> 20#include <asm/ptrace.h> 21#include <asm/asm-offsets.h> 22#include <asm/memory.h> 23#include <asm/thread_info.h> 24#include <asm/pgtable.h> 25 26#ifdef CONFIG_DEBUG_LL 27#include <mach/debug-macro.S> 28#endif 29 30/* 31 * swapper_pg_dir is the virtual address of the initial page table. 32 * We place the page tables 16K below KERNEL_RAM_VADDR. Therefore, we must 33 * make sure that KERNEL_RAM_VADDR is correctly set. Currently, we expect 34 * the least significant 16 bits to be 0x8000, but we could probably 35 * relax this restriction to KERNEL_RAM_VADDR >= PAGE_OFFSET + 0x4000. 36 */ 37#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 38#if (KERNEL_RAM_VADDR & 0xffff) != 0x8000 39#error KERNEL_RAM_VADDR must start at 0xXXXX8000 40#endif 41 42#ifdef CONFIG_ARM_LPAE 43 /* LPAE requires an additional page for the PGD */ 44#define PG_DIR_SIZE 0x5000 45#define PMD_ORDER 3 46#else 47#define PG_DIR_SIZE 0x4000 48#define PMD_ORDER 2 49#endif 50 51 .globl swapper_pg_dir 52 .equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE 53 54 .macro pgtbl, rd, phys 55 add \rd, \phys, #TEXT_OFFSET - PG_DIR_SIZE 56 .endm 57 58#ifdef CONFIG_XIP_KERNEL 59#define KERNEL_START XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) 60#define KERNEL_END _edata_loc 61#else 62#define KERNEL_START KERNEL_RAM_VADDR 63#define KERNEL_END _end 64#endif 65 66/* 67 * Kernel startup entry point. 68 * --------------------------- 69 * 70 * This is normally called from the decompressor code. The requirements 71 * are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0, 72 * r1 = machine nr, r2 = atags or dtb pointer. 73 * 74 * This code is mostly position independent, so if you link the kernel at 75 * 0xc0008000, you call this at __pa(0xc0008000). 76 * 77 * See linux/arch/arm/tools/mach-types for the complete list of machine 78 * numbers for r1. 79 * 80 * We're trying to keep crap to a minimum; DO NOT add any machine specific 81 * crap here - that's what the boot loader (or in extreme, well justified 82 * circumstances, zImage) is for. 83 */ 84 .arm 85 86 __HEAD 87ENTRY(stext) 88 89 THUMB( adr r9, BSYM(1f) ) @ Kernel is always entered in ARM. 90 THUMB( bx r9 ) @ If this is a Thumb-2 kernel, 91 THUMB( .thumb ) @ switch to Thumb now. 92 THUMB(1: ) 93 94 setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode 95 @ and irqs disabled 96 mrc p15, 0, r9, c0, c0 @ get processor id 97 bl __lookup_processor_type @ r5=procinfo r9=cpuid 98 movs r10, r5 @ invalid processor (r5=0)? 99 THUMB( it eq ) @ force fixup-able long branch encoding 100 beq __error_p @ yes, error 'p' 101 102#ifdef CONFIG_ARM_LPAE 103 mrc p15, 0, r3, c0, c1, 4 @ read ID_MMFR0 104 and r3, r3, #0xf @ extract VMSA support 105 cmp r3, #5 @ long-descriptor translation table format? 106 THUMB( it lo ) @ force fixup-able long branch encoding 107 blo __error_p @ only classic page table format 108#endif 109 110#ifndef CONFIG_XIP_KERNEL 111 adr r3, 2f 112 ldmia r3, {r4, r8} 113 sub r4, r3, r4 @ (PHYS_OFFSET - PAGE_OFFSET) 114 add r8, r8, r4 @ PHYS_OFFSET 115#else 116 ldr r8, =PHYS_OFFSET @ always constant in this case 117#endif 118 119 /* 120 * r1 = machine no, r2 = atags or dtb, 121 * r8 = phys_offset, r9 = cpuid, r10 = procinfo 122 */ 123 bl __vet_atags 124#ifdef CONFIG_SMP_ON_UP 125 bl __fixup_smp 126#endif 127#ifdef CONFIG_ARM_PATCH_PHYS_VIRT 128 bl __fixup_pv_table 129#endif 130 bl __create_page_tables 131 132 /* 133 * The following calls CPU specific code in a position independent 134 * manner. See arch/arm/mm/proc-*.S for details. r10 = base of 135 * xxx_proc_info structure selected by __lookup_processor_type 136 * above. On return, the CPU will be ready for the MMU to be 137 * turned on, and r0 will hold the CPU control register value. 138 */ 139 ldr r13, =__mmap_switched @ address to jump to after 140 @ mmu has been enabled 141 adr lr, BSYM(1f) @ return (PIC) address 142 mov r8, r4 @ set TTBR1 to swapper_pg_dir 143 ARM( add pc, r10, #PROCINFO_INITFUNC ) 144 THUMB( add r12, r10, #PROCINFO_INITFUNC ) 145 THUMB( mov pc, r12 ) 1461: b __enable_mmu 147ENDPROC(stext) 148 .ltorg 149#ifndef CONFIG_XIP_KERNEL 1502: .long . 151 .long PAGE_OFFSET 152#endif 153 154/* 155 * Setup the initial page tables. We only setup the barest 156 * amount which are required to get the kernel running, which 157 * generally means mapping in the kernel code. 158 * 159 * r8 = phys_offset, r9 = cpuid, r10 = procinfo 160 * 161 * Returns: 162 * r0, r3, r5-r7 corrupted 163 * r4 = physical page table address 164 */ 165__create_page_tables: 166 pgtbl r4, r8 @ page table address 167 168 /* 169 * Clear the swapper page table 170 */ 171 mov r0, r4 172 mov r3, #0 173 add r6, r0, #PG_DIR_SIZE 1741: str r3, [r0], #4 175 str r3, [r0], #4 176 str r3, [r0], #4 177 str r3, [r0], #4 178 teq r0, r6 179 bne 1b 180 181#ifdef CONFIG_ARM_LPAE 182 /* 183 * Build the PGD table (first level) to point to the PMD table. A PGD 184 * entry is 64-bit wide. 185 */ 186 mov r0, r4 187 add r3, r4, #0x1000 @ first PMD table address 188 orr r3, r3, #3 @ PGD block type 189 mov r6, #4 @ PTRS_PER_PGD 190 mov r7, #1 << (55 - 32) @ L_PGD_SWAPPER 1911: str r3, [r0], #4 @ set bottom PGD entry bits 192 str r7, [r0], #4 @ set top PGD entry bits 193 add r3, r3, #0x1000 @ next PMD table 194 subs r6, r6, #1 195 bne 1b 196 197 add r4, r4, #0x1000 @ point to the PMD tables 198#endif 199 200 ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags 201 202 /* 203 * Create identity mapping to cater for __enable_mmu. 204 * This identity mapping will be removed by paging_init(). 205 */ 206 adr r0, __turn_mmu_on_loc 207 ldmia r0, {r3, r5, r6} 208 sub r0, r0, r3 @ virt->phys offset 209 add r5, r5, r0 @ phys __turn_mmu_on 210 add r6, r6, r0 @ phys __turn_mmu_on_end 211 mov r5, r5, lsr #SECTION_SHIFT 212 mov r6, r6, lsr #SECTION_SHIFT 213 2141: orr r3, r7, r5, lsl #SECTION_SHIFT @ flags + kernel base 215 str r3, [r4, r5, lsl #PMD_ORDER] @ identity mapping 216 cmp r5, r6 217 addlo r5, r5, #1 @ next section 218 blo 1b 219 220 /* 221 * Now setup the pagetables for our kernel direct 222 * mapped region. 223 */ 224 mov r3, pc 225 mov r3, r3, lsr #SECTION_SHIFT 226 orr r3, r7, r3, lsl #SECTION_SHIFT 227 add r0, r4, #(KERNEL_START & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER) 228 str r3, [r0, #((KERNEL_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ORDER]! 229 ldr r6, =(KERNEL_END - 1) 230 add r0, r0, #1 << PMD_ORDER 231 add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) 2321: cmp r0, r6 233 add r3, r3, #1 << SECTION_SHIFT 234 strls r3, [r0], #1 << PMD_ORDER 235 bls 1b 236 237#ifdef CONFIG_XIP_KERNEL 238 /* 239 * Map some ram to cover our .data and .bss areas. 240 */ 241 add r3, r8, #TEXT_OFFSET 242 orr r3, r3, r7 243 add r0, r4, #(KERNEL_RAM_VADDR & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER) 244 str r3, [r0, #(KERNEL_RAM_VADDR & 0x00f00000) >> (SECTION_SHIFT - PMD_ORDER)]! 245 ldr r6, =(_end - 1) 246 add r0, r0, #4 247 add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) 2481: cmp r0, r6 249 add r3, r3, #1 << 20 250 strls r3, [r0], #4 251 bls 1b 252#endif 253 254 /* 255 * Then map boot params address in r2 or the first 1MB (2MB with LPAE) 256 * of ram if boot params address is not specified. 257 */ 258 mov r0, r2, lsr #SECTION_SHIFT 259 movs r0, r0, lsl #SECTION_SHIFT 260 moveq r0, r8 261 sub r3, r0, r8 262 add r3, r3, #PAGE_OFFSET 263 add r3, r4, r3, lsr #(SECTION_SHIFT - PMD_ORDER) 264 orr r6, r7, r0 265 str r6, [r3] 266 267#ifdef CONFIG_DEBUG_LL 268#if !defined(CONFIG_DEBUG_ICEDCC) && !defined(CONFIG_DEBUG_SEMIHOSTING) 269 /* 270 * Map in IO space for serial debugging. 271 * This allows debug messages to be output 272 * via a serial console before paging_init. 273 */ 274 addruart r7, r3, r0 275 276 mov r3, r3, lsr #SECTION_SHIFT 277 mov r3, r3, lsl #PMD_ORDER 278 279 add r0, r4, r3 280 mov r3, r7, lsr #SECTION_SHIFT 281 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags 282 orr r3, r7, r3, lsl #SECTION_SHIFT 283#ifdef CONFIG_ARM_LPAE 284 mov r7, #1 << (54 - 32) @ XN 285#else 286 orr r3, r3, #PMD_SECT_XN 287#endif 288 str r3, [r0], #4 289#ifdef CONFIG_ARM_LPAE 290 str r7, [r0], #4 291#endif 292 293#else /* CONFIG_DEBUG_ICEDCC || CONFIG_DEBUG_SEMIHOSTING */ 294 /* we don't need any serial debugging mappings */ 295 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags 296#endif 297 298#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS) 299 /* 300 * If we're using the NetWinder or CATS, we also need to map 301 * in the 16550-type serial port for the debug messages 302 */ 303 add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ORDER) 304 orr r3, r7, #0x7c000000 305 str r3, [r0] 306#endif 307#ifdef CONFIG_ARCH_RPC 308 /* 309 * Map in screen at 0x02000000 & SCREEN2_BASE 310 * Similar reasons here - for debug. This is 311 * only for Acorn RiscPC architectures. 312 */ 313 add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ORDER) 314 orr r3, r7, #0x02000000 315 str r3, [r0] 316 add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ORDER) 317 str r3, [r0] 318#endif 319#endif 320#ifdef CONFIG_ARM_LPAE 321 sub r4, r4, #0x1000 @ point to the PGD table 322#endif 323 mov pc, lr 324ENDPROC(__create_page_tables) 325 .ltorg 326 .align 327__turn_mmu_on_loc: 328 .long . 329 .long __turn_mmu_on 330 .long __turn_mmu_on_end 331 332#if defined(CONFIG_SMP) 333 __CPUINIT 334ENTRY(secondary_startup) 335 /* 336 * Common entry point for secondary CPUs. 337 * 338 * Ensure that we're in SVC mode, and IRQs are disabled. Lookup 339 * the processor type - there is no need to check the machine type 340 * as it has already been validated by the primary processor. 341 */ 342 setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 343 mrc p15, 0, r9, c0, c0 @ get processor id 344 bl __lookup_processor_type 345 movs r10, r5 @ invalid processor? 346 moveq r0, #'p' @ yes, error 'p' 347 THUMB( it eq ) @ force fixup-able long branch encoding 348 beq __error_p 349 350 /* 351 * Use the page tables supplied from __cpu_up. 352 */ 353 adr r4, __secondary_data 354 ldmia r4, {r5, r7, r12} @ address to jump to after 355 sub lr, r4, r5 @ mmu has been enabled 356 ldr r4, [r7, lr] @ get secondary_data.pgdir 357 add r7, r7, #4 358 ldr r8, [r7, lr] @ get secondary_data.swapper_pg_dir 359 adr lr, BSYM(__enable_mmu) @ return address 360 mov r13, r12 @ __secondary_switched address 361 ARM( add pc, r10, #PROCINFO_INITFUNC ) @ initialise processor 362 @ (return control reg) 363 THUMB( add r12, r10, #PROCINFO_INITFUNC ) 364 THUMB( mov pc, r12 ) 365ENDPROC(secondary_startup) 366 367 /* 368 * r6 = &secondary_data 369 */ 370ENTRY(__secondary_switched) 371 ldr sp, [r7, #4] @ get secondary_data.stack 372 mov fp, #0 373 b secondary_start_kernel 374ENDPROC(__secondary_switched) 375 376 .align 377 378 .type __secondary_data, %object 379__secondary_data: 380 .long . 381 .long secondary_data 382 .long __secondary_switched 383#endif /* defined(CONFIG_SMP) */ 384 385 386 387/* 388 * Setup common bits before finally enabling the MMU. Essentially 389 * this is just loading the page table pointer and domain access 390 * registers. 391 * 392 * r0 = cp#15 control register 393 * r1 = machine ID 394 * r2 = atags or dtb pointer 395 * r4 = page table pointer 396 * r9 = processor ID 397 * r13 = *virtual* address to jump to upon completion 398 */ 399__enable_mmu: 400#if defined(CONFIG_ALIGNMENT_TRAP) && __LINUX_ARM_ARCH__ < 6 401 orr r0, r0, #CR_A 402#else 403 bic r0, r0, #CR_A 404#endif 405#ifdef CONFIG_CPU_DCACHE_DISABLE 406 bic r0, r0, #CR_C 407#endif 408#ifdef CONFIG_CPU_BPREDICT_DISABLE 409 bic r0, r0, #CR_Z 410#endif 411#ifdef CONFIG_CPU_ICACHE_DISABLE 412 bic r0, r0, #CR_I 413#endif 414#ifdef CONFIG_ARM_LPAE 415 mov r5, #0 416 mcrr p15, 0, r4, r5, c2 @ load TTBR0 417#else 418 mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ 419 domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ 420 domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ 421 domain_val(DOMAIN_IO, DOMAIN_CLIENT)) 422 mcr p15, 0, r5, c3, c0, 0 @ load domain access register 423 mcr p15, 0, r4, c2, c0, 0 @ load page table pointer 424#endif 425 b __turn_mmu_on 426ENDPROC(__enable_mmu) 427 428/* 429 * Enable the MMU. This completely changes the structure of the visible 430 * memory space. You will not be able to trace execution through this. 431 * If you have an enquiry about this, *please* check the linux-arm-kernel 432 * mailing list archives BEFORE sending another post to the list. 433 * 434 * r0 = cp#15 control register 435 * r1 = machine ID 436 * r2 = atags or dtb pointer 437 * r9 = processor ID 438 * r13 = *virtual* address to jump to upon completion 439 * 440 * other registers depend on the function called upon completion 441 */ 442 .align 5 443 .pushsection .idmap.text, "ax" 444ENTRY(__turn_mmu_on) 445 mov r0, r0 446 instr_sync 447 mcr p15, 0, r0, c1, c0, 0 @ write control reg 448 mrc p15, 0, r3, c0, c0, 0 @ read id reg 449 instr_sync 450 mov r3, r3 451 mov r3, r13 452 mov pc, r3 453__turn_mmu_on_end: 454ENDPROC(__turn_mmu_on) 455 .popsection 456 457 458#ifdef CONFIG_SMP_ON_UP 459 __INIT 460__fixup_smp: 461 and r3, r9, #0x000f0000 @ architecture version 462 teq r3, #0x000f0000 @ CPU ID supported? 463 bne __fixup_smp_on_up @ no, assume UP 464 465 bic r3, r9, #0x00ff0000 466 bic r3, r3, #0x0000000f @ mask 0xff00fff0 467 mov r4, #0x41000000 468 orr r4, r4, #0x0000b000 469 orr r4, r4, #0x00000020 @ val 0x4100b020 470 teq r3, r4 @ ARM 11MPCore? 471 moveq pc, lr @ yes, assume SMP 472 473 mrc p15, 0, r0, c0, c0, 5 @ read MPIDR 474 and r0, r0, #0xc0000000 @ multiprocessing extensions and 475 teq r0, #0x80000000 @ not part of a uniprocessor system? 476 moveq pc, lr @ yes, assume SMP 477 478__fixup_smp_on_up: 479 adr r0, 1f 480 ldmia r0, {r3 - r5} 481 sub r3, r0, r3 482 add r4, r4, r3 483 add r5, r5, r3 484 b __do_fixup_smp_on_up 485ENDPROC(__fixup_smp) 486 487 .align 4881: .word . 489 .word __smpalt_begin 490 .word __smpalt_end 491 492 .pushsection .data 493 .globl smp_on_up 494smp_on_up: 495 ALT_SMP(.long 1) 496 ALT_UP(.long 0) 497 .popsection 498#endif 499 500 .text 501__do_fixup_smp_on_up: 502 cmp r4, r5 503 movhs pc, lr 504 ldmia r4!, {r0, r6} 505 ARM( str r6, [r0, r3] ) 506 THUMB( add r0, r0, r3 ) 507#ifdef __ARMEB__ 508 THUMB( mov r6, r6, ror #16 ) @ Convert word order for big-endian. 509#endif 510 THUMB( strh r6, [r0], #2 ) @ For Thumb-2, store as two halfwords 511 THUMB( mov r6, r6, lsr #16 ) @ to be robust against misaligned r3. 512 THUMB( strh r6, [r0] ) 513 b __do_fixup_smp_on_up 514ENDPROC(__do_fixup_smp_on_up) 515 516ENTRY(fixup_smp) 517 stmfd sp!, {r4 - r6, lr} 518 mov r4, r0 519 add r5, r0, r1 520 mov r3, #0 521 bl __do_fixup_smp_on_up 522 ldmfd sp!, {r4 - r6, pc} 523ENDPROC(fixup_smp) 524 525#ifdef CONFIG_ARM_PATCH_PHYS_VIRT 526 527/* __fixup_pv_table - patch the stub instructions with the delta between 528 * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be 16MiB aligned and 529 * can be expressed by an immediate shifter operand. The stub instruction 530 * has a form of '(add|sub) rd, rn, #imm'. 531 */ 532 __HEAD 533__fixup_pv_table: 534 adr r0, 1f 535 ldmia r0, {r3-r5, r7} 536 sub r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET 537 add r4, r4, r3 @ adjust table start address 538 add r5, r5, r3 @ adjust table end address 539 add r7, r7, r3 @ adjust __pv_phys_offset address 540 str r8, [r7] @ save computed PHYS_OFFSET to __pv_phys_offset 541 mov r6, r3, lsr #24 @ constant for add/sub instructions 542 teq r3, r6, lsl #24 @ must be 16MiB aligned 543THUMB( it ne @ cross section branch ) 544 bne __error 545 str r6, [r7, #4] @ save to __pv_offset 546 b __fixup_a_pv_table 547ENDPROC(__fixup_pv_table) 548 549 .align 5501: .long . 551 .long __pv_table_begin 552 .long __pv_table_end 5532: .long __pv_phys_offset 554 555 .text 556__fixup_a_pv_table: 557#ifdef CONFIG_THUMB2_KERNEL 558 lsls r6, #24 559 beq 2f 560 clz r7, r6 561 lsr r6, #24 562 lsl r6, r7 563 bic r6, #0x0080 564 lsrs r7, #1 565 orrcs r6, #0x0080 566 orr r6, r6, r7, lsl #12 567 orr r6, #0x4000 568 b 2f 5691: add r7, r3 570 ldrh ip, [r7, #2] 571 and ip, 0x8f00 572 orr ip, r6 @ mask in offset bits 31-24 573 strh ip, [r7, #2] 5742: cmp r4, r5 575 ldrcc r7, [r4], #4 @ use branch for delay slot 576 bcc 1b 577 bx lr 578#else 579 b 2f 5801: ldr ip, [r7, r3] 581 bic ip, ip, #0x000000ff 582 orr ip, ip, r6 @ mask in offset bits 31-24 583 str ip, [r7, r3] 5842: cmp r4, r5 585 ldrcc r7, [r4], #4 @ use branch for delay slot 586 bcc 1b 587 mov pc, lr 588#endif 589ENDPROC(__fixup_a_pv_table) 590 591ENTRY(fixup_pv_table) 592 stmfd sp!, {r4 - r7, lr} 593 ldr r2, 2f @ get address of __pv_phys_offset 594 mov r3, #0 @ no offset 595 mov r4, r0 @ r0 = table start 596 add r5, r0, r1 @ r1 = table size 597 ldr r6, [r2, #4] @ get __pv_offset 598 bl __fixup_a_pv_table 599 ldmfd sp!, {r4 - r7, pc} 600ENDPROC(fixup_pv_table) 601 602 .align 6032: .long __pv_phys_offset 604 605 .data 606 .globl __pv_phys_offset 607 .type __pv_phys_offset, %object 608__pv_phys_offset: 609 .long 0 610 .size __pv_phys_offset, . - __pv_phys_offset 611__pv_offset: 612 .long 0 613#endif 614 615#include "head-common.S" 616