1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * linux/arch/arm/kernel/head.S 4 * 5 * Copyright (C) 1994-2002 Russell King 6 * Copyright (c) 2003 ARM Limited 7 * All Rights Reserved 8 * 9 * Kernel startup code for all 32-bit CPUs 10 */ 11#include <linux/linkage.h> 12#include <linux/init.h> 13#include <linux/pgtable.h> 14 15#include <asm/assembler.h> 16#include <asm/cp15.h> 17#include <asm/domain.h> 18#include <asm/ptrace.h> 19#include <asm/asm-offsets.h> 20#include <asm/page.h> 21#include <asm/thread_info.h> 22 23#if defined(CONFIG_DEBUG_LL) && !defined(CONFIG_DEBUG_SEMIHOSTING) 24#include CONFIG_DEBUG_LL_INCLUDE 25#endif 26/* 27 * swapper_pg_dir is the virtual address of the initial page table. 28 * We place the page tables 16K below KERNEL_RAM_VADDR. Therefore, we must 29 * make sure that KERNEL_RAM_VADDR is correctly set. Currently, we expect 30 * the least significant 16 bits to be 0x8000, but we could probably 31 * relax this restriction to KERNEL_RAM_VADDR >= PAGE_OFFSET + 0x4000. 32 */ 33#define KERNEL_RAM_VADDR (KERNEL_OFFSET + TEXT_OFFSET) 34#if (KERNEL_RAM_VADDR & 0xffff) != 0x8000 35#error KERNEL_RAM_VADDR must start at 0xXXXX8000 36#endif 37 38#ifdef CONFIG_ARM_LPAE 39 /* LPAE requires an additional page for the PGD */ 40#define PG_DIR_SIZE 0x5000 41#define PMD_ENTRY_ORDER 3 /* PMD entry size is 2^PMD_ENTRY_ORDER */ 42#else 43#define PG_DIR_SIZE 0x4000 44#define PMD_ENTRY_ORDER 2 45#endif 46 47 .globl swapper_pg_dir 48 .equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE 49 50 /* 51 * This needs to be assigned at runtime when the linker symbols are 52 * resolved. These are unsigned 64bit really, but in this assembly code 53 * We store them as 32bit. 54 */ 55 .pushsection .data 56 .align 2 57 .globl kernel_sec_start 58 .globl kernel_sec_end 59kernel_sec_start: 60 .long 0 61 .long 0 62kernel_sec_end: 63 .long 0 64 .long 0 65 .popsection 66 67 .macro pgtbl, rd, phys 68 add \rd, \phys, #TEXT_OFFSET 69 sub \rd, \rd, #PG_DIR_SIZE 70 .endm 71 72/* 73 * Kernel startup entry point. 74 * --------------------------- 75 * 76 * This is normally called from the decompressor code. The requirements 77 * are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0, 78 * r1 = machine nr, r2 = atags or dtb pointer. 79 * 80 * This code is mostly position independent, so if you link the kernel at 81 * 0xc0008000, you call this at __pa(0xc0008000). 82 * 83 * See linux/arch/arm/tools/mach-types for the complete list of machine 84 * numbers for r1. 85 * 86 * We're trying to keep crap to a minimum; DO NOT add any machine specific 87 * crap here - that's what the boot loader (or in extreme, well justified 88 * circumstances, zImage) is for. 89 */ 90 .arm 91 92 __HEAD 93ENTRY(stext) 94 ARM_BE8(setend be ) @ ensure we are in BE8 mode 95 96 THUMB( badr r9, 1f ) @ Kernel is always entered in ARM. 97 THUMB( bx r9 ) @ If this is a Thumb-2 kernel, 98 THUMB( .thumb ) @ switch to Thumb now. 99 THUMB(1: ) 100 101#ifdef CONFIG_ARM_VIRT_EXT 102 bl __hyp_stub_install 103#endif 104 @ ensure svc mode and all interrupts masked 105 safe_svcmode_maskall r9 106 107 mrc p15, 0, r9, c0, c0 @ get processor id 108 bl __lookup_processor_type @ r5=procinfo r9=cpuid 109 movs r10, r5 @ invalid processor (r5=0)? 110 THUMB( it eq ) @ force fixup-able long branch encoding 111 beq __error_p @ yes, error 'p' 112 113#ifdef CONFIG_ARM_LPAE 114 mrc p15, 0, r3, c0, c1, 4 @ read ID_MMFR0 115 and r3, r3, #0xf @ extract VMSA support 116 cmp r3, #5 @ long-descriptor translation table format? 117 THUMB( it lo ) @ force fixup-able long branch encoding 118 blo __error_lpae @ only classic page table format 119#endif 120 121#ifndef CONFIG_XIP_KERNEL 122 adr_l r8, _text @ __pa(_text) 123 sub r8, r8, #TEXT_OFFSET @ PHYS_OFFSET 124#else 125 ldr r8, =PLAT_PHYS_OFFSET @ always constant in this case 126#endif 127 128 /* 129 * r1 = machine no, r2 = atags or dtb, 130 * r8 = phys_offset, r9 = cpuid, r10 = procinfo 131 */ 132 bl __vet_atags 133#ifdef CONFIG_SMP_ON_UP 134 bl __fixup_smp 135#endif 136#ifdef CONFIG_ARM_PATCH_PHYS_VIRT 137 bl __fixup_pv_table 138#endif 139 bl __create_page_tables 140 141 /* 142 * The following calls CPU specific code in a position independent 143 * manner. See arch/arm/mm/proc-*.S for details. r10 = base of 144 * xxx_proc_info structure selected by __lookup_processor_type 145 * above. 146 * 147 * The processor init function will be called with: 148 * r1 - machine type 149 * r2 - boot data (atags/dt) pointer 150 * r4 - translation table base (low word) 151 * r5 - translation table base (high word, if LPAE) 152 * r8 - translation table base 1 (pfn if LPAE) 153 * r9 - cpuid 154 * r13 - virtual address for __enable_mmu -> __turn_mmu_on 155 * 156 * On return, the CPU will be ready for the MMU to be turned on, 157 * r0 will hold the CPU control register value, r1, r2, r4, and 158 * r9 will be preserved. r5 will also be preserved if LPAE. 159 */ 160 ldr r13, =__mmap_switched @ address to jump to after 161 @ mmu has been enabled 162 badr lr, 1f @ return (PIC) address 163#ifdef CONFIG_ARM_LPAE 164 mov r5, #0 @ high TTBR0 165 mov r8, r4, lsr #12 @ TTBR1 is swapper_pg_dir pfn 166#else 167 mov r8, r4 @ set TTBR1 to swapper_pg_dir 168#endif 169 ldr r12, [r10, #PROCINFO_INITFUNC] 170 add r12, r12, r10 171 ret r12 1721: b __enable_mmu 173ENDPROC(stext) 174 .ltorg 175 176/* 177 * Setup the initial page tables. We only setup the barest 178 * amount which are required to get the kernel running, which 179 * generally means mapping in the kernel code. 180 * 181 * r8 = phys_offset, r9 = cpuid, r10 = procinfo 182 * 183 * Returns: 184 * r0, r3, r5-r7 corrupted 185 * r4 = physical page table address 186 */ 187__create_page_tables: 188 pgtbl r4, r8 @ page table address 189 190 /* 191 * Clear the swapper page table 192 */ 193 mov r0, r4 194 mov r3, #0 195 add r6, r0, #PG_DIR_SIZE 1961: str r3, [r0], #4 197 str r3, [r0], #4 198 str r3, [r0], #4 199 str r3, [r0], #4 200 teq r0, r6 201 bne 1b 202 203#ifdef CONFIG_ARM_LPAE 204 /* 205 * Build the PGD table (first level) to point to the PMD table. A PGD 206 * entry is 64-bit wide. 207 */ 208 mov r0, r4 209 add r3, r4, #0x1000 @ first PMD table address 210 orr r3, r3, #3 @ PGD block type 211 mov r6, #4 @ PTRS_PER_PGD 212 mov r7, #1 << (55 - 32) @ L_PGD_SWAPPER 2131: 214#ifdef CONFIG_CPU_ENDIAN_BE8 215 str r7, [r0], #4 @ set top PGD entry bits 216 str r3, [r0], #4 @ set bottom PGD entry bits 217#else 218 str r3, [r0], #4 @ set bottom PGD entry bits 219 str r7, [r0], #4 @ set top PGD entry bits 220#endif 221 add r3, r3, #0x1000 @ next PMD table 222 subs r6, r6, #1 223 bne 1b 224 225 add r4, r4, #0x1000 @ point to the PMD tables 226#ifdef CONFIG_CPU_ENDIAN_BE8 227 add r4, r4, #4 @ we only write the bottom word 228#endif 229#endif 230 231 ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags 232 233 /* 234 * Create identity mapping to cater for __enable_mmu. 235 * This identity mapping will be removed by paging_init(). 236 */ 237 adr_l r5, __turn_mmu_on @ _pa(__turn_mmu_on) 238 adr_l r6, __turn_mmu_on_end @ _pa(__turn_mmu_on_end) 239 mov r5, r5, lsr #SECTION_SHIFT 240 mov r6, r6, lsr #SECTION_SHIFT 241 2421: orr r3, r7, r5, lsl #SECTION_SHIFT @ flags + kernel base 243 str r3, [r4, r5, lsl #PMD_ENTRY_ORDER] @ identity mapping 244 cmp r5, r6 245 addlo r5, r5, #1 @ next section 246 blo 1b 247 248 /* 249 * The main matter: map in the kernel using section mappings, and 250 * set two variables to indicate the physical start and end of the 251 * kernel. 252 */ 253 add r0, r4, #KERNEL_OFFSET >> (SECTION_SHIFT - PMD_ENTRY_ORDER) 254 ldr r6, =(_end - 1) 255 256 /* For XIP, kernel_sec_start/kernel_sec_end are currently in RO memory */ 257#ifndef CONFIG_XIP_KERNEL 258 adr_l r5, kernel_sec_start @ _pa(kernel_sec_start) 259#if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32 260 str r8, [r5, #4] @ Save physical start of kernel (BE) 261#else 262 str r8, [r5] @ Save physical start of kernel (LE) 263#endif 264#endif 265 orr r3, r8, r7 @ Add the MMU flags 266 add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ENTRY_ORDER) 2671: str r3, [r0], #1 << PMD_ENTRY_ORDER 268 add r3, r3, #1 << SECTION_SHIFT 269 cmp r0, r6 270 bls 1b 271#ifndef CONFIG_XIP_KERNEL 272 eor r3, r3, r7 @ Remove the MMU flags 273 adr_l r5, kernel_sec_end @ _pa(kernel_sec_end) 274#if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32 275 str r3, [r5, #4] @ Save physical end of kernel (BE) 276#else 277 str r3, [r5] @ Save physical end of kernel (LE) 278#endif 279#else 280 /* 281 * Map the kernel image separately as it is not located in RAM. 282 */ 283#define XIP_START XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) 284 mov r3, pc 285 mov r3, r3, lsr #SECTION_SHIFT 286 orr r3, r7, r3, lsl #SECTION_SHIFT 287 add r0, r4, #(XIP_START & 0xff000000) >> (SECTION_SHIFT - PMD_ENTRY_ORDER) 288 str r3, [r0, #((XIP_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ENTRY_ORDER]! 289 ldr r6, =(_edata_loc - 1) 290 add r0, r0, #1 << PMD_ENTRY_ORDER 291 add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ENTRY_ORDER) 2921: cmp r0, r6 293 add r3, r3, #1 << SECTION_SHIFT 294 strls r3, [r0], #1 << PMD_ENTRY_ORDER 295 bls 1b 296#endif 297 298 /* 299 * Then map boot params address in r2 if specified. 300 * We map 2 sections in case the ATAGs/DTB crosses a section boundary. 301 */ 302 mov r0, r2, lsr #SECTION_SHIFT 303 cmp r2, #0 304 ldrne r3, =FDT_FIXED_BASE >> (SECTION_SHIFT - PMD_ENTRY_ORDER) 305 addne r3, r3, r4 306 orrne r6, r7, r0, lsl #SECTION_SHIFT 307 strne r6, [r3], #1 << PMD_ENTRY_ORDER 308 addne r6, r6, #1 << SECTION_SHIFT 309 strne r6, [r3] 310 311#if defined(CONFIG_ARM_LPAE) && defined(CONFIG_CPU_ENDIAN_BE8) 312 sub r4, r4, #4 @ Fixup page table pointer 313 @ for 64-bit descriptors 314#endif 315 316#ifdef CONFIG_DEBUG_LL 317#if !defined(CONFIG_DEBUG_ICEDCC) && !defined(CONFIG_DEBUG_SEMIHOSTING) 318 /* 319 * Map in IO space for serial debugging. 320 * This allows debug messages to be output 321 * via a serial console before paging_init. 322 */ 323 addruart r7, r3, r0 324 325 mov r3, r3, lsr #SECTION_SHIFT 326 mov r3, r3, lsl #PMD_ENTRY_ORDER 327 328 add r0, r4, r3 329 mov r3, r7, lsr #SECTION_SHIFT 330 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags 331 orr r3, r7, r3, lsl #SECTION_SHIFT 332#ifdef CONFIG_ARM_LPAE 333 mov r7, #1 << (54 - 32) @ XN 334#ifdef CONFIG_CPU_ENDIAN_BE8 335 str r7, [r0], #4 336 str r3, [r0], #4 337#else 338 str r3, [r0], #4 339 str r7, [r0], #4 340#endif 341#else 342 orr r3, r3, #PMD_SECT_XN 343 str r3, [r0], #4 344#endif 345 346#else /* CONFIG_DEBUG_ICEDCC || CONFIG_DEBUG_SEMIHOSTING */ 347 /* we don't need any serial debugging mappings */ 348 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags 349#endif 350 351#if defined(CONFIG_ARCH_NETWINDER) 352 /* 353 * If we're using the NetWinder or CATS, we also need to map 354 * in the 16550-type serial port for the debug messages 355 */ 356 add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER) 357 orr r3, r7, #0x7c000000 358 str r3, [r0] 359#endif 360#ifdef CONFIG_ARCH_RPC 361 /* 362 * Map in screen at 0x02000000 & SCREEN2_BASE 363 * Similar reasons here - for debug. This is 364 * only for Acorn RiscPC architectures. 365 */ 366 add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER) 367 orr r3, r7, #0x02000000 368 str r3, [r0] 369 add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER) 370 str r3, [r0] 371#endif 372#endif 373#ifdef CONFIG_ARM_LPAE 374 sub r4, r4, #0x1000 @ point to the PGD table 375#endif 376 ret lr 377ENDPROC(__create_page_tables) 378 .ltorg 379 380#if defined(CONFIG_SMP) 381 .text 382 .arm 383ENTRY(secondary_startup_arm) 384 THUMB( badr r9, 1f ) @ Kernel is entered in ARM. 385 THUMB( bx r9 ) @ If this is a Thumb-2 kernel, 386 THUMB( .thumb ) @ switch to Thumb now. 387 THUMB(1: ) 388ENTRY(secondary_startup) 389 /* 390 * Common entry point for secondary CPUs. 391 * 392 * Ensure that we're in SVC mode, and IRQs are disabled. Lookup 393 * the processor type - there is no need to check the machine type 394 * as it has already been validated by the primary processor. 395 */ 396 397 ARM_BE8(setend be) @ ensure we are in BE8 mode 398 399#ifdef CONFIG_ARM_VIRT_EXT 400 bl __hyp_stub_install_secondary 401#endif 402 safe_svcmode_maskall r9 403 404 mrc p15, 0, r9, c0, c0 @ get processor id 405 bl __lookup_processor_type 406 movs r10, r5 @ invalid processor? 407 moveq r0, #'p' @ yes, error 'p' 408 THUMB( it eq ) @ force fixup-able long branch encoding 409 beq __error_p 410 411 /* 412 * Use the page tables supplied from __cpu_up. 413 */ 414#ifdef CONFIG_XIP_KERNEL 415 ldr r3, =(secondary_data + PLAT_PHYS_OFFSET - PAGE_OFFSET) 416#else 417 adr_l r3, secondary_data 418#endif 419 mov_l r12, __secondary_switched 420 ldrd r4, r5, [r3, #0] @ get secondary_data.pgdir 421ARM_BE8(eor r4, r4, r5) @ Swap r5 and r4 in BE: 422ARM_BE8(eor r5, r4, r5) @ it can be done in 3 steps 423ARM_BE8(eor r4, r4, r5) @ without using a temp reg. 424 ldr r8, [r3, #8] @ get secondary_data.swapper_pg_dir 425 badr lr, __enable_mmu @ return address 426 mov r13, r12 @ __secondary_switched address 427 ldr r12, [r10, #PROCINFO_INITFUNC] 428 add r12, r12, r10 @ initialise processor 429 @ (return control reg) 430 ret r12 431ENDPROC(secondary_startup) 432ENDPROC(secondary_startup_arm) 433 434ENTRY(__secondary_switched) 435#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE) 436 @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir 437 @ as the ID map does not cover the vmalloc region. 438 mrc p15, 0, ip, c2, c0, 1 @ read TTBR1 439 mcr p15, 0, ip, c2, c0, 0 @ set TTBR0 440 instr_sync 441#endif 442 adr_l r7, secondary_data + 12 @ get secondary_data.stack 443 ldr sp, [r7] 444 ldr r0, [r7, #4] @ get secondary_data.task 445 mov fp, #0 446 b secondary_start_kernel 447ENDPROC(__secondary_switched) 448 449#endif /* defined(CONFIG_SMP) */ 450 451 452 453/* 454 * Setup common bits before finally enabling the MMU. Essentially 455 * this is just loading the page table pointer and domain access 456 * registers. All these registers need to be preserved by the 457 * processor setup function (or set in the case of r0) 458 * 459 * r0 = cp#15 control register 460 * r1 = machine ID 461 * r2 = atags or dtb pointer 462 * r4 = TTBR pointer (low word) 463 * r5 = TTBR pointer (high word if LPAE) 464 * r9 = processor ID 465 * r13 = *virtual* address to jump to upon completion 466 */ 467__enable_mmu: 468#if defined(CONFIG_ALIGNMENT_TRAP) && __LINUX_ARM_ARCH__ < 6 469 orr r0, r0, #CR_A 470#else 471 bic r0, r0, #CR_A 472#endif 473#ifdef CONFIG_CPU_DCACHE_DISABLE 474 bic r0, r0, #CR_C 475#endif 476#ifdef CONFIG_CPU_BPREDICT_DISABLE 477 bic r0, r0, #CR_Z 478#endif 479#ifdef CONFIG_CPU_ICACHE_DISABLE 480 bic r0, r0, #CR_I 481#endif 482#ifdef CONFIG_ARM_LPAE 483 mcrr p15, 0, r4, r5, c2 @ load TTBR0 484#else 485 mov r5, #DACR_INIT 486 mcr p15, 0, r5, c3, c0, 0 @ load domain access register 487 mcr p15, 0, r4, c2, c0, 0 @ load page table pointer 488#endif 489 b __turn_mmu_on 490ENDPROC(__enable_mmu) 491 492/* 493 * Enable the MMU. This completely changes the structure of the visible 494 * memory space. You will not be able to trace execution through this. 495 * If you have an enquiry about this, *please* check the linux-arm-kernel 496 * mailing list archives BEFORE sending another post to the list. 497 * 498 * r0 = cp#15 control register 499 * r1 = machine ID 500 * r2 = atags or dtb pointer 501 * r9 = processor ID 502 * r13 = *virtual* address to jump to upon completion 503 * 504 * other registers depend on the function called upon completion 505 */ 506 .align 5 507 .pushsection .idmap.text, "ax" 508ENTRY(__turn_mmu_on) 509 mov r0, r0 510 instr_sync 511 mcr p15, 0, r0, c1, c0, 0 @ write control reg 512 mrc p15, 0, r3, c0, c0, 0 @ read id reg 513 instr_sync 514 mov r3, r3 515 mov r3, r13 516 ret r3 517__turn_mmu_on_end: 518ENDPROC(__turn_mmu_on) 519 .popsection 520 521 522#ifdef CONFIG_SMP_ON_UP 523 __HEAD 524__fixup_smp: 525 and r3, r9, #0x000f0000 @ architecture version 526 teq r3, #0x000f0000 @ CPU ID supported? 527 bne __fixup_smp_on_up @ no, assume UP 528 529 bic r3, r9, #0x00ff0000 530 bic r3, r3, #0x0000000f @ mask 0xff00fff0 531 mov r4, #0x41000000 532 orr r4, r4, #0x0000b000 533 orr r4, r4, #0x00000020 @ val 0x4100b020 534 teq r3, r4 @ ARM 11MPCore? 535 reteq lr @ yes, assume SMP 536 537 mrc p15, 0, r0, c0, c0, 5 @ read MPIDR 538 and r0, r0, #0xc0000000 @ multiprocessing extensions and 539 teq r0, #0x80000000 @ not part of a uniprocessor system? 540 bne __fixup_smp_on_up @ no, assume UP 541 542 @ Core indicates it is SMP. Check for Aegis SOC where a single 543 @ Cortex-A9 CPU is present but SMP operations fault. 544 mov r4, #0x41000000 545 orr r4, r4, #0x0000c000 546 orr r4, r4, #0x00000090 547 teq r3, r4 @ Check for ARM Cortex-A9 548 retne lr @ Not ARM Cortex-A9, 549 550 @ If a future SoC *does* use 0x0 as the PERIPH_BASE, then the 551 @ below address check will need to be #ifdef'd or equivalent 552 @ for the Aegis platform. 553 mrc p15, 4, r0, c15, c0 @ get SCU base address 554 teq r0, #0x0 @ '0' on actual UP A9 hardware 555 beq __fixup_smp_on_up @ So its an A9 UP 556 ldr r0, [r0, #4] @ read SCU Config 557ARM_BE8(rev r0, r0) @ byteswap if big endian 558 and r0, r0, #0x3 @ number of CPUs 559 teq r0, #0x0 @ is 1? 560 retne lr 561 562__fixup_smp_on_up: 563 adr_l r4, __smpalt_begin 564 adr_l r5, __smpalt_end 565 b __do_fixup_smp_on_up 566ENDPROC(__fixup_smp) 567 568 .pushsection .data 569 .align 2 570 .globl smp_on_up 571smp_on_up: 572 ALT_SMP(.long 1) 573 ALT_UP(.long 0) 574 .popsection 575#endif 576 577 .text 578__do_fixup_smp_on_up: 579 cmp r4, r5 580 reths lr 581 ldmia r4, {r0, r6} 582 ARM( str r6, [r0, r4] ) 583 THUMB( add r0, r0, r4 ) 584 add r4, r4, #8 585#ifdef __ARMEB__ 586 THUMB( mov r6, r6, ror #16 ) @ Convert word order for big-endian. 587#endif 588 THUMB( strh r6, [r0], #2 ) @ For Thumb-2, store as two halfwords 589 THUMB( mov r6, r6, lsr #16 ) @ to be robust against misaligned r0. 590 THUMB( strh r6, [r0] ) 591 b __do_fixup_smp_on_up 592ENDPROC(__do_fixup_smp_on_up) 593 594ENTRY(fixup_smp) 595 stmfd sp!, {r4 - r6, lr} 596 mov r4, r0 597 add r5, r0, r1 598 bl __do_fixup_smp_on_up 599 ldmfd sp!, {r4 - r6, pc} 600ENDPROC(fixup_smp) 601 602#include "head-common.S" 603