1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * linux/arch/arm/kernel/entry-armv.S 4 * 5 * Copyright (C) 1996,1997,1998 Russell King. 6 * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk) 7 * nommu support by Hyok S. Choi (hyok.choi@samsung.com) 8 * 9 * Low-level vector interface routines 10 * 11 * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction 12 * that causes it to save wrong values... Be aware! 13 */ 14 15#include <linux/init.h> 16 17#include <asm/assembler.h> 18#include <asm/page.h> 19#include <asm/glue-df.h> 20#include <asm/glue-pf.h> 21#include <asm/vfpmacros.h> 22#include <asm/thread_notify.h> 23#include <asm/unwind.h> 24#include <asm/unistd.h> 25#include <asm/tls.h> 26#include <asm/system_info.h> 27#include <asm/uaccess-asm.h> 28#include <asm/kasan_def.h> 29 30#include "entry-header.S" 31#include <asm/probes.h> 32 33#ifdef CONFIG_HAVE_LD_DEAD_CODE_DATA_ELIMINATION 34#define RELOC_TEXT_NONE .reloc .text, R_ARM_NONE, . 35#else 36#define RELOC_TEXT_NONE 37#endif 38 39/* 40 * Interrupt handling. 41 */ 42 .macro irq_handler, from_user:req 43 mov r1, sp 44 ldr_this_cpu r2, irq_stack_ptr, r2, r3 45 .if \from_user == 0 46 @ 47 @ If we took the interrupt while running in the kernel, we may already 48 @ be using the IRQ stack, so revert to the original value in that case. 49 @ 50 subs r3, r2, r1 @ SP above bottom of IRQ stack? 51 rsbscs r3, r3, #THREAD_SIZE @ ... and below the top? 52#ifdef CONFIG_VMAP_STACK 53 ldr_va r3, high_memory, cc @ End of the linear region 54 cmpcc r3, r1 @ Stack pointer was below it? 55#endif 56 bcc 0f @ If not, switch to the IRQ stack 57 mov r0, r1 58 bl generic_handle_arch_irq 59 b 1f 600: 61 .endif 62 63 mov_l r0, generic_handle_arch_irq 64 bl call_with_stack 651: 66 .endm 67 68 .macro pabt_helper 69 @ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5 70#ifdef MULTI_PABORT 71 ldr_va ip, processor, offset=PROCESSOR_PABT_FUNC 72 bl_r ip 73#else 74 bl CPU_PABORT_HANDLER 75#endif 76 .endm 77 78 .macro dabt_helper 79 80 @ 81 @ Call the processor-specific abort handler: 82 @ 83 @ r2 - pt_regs 84 @ r4 - aborted context pc 85 @ r5 - aborted context psr 86 @ 87 @ The abort handler must return the aborted address in r0, and 88 @ the fault status register in r1. r9 must be preserved. 89 @ 90#ifdef MULTI_DABORT 91 ldr_va ip, processor, offset=PROCESSOR_DABT_FUNC 92 bl_r ip 93#else 94 bl CPU_DABORT_HANDLER 95#endif 96 .endm 97 98 .section .entry.text,"ax",%progbits 99 100/* 101 * Invalid mode handlers 102 */ 103 .macro inv_entry, reason 104 sub sp, sp, #PT_REGS_SIZE 105 ARM( stmib sp, {r1 - lr} ) 106 THUMB( stmia sp, {r0 - r12} ) 107 THUMB( str sp, [sp, #S_SP] ) 108 THUMB( str lr, [sp, #S_LR] ) 109 mov r1, #\reason 110 .endm 111 112__pabt_invalid: 113 inv_entry BAD_PREFETCH 114 b common_invalid 115ENDPROC(__pabt_invalid) 116 117__dabt_invalid: 118 inv_entry BAD_DATA 119 b common_invalid 120ENDPROC(__dabt_invalid) 121 122__irq_invalid: 123 inv_entry BAD_IRQ 124 b common_invalid 125ENDPROC(__irq_invalid) 126 127__und_invalid: 128 inv_entry BAD_UNDEFINSTR 129 130 @ 131 @ XXX fall through to common_invalid 132 @ 133 134@ 135@ common_invalid - generic code for failed exception (re-entrant version of handlers) 136@ 137common_invalid: 138 zero_fp 139 140 ldmia r0, {r4 - r6} 141 add r0, sp, #S_PC @ here for interlock avoidance 142 mov r7, #-1 @ "" "" "" "" 143 str r4, [sp] @ save preserved r0 144 stmia r0, {r5 - r7} @ lr_<exception>, 145 @ cpsr_<exception>, "old_r0" 146 147 mov r0, sp 148 b bad_mode 149ENDPROC(__und_invalid) 150 151/* 152 * SVC mode handlers 153 */ 154 155#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) 156#define SPFIX(code...) code 157#else 158#define SPFIX(code...) 159#endif 160 161 .macro svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1 162 UNWIND(.fnstart ) 163 sub sp, sp, #(SVC_REGS_SIZE + \stack_hole) 164 THUMB( add sp, r1 ) @ get SP in a GPR without 165 THUMB( sub r1, sp, r1 ) @ using a temp register 166 167 .if \overflow_check 168 UNWIND(.save {r0 - pc} ) 169 do_overflow_check (SVC_REGS_SIZE + \stack_hole) 170 .endif 171 172#ifdef CONFIG_THUMB2_KERNEL 173 tst r1, #4 @ test stack pointer alignment 174 sub r1, sp, r1 @ restore original R1 175 sub sp, r1 @ restore original SP 176#else 177 SPFIX( tst sp, #4 ) 178#endif 179 SPFIX( subne sp, sp, #4 ) 180 181 ARM( stmib sp, {r1 - r12} ) 182 THUMB( stmia sp, {r0 - r12} ) @ No STMIB in Thumb-2 183 184 ldmia r0, {r3 - r5} 185 add r7, sp, #S_SP @ here for interlock avoidance 186 mov r6, #-1 @ "" "" "" "" 187 add r2, sp, #(SVC_REGS_SIZE + \stack_hole) 188 SPFIX( addne r2, r2, #4 ) 189 str r3, [sp] @ save the "real" r0 copied 190 @ from the exception stack 191 192 mov r3, lr 193 194 @ 195 @ We are now ready to fill in the remaining blanks on the stack: 196 @ 197 @ r2 - sp_svc 198 @ r3 - lr_svc 199 @ r4 - lr_<exception>, already fixed up for correct return/restart 200 @ r5 - spsr_<exception> 201 @ r6 - orig_r0 (see pt_regs definition in ptrace.h) 202 @ 203 stmia r7, {r2 - r6} 204 205 get_thread_info tsk 206 uaccess_entry tsk, r0, r1, r2, \uaccess 207 208 .if \trace 209#ifdef CONFIG_TRACE_IRQFLAGS 210 bl trace_hardirqs_off 211#endif 212 .endif 213 .endm 214 215 .align 5 216__dabt_svc: 217 svc_entry uaccess=0 218 mov r2, sp 219 dabt_helper 220 THUMB( ldr r5, [sp, #S_PSR] ) @ potentially updated CPSR 221 svc_exit r5 @ return from exception 222 UNWIND(.fnend ) 223ENDPROC(__dabt_svc) 224 225 .align 5 226__irq_svc: 227 svc_entry 228 irq_handler from_user=0 229 230#ifdef CONFIG_PREEMPTION 231 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count 232 ldr r0, [tsk, #TI_FLAGS] @ get flags 233 teq r8, #0 @ if preempt count != 0 234 movne r0, #0 @ force flags to 0 235 tst r0, #_TIF_NEED_RESCHED 236 blne svc_preempt 237#endif 238 239 svc_exit r5, irq = 1 @ return from exception 240 UNWIND(.fnend ) 241ENDPROC(__irq_svc) 242 243 .ltorg 244 245#ifdef CONFIG_PREEMPTION 246svc_preempt: 247 mov r8, lr 2481: bl preempt_schedule_irq @ irq en/disable is done inside 249 ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS 250 tst r0, #_TIF_NEED_RESCHED 251 reteq r8 @ go again 252 b 1b 253#endif 254 255__und_fault: 256 @ Correct the PC such that it is pointing at the instruction 257 @ which caused the fault. If the faulting instruction was ARM 258 @ the PC will be pointing at the next instruction, and have to 259 @ subtract 4. Otherwise, it is Thumb, and the PC will be 260 @ pointing at the second half of the Thumb instruction. We 261 @ have to subtract 2. 262 ldr r2, [r0, #S_PC] 263 sub r2, r2, r1 264 str r2, [r0, #S_PC] 265 b do_undefinstr 266ENDPROC(__und_fault) 267 268 .align 5 269__und_svc: 270#ifdef CONFIG_KPROBES 271 @ If a kprobe is about to simulate a "stmdb sp..." instruction, 272 @ it obviously needs free stack space which then will belong to 273 @ the saved context. 274 svc_entry MAX_STACK_SIZE 275#else 276 svc_entry 277#endif 278 279 mov r1, #4 @ PC correction to apply 280 THUMB( tst r5, #PSR_T_BIT ) @ exception taken in Thumb mode? 281 THUMB( movne r1, #2 ) @ if so, fix up PC correction 282 mov r0, sp @ struct pt_regs *regs 283 bl __und_fault 284 285__und_svc_finish: 286 get_thread_info tsk 287 ldr r5, [sp, #S_PSR] @ Get SVC cpsr 288 svc_exit r5 @ return from exception 289 UNWIND(.fnend ) 290ENDPROC(__und_svc) 291 292 .align 5 293__pabt_svc: 294 svc_entry 295 mov r2, sp @ regs 296 pabt_helper 297 svc_exit r5 @ return from exception 298 UNWIND(.fnend ) 299ENDPROC(__pabt_svc) 300 301 .align 5 302__fiq_svc: 303 svc_entry trace=0 304 mov r0, sp @ struct pt_regs *regs 305 bl handle_fiq_as_nmi 306 svc_exit_via_fiq 307 UNWIND(.fnend ) 308ENDPROC(__fiq_svc) 309 310/* 311 * Abort mode handlers 312 */ 313 314@ 315@ Taking a FIQ in abort mode is similar to taking a FIQ in SVC mode 316@ and reuses the same macros. However in abort mode we must also 317@ save/restore lr_abt and spsr_abt to make nested aborts safe. 318@ 319 .align 5 320__fiq_abt: 321 svc_entry trace=0 322 323 ARM( msr cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT ) 324 THUMB( mov r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT ) 325 THUMB( msr cpsr_c, r0 ) 326 mov r1, lr @ Save lr_abt 327 mrs r2, spsr @ Save spsr_abt, abort is now safe 328 ARM( msr cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT ) 329 THUMB( mov r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT ) 330 THUMB( msr cpsr_c, r0 ) 331 stmfd sp!, {r1 - r2} 332 333 add r0, sp, #8 @ struct pt_regs *regs 334 bl handle_fiq_as_nmi 335 336 ldmfd sp!, {r1 - r2} 337 ARM( msr cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT ) 338 THUMB( mov r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT ) 339 THUMB( msr cpsr_c, r0 ) 340 mov lr, r1 @ Restore lr_abt, abort is unsafe 341 msr spsr_cxsf, r2 @ Restore spsr_abt 342 ARM( msr cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT ) 343 THUMB( mov r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT ) 344 THUMB( msr cpsr_c, r0 ) 345 346 svc_exit_via_fiq 347 UNWIND(.fnend ) 348ENDPROC(__fiq_abt) 349 350/* 351 * User mode handlers 352 * 353 * EABI note: sp_svc is always 64-bit aligned here, so should PT_REGS_SIZE 354 */ 355 356#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (PT_REGS_SIZE & 7) 357#error "sizeof(struct pt_regs) must be a multiple of 8" 358#endif 359 360 .macro usr_entry, trace=1, uaccess=1 361 UNWIND(.fnstart ) 362 UNWIND(.cantunwind ) @ don't unwind the user space 363 sub sp, sp, #PT_REGS_SIZE 364 ARM( stmib sp, {r1 - r12} ) 365 THUMB( stmia sp, {r0 - r12} ) 366 367 ATRAP( mrc p15, 0, r7, c1, c0, 0) 368 ATRAP( ldr_va r8, cr_alignment) 369 370 ldmia r0, {r3 - r5} 371 add r0, sp, #S_PC @ here for interlock avoidance 372 mov r6, #-1 @ "" "" "" "" 373 374 str r3, [sp] @ save the "real" r0 copied 375 @ from the exception stack 376 377 @ 378 @ We are now ready to fill in the remaining blanks on the stack: 379 @ 380 @ r4 - lr_<exception>, already fixed up for correct return/restart 381 @ r5 - spsr_<exception> 382 @ r6 - orig_r0 (see pt_regs definition in ptrace.h) 383 @ 384 @ Also, separately save sp_usr and lr_usr 385 @ 386 stmia r0, {r4 - r6} 387 ARM( stmdb r0, {sp, lr}^ ) 388 THUMB( store_user_sp_lr r0, r1, S_SP - S_PC ) 389 390 .if \uaccess 391 uaccess_disable ip 392 .endif 393 394 @ Enable the alignment trap while in kernel mode 395 ATRAP( teq r8, r7) 396 ATRAP( mcrne p15, 0, r8, c1, c0, 0) 397 398 reload_current r7, r8 399 400 @ 401 @ Clear FP to mark the first stack frame 402 @ 403 zero_fp 404 405 .if \trace 406#ifdef CONFIG_TRACE_IRQFLAGS 407 bl trace_hardirqs_off 408#endif 409 ct_user_exit save = 0 410 .endif 411 .endm 412 413 .macro kuser_cmpxchg_check 414#if !defined(CONFIG_CPU_32v6K) && defined(CONFIG_KUSER_HELPERS) 415#ifndef CONFIG_MMU 416#warning "NPTL on non MMU needs fixing" 417#else 418 @ Make sure our user space atomic helper is restarted 419 @ if it was interrupted in a critical region. Here we 420 @ perform a quick test inline since it should be false 421 @ 99.9999% of the time. The rest is done out of line. 422 ldr r0, =TASK_SIZE 423 cmp r4, r0 424 blhs kuser_cmpxchg64_fixup 425#endif 426#endif 427 .endm 428 429 .align 5 430__dabt_usr: 431 usr_entry uaccess=0 432 kuser_cmpxchg_check 433 mov r2, sp 434 dabt_helper 435 b ret_from_exception 436 UNWIND(.fnend ) 437ENDPROC(__dabt_usr) 438 439 .align 5 440__irq_usr: 441 usr_entry 442 kuser_cmpxchg_check 443 irq_handler from_user=1 444 get_thread_info tsk 445 mov why, #0 446 b ret_to_user_from_irq 447 UNWIND(.fnend ) 448ENDPROC(__irq_usr) 449 450 .ltorg 451 452 .align 5 453__und_usr: 454 usr_entry uaccess=0 455 456 @ IRQs must be enabled before attempting to read the instruction from 457 @ user space since that could cause a page/translation fault if the 458 @ page table was modified by another CPU. 459 enable_irq 460 461 tst r5, #PSR_T_BIT @ Thumb mode? 462 mov r1, #2 @ set insn size to 2 for Thumb 463 bne 0f @ handle as Thumb undef exception 464#ifdef CONFIG_FPE_NWFPE 465 adr r9, ret_from_exception 466 bl call_fpe @ returns via R9 on success 467#endif 468 mov r1, #4 @ set insn size to 4 for ARM 4690: mov r0, sp 470 uaccess_disable ip 471 bl __und_fault 472 b ret_from_exception 473 UNWIND(.fnend) 474ENDPROC(__und_usr) 475 476 .align 5 477__pabt_usr: 478 usr_entry 479 mov r2, sp @ regs 480 pabt_helper 481 UNWIND(.fnend ) 482 /* fall through */ 483/* 484 * This is the return code to user mode for abort handlers 485 */ 486ENTRY(ret_from_exception) 487 UNWIND(.fnstart ) 488 UNWIND(.cantunwind ) 489 get_thread_info tsk 490 mov why, #0 491 b ret_to_user 492 UNWIND(.fnend ) 493ENDPROC(__pabt_usr) 494ENDPROC(ret_from_exception) 495 496 .align 5 497__fiq_usr: 498 usr_entry trace=0 499 kuser_cmpxchg_check 500 mov r0, sp @ struct pt_regs *regs 501 bl handle_fiq_as_nmi 502 get_thread_info tsk 503 restore_user_regs fast = 0, offset = 0 504 UNWIND(.fnend ) 505ENDPROC(__fiq_usr) 506 507/* 508 * Register switch for ARMv3 and ARMv4 processors 509 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info 510 * previous and next are guaranteed not to be the same. 511 */ 512ENTRY(__switch_to) 513 UNWIND(.fnstart ) 514 UNWIND(.cantunwind ) 515 add ip, r1, #TI_CPU_SAVE 516 ARM( stmia ip!, {r4 - sl, fp, sp, lr} ) @ Store most regs on stack 517 THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack 518 THUMB( str sp, [ip], #4 ) 519 THUMB( str lr, [ip], #4 ) 520 ldr r4, [r2, #TI_TP_VALUE] 521 ldr r5, [r2, #TI_TP_VALUE + 4] 522#ifdef CONFIG_CPU_USE_DOMAINS 523 mrc p15, 0, r6, c3, c0, 0 @ Get domain register 524 str r6, [r1, #TI_CPU_DOMAIN] @ Save old domain register 525 ldr r6, [r2, #TI_CPU_DOMAIN] 526#endif 527 switch_tls r1, r4, r5, r3, r7 528#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ 529 !defined(CONFIG_STACKPROTECTOR_PER_TASK) 530 ldr r8, =__stack_chk_guard 531 .if (TSK_STACK_CANARY > IMM12_MASK) 532 add r9, r2, #TSK_STACK_CANARY & ~IMM12_MASK 533 ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK] 534 .else 535 ldr r9, [r2, #TSK_STACK_CANARY & IMM12_MASK] 536 .endif 537#endif 538 mov r7, r2 @ Preserve 'next' 539#ifdef CONFIG_CPU_USE_DOMAINS 540 mcr p15, 0, r6, c3, c0, 0 @ Set domain register 541#endif 542 mov r5, r0 543 add r4, r2, #TI_CPU_SAVE 544 ldr r0, =thread_notify_head 545 mov r1, #THREAD_NOTIFY_SWITCH 546 bl atomic_notifier_call_chain 547#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ 548 !defined(CONFIG_STACKPROTECTOR_PER_TASK) 549 str r9, [r8] 550#endif 551 mov r0, r5 552#if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK) 553 set_current r7, r8 554 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously 555#else 556 mov r1, r7 557 ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously 558#ifdef CONFIG_VMAP_STACK 559 @ 560 @ For a non-lazy mm switch, check_vmalloc_seq() has ensured that 561 @ that the active mm's page tables have mappings for the prev 562 @ task's stack and the next task's stack. 563 @ 564 @ For a lazy mm switch the active mm's page tables have mappings 565 @ for the prev task's stack but might not have mappings for the 566 @ new task's stack. Do a dummy read from the new stack while 567 @ running from the old stack so that we can rely on 568 @ do_translation_fault() to populate missing PMD entries covering the 569 @ new task's stack in the old task's page tables. 570 @ 571 ldr r2, [ip] 572#ifdef CONFIG_KASAN_VMALLOC 573 @ Also dummy read from the KASAN shadow memory for the new stack if we 574 @ are using KASAN 575 mov_l r2, KASAN_SHADOW_OFFSET 576 add r2, r2, ip, lsr #KASAN_SHADOW_SCALE_SHIFT 577 ldrb r2, [r2] 578#endif 579#endif 580 581 @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what 582 @ effectuates the task switch, as that is what causes the observable 583 @ values of current and current_thread_info to change. When 584 @ CONFIG_THREAD_INFO_IN_TASK=y, setting current (and therefore 585 @ current_thread_info) is done explicitly, and the update of SP just 586 @ switches us to another stack, with few other side effects. In order 587 @ to prevent this distinction from causing any inconsistencies, let's 588 @ keep the 'set_current' call as close as we can to the update of SP. 589 set_current r1, r2 590 mov sp, ip 591 ret lr 592#endif 593 UNWIND(.fnend ) 594ENDPROC(__switch_to) 595 596#ifdef CONFIG_VMAP_STACK 597 .text 598 .align 2 599__bad_stack: 600 @ 601 @ We've just detected an overflow. We need to load the address of this 602 @ CPU's overflow stack into the stack pointer register. We have only one 603 @ scratch register so let's use a sequence of ADDs including one 604 @ involving the PC, and decorate them with PC-relative group 605 @ relocations. As these are ARM only, switch to ARM mode first. 606 @ 607 @ We enter here with IP clobbered and its value stashed on the mode 608 @ stack. 609 @ 610THUMB( bx pc ) 611THUMB( nop ) 612THUMB( .arm ) 613 ldr_this_cpu_armv6 ip, overflow_stack_ptr 614 615 str sp, [ip, #-4]! @ Preserve original SP value 616 mov sp, ip @ Switch to overflow stack 617 pop {ip} @ Original SP in IP 618 619#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) 620 mov ip, ip @ mov expected by unwinder 621 push {fp, ip, lr, pc} @ GCC flavor frame record 622#else 623 str ip, [sp, #-8]! @ store original SP 624 push {fpreg, lr} @ Clang flavor frame record 625#endif 626UNWIND( ldr ip, [r0, #4] ) @ load exception LR 627UNWIND( str ip, [sp, #12] ) @ store in the frame record 628 ldr ip, [r0, #12] @ reload IP 629 630 @ Store the original GPRs to the new stack. 631 svc_entry uaccess=0, overflow_check=0 632 633UNWIND( .save {sp, pc} ) 634UNWIND( .save {fpreg, lr} ) 635UNWIND( .setfp fpreg, sp ) 636 637 ldr fpreg, [sp, #S_SP] @ Add our frame record 638 @ to the linked list 639#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) 640 ldr r1, [fp, #4] @ reload SP at entry 641 add fp, fp, #12 642#else 643 ldr r1, [fpreg, #8] 644#endif 645 str r1, [sp, #S_SP] @ store in pt_regs 646 647 @ Stash the regs for handle_bad_stack 648 mov r0, sp 649 650 @ Time to die 651 bl handle_bad_stack 652 nop 653UNWIND( .fnend ) 654ENDPROC(__bad_stack) 655#endif 656 657 __INIT 658 659/* 660 * User helpers. 661 * 662 * Each segment is 32-byte aligned and will be moved to the top of the high 663 * vector page. New segments (if ever needed) must be added in front of 664 * existing ones. This mechanism should be used only for things that are 665 * really small and justified, and not be abused freely. 666 * 667 * See Documentation/arch/arm/kernel_user_helpers.rst for formal definitions. 668 */ 669 THUMB( .arm ) 670 671 .macro usr_ret, reg 672#ifdef CONFIG_ARM_THUMB 673 bx \reg 674#else 675 ret \reg 676#endif 677 .endm 678 679 .macro kuser_pad, sym, size 680 .if (. - \sym) & 3 681 .rept 4 - (. - \sym) & 3 682 .byte 0 683 .endr 684 .endif 685 .rept (\size - (. - \sym)) / 4 686 .word 0xe7fddef1 687 .endr 688 .endm 689 690#ifdef CONFIG_KUSER_HELPERS 691 .align 5 692 .globl __kuser_helper_start 693__kuser_helper_start: 694 695/* 696 * Due to the length of some sequences, __kuser_cmpxchg64 spans 2 regular 697 * kuser "slots", therefore 0xffff0f80 is not used as a valid entry point. 698 */ 699 700__kuser_cmpxchg64: @ 0xffff0f60 701 702#if defined(CONFIG_CPU_32v6K) 703 704 stmfd sp!, {r4, r5, r6, r7} 705 ldrd r4, r5, [r0] @ load old val 706 ldrd r6, r7, [r1] @ load new val 707 smp_dmb arm 7081: ldrexd r0, r1, [r2] @ load current val 709 eors r3, r0, r4 @ compare with oldval (1) 710 eorseq r3, r1, r5 @ compare with oldval (2) 711 strexdeq r3, r6, r7, [r2] @ store newval if eq 712 teqeq r3, #1 @ success? 713 beq 1b @ if no then retry 714 smp_dmb arm 715 rsbs r0, r3, #0 @ set returned val and C flag 716 ldmfd sp!, {r4, r5, r6, r7} 717 usr_ret lr 718 719#elif !defined(CONFIG_SMP) 720 721#ifdef CONFIG_MMU 722 723 /* 724 * The only thing that can break atomicity in this cmpxchg64 725 * implementation is either an IRQ or a data abort exception 726 * causing another process/thread to be scheduled in the middle of 727 * the critical sequence. The same strategy as for cmpxchg is used. 728 */ 729 stmfd sp!, {r4, r5, r6, lr} 730 ldmia r0, {r4, r5} @ load old val 731 ldmia r1, {r6, lr} @ load new val 7321: ldmia r2, {r0, r1} @ load current val 733 eors r3, r0, r4 @ compare with oldval (1) 734 eorseq r3, r1, r5 @ compare with oldval (2) 7352: stmiaeq r2, {r6, lr} @ store newval if eq 736 rsbs r0, r3, #0 @ set return val and C flag 737 ldmfd sp!, {r4, r5, r6, pc} 738 739 .text 740kuser_cmpxchg64_fixup: 741 @ Called from kuser_cmpxchg_fixup. 742 @ r4 = address of interrupted insn (must be preserved). 743 @ sp = saved regs. r7 and r8 are clobbered. 744 @ 1b = first critical insn, 2b = last critical insn. 745 @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b. 746 mov r7, #0xffff0fff 747 sub r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64))) 748 subs r8, r4, r7 749 rsbscs r8, r8, #(2b - 1b) 750 strcs r7, [sp, #S_PC] 751#if __LINUX_ARM_ARCH__ < 6 752 bcc kuser_cmpxchg32_fixup 753#endif 754 ret lr 755 .previous 756 757#else 758#warning "NPTL on non MMU needs fixing" 759 mov r0, #-1 760 adds r0, r0, #0 761 usr_ret lr 762#endif 763 764#else 765#error "incoherent kernel configuration" 766#endif 767 768 kuser_pad __kuser_cmpxchg64, 64 769 770__kuser_memory_barrier: @ 0xffff0fa0 771 smp_dmb arm 772 usr_ret lr 773 774 kuser_pad __kuser_memory_barrier, 32 775 776__kuser_cmpxchg: @ 0xffff0fc0 777 778#if __LINUX_ARM_ARCH__ < 6 779 780#ifdef CONFIG_MMU 781 782 /* 783 * The only thing that can break atomicity in this cmpxchg 784 * implementation is either an IRQ or a data abort exception 785 * causing another process/thread to be scheduled in the middle 786 * of the critical sequence. To prevent this, code is added to 787 * the IRQ and data abort exception handlers to set the pc back 788 * to the beginning of the critical section if it is found to be 789 * within that critical section (see kuser_cmpxchg_fixup). 790 */ 7911: ldr r3, [r2] @ load current val 792 subs r3, r3, r0 @ compare with oldval 7932: streq r1, [r2] @ store newval if eq 794 rsbs r0, r3, #0 @ set return val and C flag 795 usr_ret lr 796 797 .text 798kuser_cmpxchg32_fixup: 799 @ Called from kuser_cmpxchg_check macro. 800 @ r4 = address of interrupted insn (must be preserved). 801 @ sp = saved regs. r7 and r8 are clobbered. 802 @ 1b = first critical insn, 2b = last critical insn. 803 @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b. 804 mov r7, #0xffff0fff 805 sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) 806 subs r8, r4, r7 807 rsbscs r8, r8, #(2b - 1b) 808 strcs r7, [sp, #S_PC] 809 ret lr 810 .previous 811 812#else 813#warning "NPTL on non MMU needs fixing" 814 mov r0, #-1 815 adds r0, r0, #0 816 usr_ret lr 817#endif 818 819#else 820 821 smp_dmb arm 8221: ldrex r3, [r2] 823 subs r3, r3, r0 824 strexeq r3, r1, [r2] 825 teqeq r3, #1 826 beq 1b 827 rsbs r0, r3, #0 828 /* beware -- each __kuser slot must be 8 instructions max */ 829 ALT_SMP(b __kuser_memory_barrier) 830 ALT_UP(usr_ret lr) 831 832#endif 833 834 kuser_pad __kuser_cmpxchg, 32 835 836__kuser_get_tls: @ 0xffff0fe0 837 ldr r0, [pc, #(16 - 8)] @ read TLS, set in kuser_get_tls_init 838 usr_ret lr 839 mrc p15, 0, r0, c13, c0, 3 @ 0xffff0fe8 hardware TLS code 840 kuser_pad __kuser_get_tls, 16 841 .rep 3 842 .word 0 @ 0xffff0ff0 software TLS value, then 843 .endr @ pad up to __kuser_helper_version 844 845__kuser_helper_version: @ 0xffff0ffc 846 .word ((__kuser_helper_end - __kuser_helper_start) >> 5) 847 848 .globl __kuser_helper_end 849__kuser_helper_end: 850 851#endif 852 853 THUMB( .thumb ) 854 855/* 856 * Vector stubs. 857 * 858 * This code is copied to 0xffff1000 so we can use branches in the 859 * vectors, rather than ldr's. Note that this code must not exceed 860 * a page size. 861 * 862 * Common stub entry macro: 863 * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 864 * 865 * SP points to a minimal amount of processor-private memory, the address 866 * of which is copied into r0 for the mode specific abort handler. 867 */ 868 .macro vector_stub, name, mode, correction=0 869 .align 5 870#ifdef CONFIG_HARDEN_BRANCH_HISTORY 871vector_bhb_bpiall_\name: 872 mcr p15, 0, r0, c7, c5, 6 @ BPIALL 873 @ isb not needed due to "movs pc, lr" in the vector stub 874 @ which gives a "context synchronisation". 875#endif 876 877vector_\name: 878 .if \correction 879 sub lr, lr, #\correction 880 .endif 881 882 @ Save r0, lr_<exception> (parent PC) 883 stmia sp, {r0, lr} @ save r0, lr 884 885 @ Save spsr_<exception> (parent CPSR) 886.Lvec_\name: 887 mrs lr, spsr 888 str lr, [sp, #8] @ save spsr 889 890 @ 891 @ Prepare for SVC32 mode. IRQs remain disabled. 892 @ 893 mrs r0, cpsr 894 eor r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE) 895 msr spsr_cxsf, r0 896 897 @ 898 @ the branch table must immediately follow this code 899 @ 900 and lr, lr, #0x0f 901 THUMB( adr r0, 1f ) 902 THUMB( ldr lr, [r0, lr, lsl #2] ) 903 mov r0, sp 904 ARM( ldr lr, [pc, lr, lsl #2] ) 905 movs pc, lr @ branch to handler in SVC mode 906ENDPROC(vector_\name) 907 908#ifdef CONFIG_HARDEN_BRANCH_HISTORY 909 .subsection 1 910 .align 5 911vector_bhb_loop8_\name: 912 .if \correction 913 sub lr, lr, #\correction 914 .endif 915 916 @ Save r0, lr_<exception> (parent PC) 917 stmia sp, {r0, lr} 918 919 @ bhb workaround 920 mov r0, #8 9213: W(b) . + 4 922 subs r0, r0, #1 923 bne 3b 924 dsb nsh 925 @ isb not needed due to "movs pc, lr" in the vector stub 926 @ which gives a "context synchronisation". 927 b .Lvec_\name 928ENDPROC(vector_bhb_loop8_\name) 929 .previous 930#endif 931 932 .align 2 933 @ handler addresses follow this label 9341: 935 .endm 936 937 .section .stubs, "ax", %progbits 938 @ These need to remain at the start of the section so that 939 @ they are in range of the 'SWI' entries in the vector tables 940 @ located 4k down. 941.L__vector_swi: 942 .word vector_swi 943#ifdef CONFIG_HARDEN_BRANCH_HISTORY 944.L__vector_bhb_loop8_swi: 945 .word vector_bhb_loop8_swi 946.L__vector_bhb_bpiall_swi: 947 .word vector_bhb_bpiall_swi 948#endif 949 950vector_rst: 951 ARM( swi SYS_ERROR0 ) 952 THUMB( svc #0 ) 953 THUMB( nop ) 954 b vector_und 955 956/* 957 * Interrupt dispatcher 958 */ 959 vector_stub irq, IRQ_MODE, 4 960 961 .long __irq_usr @ 0 (USR_26 / USR_32) 962 .long __irq_invalid @ 1 (FIQ_26 / FIQ_32) 963 .long __irq_invalid @ 2 (IRQ_26 / IRQ_32) 964 .long __irq_svc @ 3 (SVC_26 / SVC_32) 965 .long __irq_invalid @ 4 966 .long __irq_invalid @ 5 967 .long __irq_invalid @ 6 968 .long __irq_invalid @ 7 969 .long __irq_invalid @ 8 970 .long __irq_invalid @ 9 971 .long __irq_invalid @ a 972 .long __irq_invalid @ b 973 .long __irq_invalid @ c 974 .long __irq_invalid @ d 975 .long __irq_invalid @ e 976 .long __irq_invalid @ f 977 978/* 979 * Data abort dispatcher 980 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 981 */ 982 vector_stub dabt, ABT_MODE, 8 983 984 .long __dabt_usr @ 0 (USR_26 / USR_32) 985 .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32) 986 .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32) 987 .long __dabt_svc @ 3 (SVC_26 / SVC_32) 988 .long __dabt_invalid @ 4 989 .long __dabt_invalid @ 5 990 .long __dabt_invalid @ 6 991 .long __dabt_invalid @ 7 992 .long __dabt_invalid @ 8 993 .long __dabt_invalid @ 9 994 .long __dabt_invalid @ a 995 .long __dabt_invalid @ b 996 .long __dabt_invalid @ c 997 .long __dabt_invalid @ d 998 .long __dabt_invalid @ e 999 .long __dabt_invalid @ f 1000 1001/* 1002 * Prefetch abort dispatcher 1003 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 1004 */ 1005 vector_stub pabt, ABT_MODE, 4 1006 1007 .long __pabt_usr @ 0 (USR_26 / USR_32) 1008 .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32) 1009 .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32) 1010 .long __pabt_svc @ 3 (SVC_26 / SVC_32) 1011 .long __pabt_invalid @ 4 1012 .long __pabt_invalid @ 5 1013 .long __pabt_invalid @ 6 1014 .long __pabt_invalid @ 7 1015 .long __pabt_invalid @ 8 1016 .long __pabt_invalid @ 9 1017 .long __pabt_invalid @ a 1018 .long __pabt_invalid @ b 1019 .long __pabt_invalid @ c 1020 .long __pabt_invalid @ d 1021 .long __pabt_invalid @ e 1022 .long __pabt_invalid @ f 1023 1024/* 1025 * Undef instr entry dispatcher 1026 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 1027 */ 1028 vector_stub und, UND_MODE 1029 1030 .long __und_usr @ 0 (USR_26 / USR_32) 1031 .long __und_invalid @ 1 (FIQ_26 / FIQ_32) 1032 .long __und_invalid @ 2 (IRQ_26 / IRQ_32) 1033 .long __und_svc @ 3 (SVC_26 / SVC_32) 1034 .long __und_invalid @ 4 1035 .long __und_invalid @ 5 1036 .long __und_invalid @ 6 1037 .long __und_invalid @ 7 1038 .long __und_invalid @ 8 1039 .long __und_invalid @ 9 1040 .long __und_invalid @ a 1041 .long __und_invalid @ b 1042 .long __und_invalid @ c 1043 .long __und_invalid @ d 1044 .long __und_invalid @ e 1045 .long __und_invalid @ f 1046 1047 .align 5 1048 1049/*============================================================================= 1050 * Address exception handler 1051 *----------------------------------------------------------------------------- 1052 * These aren't too critical. 1053 * (they're not supposed to happen, and won't happen in 32-bit data mode). 1054 */ 1055 1056vector_addrexcptn: 1057 b vector_addrexcptn 1058 1059/*============================================================================= 1060 * FIQ "NMI" handler 1061 *----------------------------------------------------------------------------- 1062 * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86 1063 * systems. This must be the last vector stub, so lets place it in its own 1064 * subsection. 1065 */ 1066 .subsection 2 1067 vector_stub fiq, FIQ_MODE, 4 1068 1069 .long __fiq_usr @ 0 (USR_26 / USR_32) 1070 .long __fiq_svc @ 1 (FIQ_26 / FIQ_32) 1071 .long __fiq_svc @ 2 (IRQ_26 / IRQ_32) 1072 .long __fiq_svc @ 3 (SVC_26 / SVC_32) 1073 .long __fiq_svc @ 4 1074 .long __fiq_svc @ 5 1075 .long __fiq_svc @ 6 1076 .long __fiq_abt @ 7 1077 .long __fiq_svc @ 8 1078 .long __fiq_svc @ 9 1079 .long __fiq_svc @ a 1080 .long __fiq_svc @ b 1081 .long __fiq_svc @ c 1082 .long __fiq_svc @ d 1083 .long __fiq_svc @ e 1084 .long __fiq_svc @ f 1085 1086 .globl vector_fiq 1087 1088 .section .vectors, "ax", %progbits 1089 RELOC_TEXT_NONE 1090 W(b) vector_rst 1091 W(b) vector_und 1092ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_swi ) 1093THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_swi ) 1094 W(ldr) pc, . 1095 W(b) vector_pabt 1096 W(b) vector_dabt 1097 W(b) vector_addrexcptn 1098 W(b) vector_irq 1099 W(b) vector_fiq 1100 1101#ifdef CONFIG_HARDEN_BRANCH_HISTORY 1102 .section .vectors.bhb.loop8, "ax", %progbits 1103 RELOC_TEXT_NONE 1104 W(b) vector_rst 1105 W(b) vector_bhb_loop8_und 1106ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_loop8_swi ) 1107THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_loop8_swi ) 1108 W(ldr) pc, . 1109 W(b) vector_bhb_loop8_pabt 1110 W(b) vector_bhb_loop8_dabt 1111 W(b) vector_addrexcptn 1112 W(b) vector_bhb_loop8_irq 1113 W(b) vector_bhb_loop8_fiq 1114 1115 .section .vectors.bhb.bpiall, "ax", %progbits 1116 RELOC_TEXT_NONE 1117 W(b) vector_rst 1118 W(b) vector_bhb_bpiall_und 1119ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_bpiall_swi ) 1120THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_bpiall_swi ) 1121 W(ldr) pc, . 1122 W(b) vector_bhb_bpiall_pabt 1123 W(b) vector_bhb_bpiall_dabt 1124 W(b) vector_addrexcptn 1125 W(b) vector_bhb_bpiall_irq 1126 W(b) vector_bhb_bpiall_fiq 1127#endif 1128 1129 .data 1130 .align 2 1131 1132 .globl cr_alignment 1133cr_alignment: 1134 .space 4 1135