1/* 2 * linux/arch/arm/kernel/entry-armv.S 3 * 4 * Copyright (C) 1996,1997,1998 Russell King. 5 * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk) 6 * nommu support by Hyok S. Choi (hyok.choi@samsung.com) 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * Low-level vector interface routines 13 * 14 * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction 15 * that causes it to save wrong values... Be aware! 16 */ 17 18#include <asm/memory.h> 19#include <asm/glue.h> 20#include <asm/vfpmacros.h> 21#include <mach/entry-macro.S> 22#include <asm/thread_notify.h> 23 24#include "entry-header.S" 25 26/* 27 * Interrupt handling. Preserves r7, r8, r9 28 */ 29 .macro irq_handler 30 get_irqnr_preamble r5, lr 311: get_irqnr_and_base r0, r6, r5, lr 32 movne r1, sp 33 @ 34 @ routine called with r0 = irq number, r1 = struct pt_regs * 35 @ 36 adrne lr, 1b 37 bne asm_do_IRQ 38 39#ifdef CONFIG_SMP 40 /* 41 * XXX 42 * 43 * this macro assumes that irqstat (r6) and base (r5) are 44 * preserved from get_irqnr_and_base above 45 */ 46 test_for_ipi r0, r6, r5, lr 47 movne r0, sp 48 adrne lr, 1b 49 bne do_IPI 50 51#ifdef CONFIG_LOCAL_TIMERS 52 test_for_ltirq r0, r6, r5, lr 53 movne r0, sp 54 adrne lr, 1b 55 bne do_local_timer 56#endif 57#endif 58 59 .endm 60 61#ifdef CONFIG_KPROBES 62 .section .kprobes.text,"ax",%progbits 63#else 64 .text 65#endif 66 67/* 68 * Invalid mode handlers 69 */ 70 .macro inv_entry, reason 71 sub sp, sp, #S_FRAME_SIZE 72 stmib sp, {r1 - lr} 73 mov r1, #\reason 74 .endm 75 76__pabt_invalid: 77 inv_entry BAD_PREFETCH 78 b common_invalid 79ENDPROC(__pabt_invalid) 80 81__dabt_invalid: 82 inv_entry BAD_DATA 83 b common_invalid 84ENDPROC(__dabt_invalid) 85 86__irq_invalid: 87 inv_entry BAD_IRQ 88 b common_invalid 89ENDPROC(__irq_invalid) 90 91__und_invalid: 92 inv_entry BAD_UNDEFINSTR 93 94 @ 95 @ XXX fall through to common_invalid 96 @ 97 98@ 99@ common_invalid - generic code for failed exception (re-entrant version of handlers) 100@ 101common_invalid: 102 zero_fp 103 104 ldmia r0, {r4 - r6} 105 add r0, sp, #S_PC @ here for interlock avoidance 106 mov r7, #-1 @ "" "" "" "" 107 str r4, [sp] @ save preserved r0 108 stmia r0, {r5 - r7} @ lr_<exception>, 109 @ cpsr_<exception>, "old_r0" 110 111 mov r0, sp 112 b bad_mode 113ENDPROC(__und_invalid) 114 115/* 116 * SVC mode handlers 117 */ 118 119#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) 120#define SPFIX(code...) code 121#else 122#define SPFIX(code...) 123#endif 124 125 .macro svc_entry, stack_hole=0 126 sub sp, sp, #(S_FRAME_SIZE + \stack_hole) 127 SPFIX( tst sp, #4 ) 128 SPFIX( bicne sp, sp, #4 ) 129 stmib sp, {r1 - r12} 130 131 ldmia r0, {r1 - r3} 132 add r5, sp, #S_SP @ here for interlock avoidance 133 mov r4, #-1 @ "" "" "" "" 134 add r0, sp, #(S_FRAME_SIZE + \stack_hole) 135 SPFIX( addne r0, r0, #4 ) 136 str r1, [sp] @ save the "real" r0 copied 137 @ from the exception stack 138 139 mov r1, lr 140 141 @ 142 @ We are now ready to fill in the remaining blanks on the stack: 143 @ 144 @ r0 - sp_svc 145 @ r1 - lr_svc 146 @ r2 - lr_<exception>, already fixed up for correct return/restart 147 @ r3 - spsr_<exception> 148 @ r4 - orig_r0 (see pt_regs definition in ptrace.h) 149 @ 150 stmia r5, {r0 - r4} 151 .endm 152 153 .align 5 154__dabt_svc: 155 svc_entry 156 157 @ 158 @ get ready to re-enable interrupts if appropriate 159 @ 160 mrs r9, cpsr 161 tst r3, #PSR_I_BIT 162 biceq r9, r9, #PSR_I_BIT 163 164 @ 165 @ Call the processor-specific abort handler: 166 @ 167 @ r2 - aborted context pc 168 @ r3 - aborted context cpsr 169 @ 170 @ The abort handler must return the aborted address in r0, and 171 @ the fault status register in r1. r9 must be preserved. 172 @ 173#ifdef MULTI_DABORT 174 ldr r4, .LCprocfns 175 mov lr, pc 176 ldr pc, [r4, #PROCESSOR_DABT_FUNC] 177#else 178 bl CPU_DABORT_HANDLER 179#endif 180 181 @ 182 @ set desired IRQ state, then call main handler 183 @ 184 msr cpsr_c, r9 185 mov r2, sp 186 bl do_DataAbort 187 188 @ 189 @ IRQs off again before pulling preserved data off the stack 190 @ 191 disable_irq 192 193 @ 194 @ restore SPSR and restart the instruction 195 @ 196 ldr r0, [sp, #S_PSR] 197 msr spsr_cxsf, r0 198 ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr 199ENDPROC(__dabt_svc) 200 201 .align 5 202__irq_svc: 203 svc_entry 204 205#ifdef CONFIG_TRACE_IRQFLAGS 206 bl trace_hardirqs_off 207#endif 208#ifdef CONFIG_PREEMPT 209 get_thread_info tsk 210 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count 211 add r7, r8, #1 @ increment it 212 str r7, [tsk, #TI_PREEMPT] 213#endif 214 215 irq_handler 216#ifdef CONFIG_PREEMPT 217 str r8, [tsk, #TI_PREEMPT] @ restore preempt count 218 ldr r0, [tsk, #TI_FLAGS] @ get flags 219 teq r8, #0 @ if preempt count != 0 220 movne r0, #0 @ force flags to 0 221 tst r0, #_TIF_NEED_RESCHED 222 blne svc_preempt 223#endif 224 ldr r0, [sp, #S_PSR] @ irqs are already disabled 225 msr spsr_cxsf, r0 226#ifdef CONFIG_TRACE_IRQFLAGS 227 tst r0, #PSR_I_BIT 228 bleq trace_hardirqs_on 229#endif 230 ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr 231ENDPROC(__irq_svc) 232 233 .ltorg 234 235#ifdef CONFIG_PREEMPT 236svc_preempt: 237 mov r8, lr 2381: bl preempt_schedule_irq @ irq en/disable is done inside 239 ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS 240 tst r0, #_TIF_NEED_RESCHED 241 moveq pc, r8 @ go again 242 b 1b 243#endif 244 245 .align 5 246__und_svc: 247#ifdef CONFIG_KPROBES 248 @ If a kprobe is about to simulate a "stmdb sp..." instruction, 249 @ it obviously needs free stack space which then will belong to 250 @ the saved context. 251 svc_entry 64 252#else 253 svc_entry 254#endif 255 256 @ 257 @ call emulation code, which returns using r9 if it has emulated 258 @ the instruction, or the more conventional lr if we are to treat 259 @ this as a real undefined instruction 260 @ 261 @ r0 - instruction 262 @ 263 ldr r0, [r2, #-4] 264 adr r9, 1f 265 bl call_fpe 266 267 mov r0, sp @ struct pt_regs *regs 268 bl do_undefinstr 269 270 @ 271 @ IRQs off again before pulling preserved data off the stack 272 @ 2731: disable_irq 274 275 @ 276 @ restore SPSR and restart the instruction 277 @ 278 ldr lr, [sp, #S_PSR] @ Get SVC cpsr 279 msr spsr_cxsf, lr 280 ldmia sp, {r0 - pc}^ @ Restore SVC registers 281ENDPROC(__und_svc) 282 283 .align 5 284__pabt_svc: 285 svc_entry 286 287 @ 288 @ re-enable interrupts if appropriate 289 @ 290 mrs r9, cpsr 291 tst r3, #PSR_I_BIT 292 biceq r9, r9, #PSR_I_BIT 293 294 @ 295 @ set args, then call main handler 296 @ 297 @ r0 - address of faulting instruction 298 @ r1 - pointer to registers on stack 299 @ 300#ifdef MULTI_PABORT 301 mov r0, r2 @ pass address of aborted instruction. 302 ldr r4, .LCprocfns 303 mov lr, pc 304 ldr pc, [r4, #PROCESSOR_PABT_FUNC] 305#else 306 CPU_PABORT_HANDLER(r0, r2) 307#endif 308 msr cpsr_c, r9 @ Maybe enable interrupts 309 mov r1, sp @ regs 310 bl do_PrefetchAbort @ call abort handler 311 312 @ 313 @ IRQs off again before pulling preserved data off the stack 314 @ 315 disable_irq 316 317 @ 318 @ restore SPSR and restart the instruction 319 @ 320 ldr r0, [sp, #S_PSR] 321 msr spsr_cxsf, r0 322 ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr 323ENDPROC(__pabt_svc) 324 325 .align 5 326.LCcralign: 327 .word cr_alignment 328#ifdef MULTI_DABORT 329.LCprocfns: 330 .word processor 331#endif 332.LCfp: 333 .word fp_enter 334 335/* 336 * User mode handlers 337 * 338 * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE 339 */ 340 341#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7) 342#error "sizeof(struct pt_regs) must be a multiple of 8" 343#endif 344 345 .macro usr_entry 346 sub sp, sp, #S_FRAME_SIZE 347 stmib sp, {r1 - r12} 348 349 ldmia r0, {r1 - r3} 350 add r0, sp, #S_PC @ here for interlock avoidance 351 mov r4, #-1 @ "" "" "" "" 352 353 str r1, [sp] @ save the "real" r0 copied 354 @ from the exception stack 355 356 @ 357 @ We are now ready to fill in the remaining blanks on the stack: 358 @ 359 @ r2 - lr_<exception>, already fixed up for correct return/restart 360 @ r3 - spsr_<exception> 361 @ r4 - orig_r0 (see pt_regs definition in ptrace.h) 362 @ 363 @ Also, separately save sp_usr and lr_usr 364 @ 365 stmia r0, {r2 - r4} 366 stmdb r0, {sp, lr}^ 367 368 @ 369 @ Enable the alignment trap while in kernel mode 370 @ 371 alignment_trap r0 372 373 @ 374 @ Clear FP to mark the first stack frame 375 @ 376 zero_fp 377 .endm 378 379 .macro kuser_cmpxchg_check 380#if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) 381#ifndef CONFIG_MMU 382#warning "NPTL on non MMU needs fixing" 383#else 384 @ Make sure our user space atomic helper is restarted 385 @ if it was interrupted in a critical region. Here we 386 @ perform a quick test inline since it should be false 387 @ 99.9999% of the time. The rest is done out of line. 388 cmp r2, #TASK_SIZE 389 blhs kuser_cmpxchg_fixup 390#endif 391#endif 392 .endm 393 394 .align 5 395__dabt_usr: 396 usr_entry 397 kuser_cmpxchg_check 398 399 @ 400 @ Call the processor-specific abort handler: 401 @ 402 @ r2 - aborted context pc 403 @ r3 - aborted context cpsr 404 @ 405 @ The abort handler must return the aborted address in r0, and 406 @ the fault status register in r1. 407 @ 408#ifdef MULTI_DABORT 409 ldr r4, .LCprocfns 410 mov lr, pc 411 ldr pc, [r4, #PROCESSOR_DABT_FUNC] 412#else 413 bl CPU_DABORT_HANDLER 414#endif 415 416 @ 417 @ IRQs on, then call the main handler 418 @ 419 enable_irq 420 mov r2, sp 421 adr lr, ret_from_exception 422 b do_DataAbort 423ENDPROC(__dabt_usr) 424 425 .align 5 426__irq_usr: 427 usr_entry 428 kuser_cmpxchg_check 429 430#ifdef CONFIG_TRACE_IRQFLAGS 431 bl trace_hardirqs_off 432#endif 433 get_thread_info tsk 434#ifdef CONFIG_PREEMPT 435 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count 436 add r7, r8, #1 @ increment it 437 str r7, [tsk, #TI_PREEMPT] 438#endif 439 440 irq_handler 441#ifdef CONFIG_PREEMPT 442 ldr r0, [tsk, #TI_PREEMPT] 443 str r8, [tsk, #TI_PREEMPT] 444 teq r0, r7 445 strne r0, [r0, -r0] 446#endif 447#ifdef CONFIG_TRACE_IRQFLAGS 448 bl trace_hardirqs_on 449#endif 450 451 mov why, #0 452 b ret_to_user 453ENDPROC(__irq_usr) 454 455 .ltorg 456 457 .align 5 458__und_usr: 459 usr_entry 460 461 @ 462 @ fall through to the emulation code, which returns using r9 if 463 @ it has emulated the instruction, or the more conventional lr 464 @ if we are to treat this as a real undefined instruction 465 @ 466 @ r0 - instruction 467 @ 468 adr r9, ret_from_exception 469 adr lr, __und_usr_unknown 470 tst r3, #PSR_T_BIT @ Thumb mode? 471 subeq r4, r2, #4 @ ARM instr at LR - 4 472 subne r4, r2, #2 @ Thumb instr at LR - 2 4731: ldreqt r0, [r4] 474 beq call_fpe 475 @ Thumb instruction 476#if __LINUX_ARM_ARCH__ >= 7 4772: ldrht r5, [r4], #2 478 and r0, r5, #0xf800 @ mask bits 111x x... .... .... 479 cmp r0, #0xe800 @ 32bit instruction if xx != 0 480 blo __und_usr_unknown 4813: ldrht r0, [r4] 482 add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 483 orr r0, r0, r5, lsl #16 484#else 485 b __und_usr_unknown 486#endif 487ENDPROC(__und_usr) 488 489 @ 490 @ fallthrough to call_fpe 491 @ 492 493/* 494 * The out of line fixup for the ldrt above. 495 */ 496 .section .fixup, "ax" 4974: mov pc, r9 498 .previous 499 .section __ex_table,"a" 500 .long 1b, 4b 501#if __LINUX_ARM_ARCH__ >= 7 502 .long 2b, 4b 503 .long 3b, 4b 504#endif 505 .previous 506 507/* 508 * Check whether the instruction is a co-processor instruction. 509 * If yes, we need to call the relevant co-processor handler. 510 * 511 * Note that we don't do a full check here for the co-processor 512 * instructions; all instructions with bit 27 set are well 513 * defined. The only instructions that should fault are the 514 * co-processor instructions. However, we have to watch out 515 * for the ARM6/ARM7 SWI bug. 516 * 517 * NEON is a special case that has to be handled here. Not all 518 * NEON instructions are co-processor instructions, so we have 519 * to make a special case of checking for them. Plus, there's 520 * five groups of them, so we have a table of mask/opcode pairs 521 * to check against, and if any match then we branch off into the 522 * NEON handler code. 523 * 524 * Emulators may wish to make use of the following registers: 525 * r0 = instruction opcode. 526 * r2 = PC+4 527 * r9 = normal "successful" return address 528 * r10 = this threads thread_info structure. 529 * lr = unrecognised instruction return address 530 */ 531 @ 532 @ Fall-through from Thumb-2 __und_usr 533 @ 534#ifdef CONFIG_NEON 535 adr r6, .LCneon_thumb_opcodes 536 b 2f 537#endif 538call_fpe: 539#ifdef CONFIG_NEON 540 adr r6, .LCneon_arm_opcodes 5412: 542 ldr r7, [r6], #4 @ mask value 543 cmp r7, #0 @ end mask? 544 beq 1f 545 and r8, r0, r7 546 ldr r7, [r6], #4 @ opcode bits matching in mask 547 cmp r8, r7 @ NEON instruction? 548 bne 2b 549 get_thread_info r10 550 mov r7, #1 551 strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used 552 strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used 553 b do_vfp @ let VFP handler handle this 5541: 555#endif 556 tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 557 tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 558#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710) 559 and r8, r0, #0x0f000000 @ mask out op-code bits 560 teqne r8, #0x0f000000 @ SWI (ARM6/7 bug)? 561#endif 562 moveq pc, lr 563 get_thread_info r10 @ get current thread 564 and r8, r0, #0x00000f00 @ mask out CP number 565 mov r7, #1 566 add r6, r10, #TI_USED_CP 567 strb r7, [r6, r8, lsr #8] @ set appropriate used_cp[] 568#ifdef CONFIG_IWMMXT 569 @ Test if we need to give access to iWMMXt coprocessors 570 ldr r5, [r10, #TI_FLAGS] 571 rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only 572 movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1) 573 bcs iwmmxt_task_enable 574#endif 575 add pc, pc, r8, lsr #6 576 mov r0, r0 577 578 mov pc, lr @ CP#0 579 b do_fpe @ CP#1 (FPE) 580 b do_fpe @ CP#2 (FPE) 581 mov pc, lr @ CP#3 582#ifdef CONFIG_CRUNCH 583 b crunch_task_enable @ CP#4 (MaverickCrunch) 584 b crunch_task_enable @ CP#5 (MaverickCrunch) 585 b crunch_task_enable @ CP#6 (MaverickCrunch) 586#else 587 mov pc, lr @ CP#4 588 mov pc, lr @ CP#5 589 mov pc, lr @ CP#6 590#endif 591 mov pc, lr @ CP#7 592 mov pc, lr @ CP#8 593 mov pc, lr @ CP#9 594#ifdef CONFIG_VFP 595 b do_vfp @ CP#10 (VFP) 596 b do_vfp @ CP#11 (VFP) 597#else 598 mov pc, lr @ CP#10 (VFP) 599 mov pc, lr @ CP#11 (VFP) 600#endif 601 mov pc, lr @ CP#12 602 mov pc, lr @ CP#13 603 mov pc, lr @ CP#14 (Debug) 604 mov pc, lr @ CP#15 (Control) 605 606#ifdef CONFIG_NEON 607 .align 6 608 609.LCneon_arm_opcodes: 610 .word 0xfe000000 @ mask 611 .word 0xf2000000 @ opcode 612 613 .word 0xff100000 @ mask 614 .word 0xf4000000 @ opcode 615 616 .word 0x00000000 @ mask 617 .word 0x00000000 @ opcode 618 619.LCneon_thumb_opcodes: 620 .word 0xef000000 @ mask 621 .word 0xef000000 @ opcode 622 623 .word 0xff100000 @ mask 624 .word 0xf9000000 @ opcode 625 626 .word 0x00000000 @ mask 627 .word 0x00000000 @ opcode 628#endif 629 630do_fpe: 631 enable_irq 632 ldr r4, .LCfp 633 add r10, r10, #TI_FPSTATE @ r10 = workspace 634 ldr pc, [r4] @ Call FP module USR entry point 635 636/* 637 * The FP module is called with these registers set: 638 * r0 = instruction 639 * r2 = PC+4 640 * r9 = normal "successful" return address 641 * r10 = FP workspace 642 * lr = unrecognised FP instruction return address 643 */ 644 645 .data 646ENTRY(fp_enter) 647 .word no_fp 648 .previous 649 650no_fp: mov pc, lr 651 652__und_usr_unknown: 653 enable_irq 654 mov r0, sp 655 adr lr, ret_from_exception 656 b do_undefinstr 657ENDPROC(__und_usr_unknown) 658 659 .align 5 660__pabt_usr: 661 usr_entry 662 663#ifdef MULTI_PABORT 664 mov r0, r2 @ pass address of aborted instruction. 665 ldr r4, .LCprocfns 666 mov lr, pc 667 ldr pc, [r4, #PROCESSOR_PABT_FUNC] 668#else 669 CPU_PABORT_HANDLER(r0, r2) 670#endif 671 enable_irq @ Enable interrupts 672 mov r1, sp @ regs 673 bl do_PrefetchAbort @ call abort handler 674 /* fall through */ 675/* 676 * This is the return code to user mode for abort handlers 677 */ 678ENTRY(ret_from_exception) 679 get_thread_info tsk 680 mov why, #0 681 b ret_to_user 682ENDPROC(__pabt_usr) 683ENDPROC(ret_from_exception) 684 685/* 686 * Register switch for ARMv3 and ARMv4 processors 687 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info 688 * previous and next are guaranteed not to be the same. 689 */ 690ENTRY(__switch_to) 691 add ip, r1, #TI_CPU_SAVE 692 ldr r3, [r2, #TI_TP_VALUE] 693 stmia ip!, {r4 - sl, fp, sp, lr} @ Store most regs on stack 694#ifdef CONFIG_MMU 695 ldr r6, [r2, #TI_CPU_DOMAIN] 696#endif 697#if __LINUX_ARM_ARCH__ >= 6 698#ifdef CONFIG_CPU_32v6K 699 clrex 700#else 701 strex r5, r4, [ip] @ Clear exclusive monitor 702#endif 703#endif 704#if defined(CONFIG_HAS_TLS_REG) 705 mcr p15, 0, r3, c13, c0, 3 @ set TLS register 706#elif !defined(CONFIG_TLS_REG_EMUL) 707 mov r4, #0xffff0fff 708 str r3, [r4, #-15] @ TLS val at 0xffff0ff0 709#endif 710#ifdef CONFIG_MMU 711 mcr p15, 0, r6, c3, c0, 0 @ Set domain register 712#endif 713 mov r5, r0 714 add r4, r2, #TI_CPU_SAVE 715 ldr r0, =thread_notify_head 716 mov r1, #THREAD_NOTIFY_SWITCH 717 bl atomic_notifier_call_chain 718 mov r0, r5 719 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously 720ENDPROC(__switch_to) 721 722 __INIT 723 724/* 725 * User helpers. 726 * 727 * These are segment of kernel provided user code reachable from user space 728 * at a fixed address in kernel memory. This is used to provide user space 729 * with some operations which require kernel help because of unimplemented 730 * native feature and/or instructions in many ARM CPUs. The idea is for 731 * this code to be executed directly in user mode for best efficiency but 732 * which is too intimate with the kernel counter part to be left to user 733 * libraries. In fact this code might even differ from one CPU to another 734 * depending on the available instruction set and restrictions like on 735 * SMP systems. In other words, the kernel reserves the right to change 736 * this code as needed without warning. Only the entry points and their 737 * results are guaranteed to be stable. 738 * 739 * Each segment is 32-byte aligned and will be moved to the top of the high 740 * vector page. New segments (if ever needed) must be added in front of 741 * existing ones. This mechanism should be used only for things that are 742 * really small and justified, and not be abused freely. 743 * 744 * User space is expected to implement those things inline when optimizing 745 * for a processor that has the necessary native support, but only if such 746 * resulting binaries are already to be incompatible with earlier ARM 747 * processors due to the use of unsupported instructions other than what 748 * is provided here. In other words don't make binaries unable to run on 749 * earlier processors just for the sake of not using these kernel helpers 750 * if your compiled code is not going to use the new instructions for other 751 * purpose. 752 */ 753 754 .macro usr_ret, reg 755#ifdef CONFIG_ARM_THUMB 756 bx \reg 757#else 758 mov pc, \reg 759#endif 760 .endm 761 762 .align 5 763 .globl __kuser_helper_start 764__kuser_helper_start: 765 766/* 767 * Reference prototype: 768 * 769 * void __kernel_memory_barrier(void) 770 * 771 * Input: 772 * 773 * lr = return address 774 * 775 * Output: 776 * 777 * none 778 * 779 * Clobbered: 780 * 781 * none 782 * 783 * Definition and user space usage example: 784 * 785 * typedef void (__kernel_dmb_t)(void); 786 * #define __kernel_dmb (*(__kernel_dmb_t *)0xffff0fa0) 787 * 788 * Apply any needed memory barrier to preserve consistency with data modified 789 * manually and __kuser_cmpxchg usage. 790 * 791 * This could be used as follows: 792 * 793 * #define __kernel_dmb() \ 794 * asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \ 795 * : : : "r0", "lr","cc" ) 796 */ 797 798__kuser_memory_barrier: @ 0xffff0fa0 799 800#if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_SMP) 801 mcr p15, 0, r0, c7, c10, 5 @ dmb 802#endif 803 usr_ret lr 804 805 .align 5 806 807/* 808 * Reference prototype: 809 * 810 * int __kernel_cmpxchg(int oldval, int newval, int *ptr) 811 * 812 * Input: 813 * 814 * r0 = oldval 815 * r1 = newval 816 * r2 = ptr 817 * lr = return address 818 * 819 * Output: 820 * 821 * r0 = returned value (zero or non-zero) 822 * C flag = set if r0 == 0, clear if r0 != 0 823 * 824 * Clobbered: 825 * 826 * r3, ip, flags 827 * 828 * Definition and user space usage example: 829 * 830 * typedef int (__kernel_cmpxchg_t)(int oldval, int newval, int *ptr); 831 * #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *)0xffff0fc0) 832 * 833 * Atomically store newval in *ptr if *ptr is equal to oldval for user space. 834 * Return zero if *ptr was changed or non-zero if no exchange happened. 835 * The C flag is also set if *ptr was changed to allow for assembly 836 * optimization in the calling code. 837 * 838 * Notes: 839 * 840 * - This routine already includes memory barriers as needed. 841 * 842 * For example, a user space atomic_add implementation could look like this: 843 * 844 * #define atomic_add(ptr, val) \ 845 * ({ register unsigned int *__ptr asm("r2") = (ptr); \ 846 * register unsigned int __result asm("r1"); \ 847 * asm volatile ( \ 848 * "1: @ atomic_add\n\t" \ 849 * "ldr r0, [r2]\n\t" \ 850 * "mov r3, #0xffff0fff\n\t" \ 851 * "add lr, pc, #4\n\t" \ 852 * "add r1, r0, %2\n\t" \ 853 * "add pc, r3, #(0xffff0fc0 - 0xffff0fff)\n\t" \ 854 * "bcc 1b" \ 855 * : "=&r" (__result) \ 856 * : "r" (__ptr), "rIL" (val) \ 857 * : "r0","r3","ip","lr","cc","memory" ); \ 858 * __result; }) 859 */ 860 861__kuser_cmpxchg: @ 0xffff0fc0 862 863#if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) 864 865 /* 866 * Poor you. No fast solution possible... 867 * The kernel itself must perform the operation. 868 * A special ghost syscall is used for that (see traps.c). 869 */ 870 stmfd sp!, {r7, lr} 871 mov r7, #0xff00 @ 0xfff0 into r7 for EABI 872 orr r7, r7, #0xf0 873 swi #0x9ffff0 874 ldmfd sp!, {r7, pc} 875 876#elif __LINUX_ARM_ARCH__ < 6 877 878#ifdef CONFIG_MMU 879 880 /* 881 * The only thing that can break atomicity in this cmpxchg 882 * implementation is either an IRQ or a data abort exception 883 * causing another process/thread to be scheduled in the middle 884 * of the critical sequence. To prevent this, code is added to 885 * the IRQ and data abort exception handlers to set the pc back 886 * to the beginning of the critical section if it is found to be 887 * within that critical section (see kuser_cmpxchg_fixup). 888 */ 8891: ldr r3, [r2] @ load current val 890 subs r3, r3, r0 @ compare with oldval 8912: streq r1, [r2] @ store newval if eq 892 rsbs r0, r3, #0 @ set return val and C flag 893 usr_ret lr 894 895 .text 896kuser_cmpxchg_fixup: 897 @ Called from kuser_cmpxchg_check macro. 898 @ r2 = address of interrupted insn (must be preserved). 899 @ sp = saved regs. r7 and r8 are clobbered. 900 @ 1b = first critical insn, 2b = last critical insn. 901 @ If r2 >= 1b and r2 <= 2b then saved pc_usr is set to 1b. 902 mov r7, #0xffff0fff 903 sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) 904 subs r8, r2, r7 905 rsbcss r8, r8, #(2b - 1b) 906 strcs r7, [sp, #S_PC] 907 mov pc, lr 908 .previous 909 910#else 911#warning "NPTL on non MMU needs fixing" 912 mov r0, #-1 913 adds r0, r0, #0 914 usr_ret lr 915#endif 916 917#else 918 919#ifdef CONFIG_SMP 920 mcr p15, 0, r0, c7, c10, 5 @ dmb 921#endif 9221: ldrex r3, [r2] 923 subs r3, r3, r0 924 strexeq r3, r1, [r2] 925 teqeq r3, #1 926 beq 1b 927 rsbs r0, r3, #0 928 /* beware -- each __kuser slot must be 8 instructions max */ 929#ifdef CONFIG_SMP 930 b __kuser_memory_barrier 931#else 932 usr_ret lr 933#endif 934 935#endif 936 937 .align 5 938 939/* 940 * Reference prototype: 941 * 942 * int __kernel_get_tls(void) 943 * 944 * Input: 945 * 946 * lr = return address 947 * 948 * Output: 949 * 950 * r0 = TLS value 951 * 952 * Clobbered: 953 * 954 * none 955 * 956 * Definition and user space usage example: 957 * 958 * typedef int (__kernel_get_tls_t)(void); 959 * #define __kernel_get_tls (*(__kernel_get_tls_t *)0xffff0fe0) 960 * 961 * Get the TLS value as previously set via the __ARM_NR_set_tls syscall. 962 * 963 * This could be used as follows: 964 * 965 * #define __kernel_get_tls() \ 966 * ({ register unsigned int __val asm("r0"); \ 967 * asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \ 968 * : "=r" (__val) : : "lr","cc" ); \ 969 * __val; }) 970 */ 971 972__kuser_get_tls: @ 0xffff0fe0 973 974#if !defined(CONFIG_HAS_TLS_REG) && !defined(CONFIG_TLS_REG_EMUL) 975 ldr r0, [pc, #(16 - 8)] @ TLS stored at 0xffff0ff0 976#else 977 mrc p15, 0, r0, c13, c0, 3 @ read TLS register 978#endif 979 usr_ret lr 980 981 .rep 5 982 .word 0 @ pad up to __kuser_helper_version 983 .endr 984 985/* 986 * Reference declaration: 987 * 988 * extern unsigned int __kernel_helper_version; 989 * 990 * Definition and user space usage example: 991 * 992 * #define __kernel_helper_version (*(unsigned int *)0xffff0ffc) 993 * 994 * User space may read this to determine the curent number of helpers 995 * available. 996 */ 997 998__kuser_helper_version: @ 0xffff0ffc 999 .word ((__kuser_helper_end - __kuser_helper_start) >> 5) 1000 1001 .globl __kuser_helper_end 1002__kuser_helper_end: 1003 1004 1005/* 1006 * Vector stubs. 1007 * 1008 * This code is copied to 0xffff0200 so we can use branches in the 1009 * vectors, rather than ldr's. Note that this code must not 1010 * exceed 0x300 bytes. 1011 * 1012 * Common stub entry macro: 1013 * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 1014 * 1015 * SP points to a minimal amount of processor-private memory, the address 1016 * of which is copied into r0 for the mode specific abort handler. 1017 */ 1018 .macro vector_stub, name, mode, correction=0 1019 .align 5 1020 1021vector_\name: 1022 .if \correction 1023 sub lr, lr, #\correction 1024 .endif 1025 1026 @ 1027 @ Save r0, lr_<exception> (parent PC) and spsr_<exception> 1028 @ (parent CPSR) 1029 @ 1030 stmia sp, {r0, lr} @ save r0, lr 1031 mrs lr, spsr 1032 str lr, [sp, #8] @ save spsr 1033 1034 @ 1035 @ Prepare for SVC32 mode. IRQs remain disabled. 1036 @ 1037 mrs r0, cpsr 1038 eor r0, r0, #(\mode ^ SVC_MODE) 1039 msr spsr_cxsf, r0 1040 1041 @ 1042 @ the branch table must immediately follow this code 1043 @ 1044 and lr, lr, #0x0f 1045 mov r0, sp 1046 ldr lr, [pc, lr, lsl #2] 1047 movs pc, lr @ branch to handler in SVC mode 1048ENDPROC(vector_\name) 1049 .endm 1050 1051 .globl __stubs_start 1052__stubs_start: 1053/* 1054 * Interrupt dispatcher 1055 */ 1056 vector_stub irq, IRQ_MODE, 4 1057 1058 .long __irq_usr @ 0 (USR_26 / USR_32) 1059 .long __irq_invalid @ 1 (FIQ_26 / FIQ_32) 1060 .long __irq_invalid @ 2 (IRQ_26 / IRQ_32) 1061 .long __irq_svc @ 3 (SVC_26 / SVC_32) 1062 .long __irq_invalid @ 4 1063 .long __irq_invalid @ 5 1064 .long __irq_invalid @ 6 1065 .long __irq_invalid @ 7 1066 .long __irq_invalid @ 8 1067 .long __irq_invalid @ 9 1068 .long __irq_invalid @ a 1069 .long __irq_invalid @ b 1070 .long __irq_invalid @ c 1071 .long __irq_invalid @ d 1072 .long __irq_invalid @ e 1073 .long __irq_invalid @ f 1074 1075/* 1076 * Data abort dispatcher 1077 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 1078 */ 1079 vector_stub dabt, ABT_MODE, 8 1080 1081 .long __dabt_usr @ 0 (USR_26 / USR_32) 1082 .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32) 1083 .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32) 1084 .long __dabt_svc @ 3 (SVC_26 / SVC_32) 1085 .long __dabt_invalid @ 4 1086 .long __dabt_invalid @ 5 1087 .long __dabt_invalid @ 6 1088 .long __dabt_invalid @ 7 1089 .long __dabt_invalid @ 8 1090 .long __dabt_invalid @ 9 1091 .long __dabt_invalid @ a 1092 .long __dabt_invalid @ b 1093 .long __dabt_invalid @ c 1094 .long __dabt_invalid @ d 1095 .long __dabt_invalid @ e 1096 .long __dabt_invalid @ f 1097 1098/* 1099 * Prefetch abort dispatcher 1100 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 1101 */ 1102 vector_stub pabt, ABT_MODE, 4 1103 1104 .long __pabt_usr @ 0 (USR_26 / USR_32) 1105 .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32) 1106 .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32) 1107 .long __pabt_svc @ 3 (SVC_26 / SVC_32) 1108 .long __pabt_invalid @ 4 1109 .long __pabt_invalid @ 5 1110 .long __pabt_invalid @ 6 1111 .long __pabt_invalid @ 7 1112 .long __pabt_invalid @ 8 1113 .long __pabt_invalid @ 9 1114 .long __pabt_invalid @ a 1115 .long __pabt_invalid @ b 1116 .long __pabt_invalid @ c 1117 .long __pabt_invalid @ d 1118 .long __pabt_invalid @ e 1119 .long __pabt_invalid @ f 1120 1121/* 1122 * Undef instr entry dispatcher 1123 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 1124 */ 1125 vector_stub und, UND_MODE 1126 1127 .long __und_usr @ 0 (USR_26 / USR_32) 1128 .long __und_invalid @ 1 (FIQ_26 / FIQ_32) 1129 .long __und_invalid @ 2 (IRQ_26 / IRQ_32) 1130 .long __und_svc @ 3 (SVC_26 / SVC_32) 1131 .long __und_invalid @ 4 1132 .long __und_invalid @ 5 1133 .long __und_invalid @ 6 1134 .long __und_invalid @ 7 1135 .long __und_invalid @ 8 1136 .long __und_invalid @ 9 1137 .long __und_invalid @ a 1138 .long __und_invalid @ b 1139 .long __und_invalid @ c 1140 .long __und_invalid @ d 1141 .long __und_invalid @ e 1142 .long __und_invalid @ f 1143 1144 .align 5 1145 1146/*============================================================================= 1147 * Undefined FIQs 1148 *----------------------------------------------------------------------------- 1149 * Enter in FIQ mode, spsr = ANY CPSR, lr = ANY PC 1150 * MUST PRESERVE SVC SPSR, but need to switch to SVC mode to show our msg. 1151 * Basically to switch modes, we *HAVE* to clobber one register... brain 1152 * damage alert! I don't think that we can execute any code in here in any 1153 * other mode than FIQ... Ok you can switch to another mode, but you can't 1154 * get out of that mode without clobbering one register. 1155 */ 1156vector_fiq: 1157 disable_fiq 1158 subs pc, lr, #4 1159 1160/*============================================================================= 1161 * Address exception handler 1162 *----------------------------------------------------------------------------- 1163 * These aren't too critical. 1164 * (they're not supposed to happen, and won't happen in 32-bit data mode). 1165 */ 1166 1167vector_addrexcptn: 1168 b vector_addrexcptn 1169 1170/* 1171 * We group all the following data together to optimise 1172 * for CPUs with separate I & D caches. 1173 */ 1174 .align 5 1175 1176.LCvswi: 1177 .word vector_swi 1178 1179 .globl __stubs_end 1180__stubs_end: 1181 1182 .equ stubs_offset, __vectors_start + 0x200 - __stubs_start 1183 1184 .globl __vectors_start 1185__vectors_start: 1186 swi SYS_ERROR0 1187 b vector_und + stubs_offset 1188 ldr pc, .LCvswi + stubs_offset 1189 b vector_pabt + stubs_offset 1190 b vector_dabt + stubs_offset 1191 b vector_addrexcptn + stubs_offset 1192 b vector_irq + stubs_offset 1193 b vector_fiq + stubs_offset 1194 1195 .globl __vectors_end 1196__vectors_end: 1197 1198 .data 1199 1200 .globl cr_alignment 1201 .globl cr_no_alignment 1202cr_alignment: 1203 .space 4 1204cr_no_alignment: 1205 .space 4 1206