1/* 2 * linux/arch/arm/kernel/entry-armv.S 3 * 4 * Copyright (C) 1996,1997,1998 Russell King. 5 * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk) 6 * nommu support by Hyok S. Choi (hyok.choi@samsung.com) 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * Low-level vector interface routines 13 * 14 * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction 15 * that causes it to save wrong values... Be aware! 16 */ 17 18#include <asm/memory.h> 19#include <asm/glue.h> 20#include <asm/vfpmacros.h> 21#include <mach/entry-macro.S> 22#include <asm/thread_notify.h> 23 24#include "entry-header.S" 25 26/* 27 * Interrupt handling. Preserves r7, r8, r9 28 */ 29 .macro irq_handler 30 get_irqnr_preamble r5, lr 311: get_irqnr_and_base r0, r6, r5, lr 32 movne r1, sp 33 @ 34 @ routine called with r0 = irq number, r1 = struct pt_regs * 35 @ 36 adrne lr, 1b 37 bne asm_do_IRQ 38 39#ifdef CONFIG_SMP 40 /* 41 * XXX 42 * 43 * this macro assumes that irqstat (r6) and base (r5) are 44 * preserved from get_irqnr_and_base above 45 */ 46 test_for_ipi r0, r6, r5, lr 47 movne r0, sp 48 adrne lr, 1b 49 bne do_IPI 50 51#ifdef CONFIG_LOCAL_TIMERS 52 test_for_ltirq r0, r6, r5, lr 53 movne r0, sp 54 adrne lr, 1b 55 bne do_local_timer 56#endif 57#endif 58 59 .endm 60 61#ifdef CONFIG_KPROBES 62 .section .kprobes.text,"ax",%progbits 63#else 64 .text 65#endif 66 67/* 68 * Invalid mode handlers 69 */ 70 .macro inv_entry, reason 71 sub sp, sp, #S_FRAME_SIZE 72 stmib sp, {r1 - lr} 73 mov r1, #\reason 74 .endm 75 76__pabt_invalid: 77 inv_entry BAD_PREFETCH 78 b common_invalid 79ENDPROC(__pabt_invalid) 80 81__dabt_invalid: 82 inv_entry BAD_DATA 83 b common_invalid 84ENDPROC(__dabt_invalid) 85 86__irq_invalid: 87 inv_entry BAD_IRQ 88 b common_invalid 89ENDPROC(__irq_invalid) 90 91__und_invalid: 92 inv_entry BAD_UNDEFINSTR 93 94 @ 95 @ XXX fall through to common_invalid 96 @ 97 98@ 99@ common_invalid - generic code for failed exception (re-entrant version of handlers) 100@ 101common_invalid: 102 zero_fp 103 104 ldmia r0, {r4 - r6} 105 add r0, sp, #S_PC @ here for interlock avoidance 106 mov r7, #-1 @ "" "" "" "" 107 str r4, [sp] @ save preserved r0 108 stmia r0, {r5 - r7} @ lr_<exception>, 109 @ cpsr_<exception>, "old_r0" 110 111 mov r0, sp 112 b bad_mode 113ENDPROC(__und_invalid) 114 115/* 116 * SVC mode handlers 117 */ 118 119#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) 120#define SPFIX(code...) code 121#else 122#define SPFIX(code...) 123#endif 124 125 .macro svc_entry, stack_hole=0 126 sub sp, sp, #(S_FRAME_SIZE + \stack_hole) 127 SPFIX( tst sp, #4 ) 128 SPFIX( bicne sp, sp, #4 ) 129 stmib sp, {r1 - r12} 130 131 ldmia r0, {r1 - r3} 132 add r5, sp, #S_SP @ here for interlock avoidance 133 mov r4, #-1 @ "" "" "" "" 134 add r0, sp, #(S_FRAME_SIZE + \stack_hole) 135 SPFIX( addne r0, r0, #4 ) 136 str r1, [sp] @ save the "real" r0 copied 137 @ from the exception stack 138 139 mov r1, lr 140 141 @ 142 @ We are now ready to fill in the remaining blanks on the stack: 143 @ 144 @ r0 - sp_svc 145 @ r1 - lr_svc 146 @ r2 - lr_<exception>, already fixed up for correct return/restart 147 @ r3 - spsr_<exception> 148 @ r4 - orig_r0 (see pt_regs definition in ptrace.h) 149 @ 150 stmia r5, {r0 - r4} 151 .endm 152 153 .align 5 154__dabt_svc: 155 svc_entry 156 157 @ 158 @ get ready to re-enable interrupts if appropriate 159 @ 160 mrs r9, cpsr 161 tst r3, #PSR_I_BIT 162 biceq r9, r9, #PSR_I_BIT 163 164 @ 165 @ Call the processor-specific abort handler: 166 @ 167 @ r2 - aborted context pc 168 @ r3 - aborted context cpsr 169 @ 170 @ The abort handler must return the aborted address in r0, and 171 @ the fault status register in r1. r9 must be preserved. 172 @ 173#ifdef MULTI_DABORT 174 ldr r4, .LCprocfns 175 mov lr, pc 176 ldr pc, [r4, #PROCESSOR_DABT_FUNC] 177#else 178 bl CPU_DABORT_HANDLER 179#endif 180 181 @ 182 @ set desired IRQ state, then call main handler 183 @ 184 msr cpsr_c, r9 185 mov r2, sp 186 bl do_DataAbort 187 188 @ 189 @ IRQs off again before pulling preserved data off the stack 190 @ 191 disable_irq 192 193 @ 194 @ restore SPSR and restart the instruction 195 @ 196 ldr r0, [sp, #S_PSR] 197 msr spsr_cxsf, r0 198 ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr 199ENDPROC(__dabt_svc) 200 201 .align 5 202__irq_svc: 203 svc_entry 204 205#ifdef CONFIG_TRACE_IRQFLAGS 206 bl trace_hardirqs_off 207#endif 208#ifdef CONFIG_PREEMPT 209 get_thread_info tsk 210 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count 211 add r7, r8, #1 @ increment it 212 str r7, [tsk, #TI_PREEMPT] 213#endif 214 215 irq_handler 216#ifdef CONFIG_PREEMPT 217 str r8, [tsk, #TI_PREEMPT] @ restore preempt count 218 ldr r0, [tsk, #TI_FLAGS] @ get flags 219 teq r8, #0 @ if preempt count != 0 220 movne r0, #0 @ force flags to 0 221 tst r0, #_TIF_NEED_RESCHED 222 blne svc_preempt 223#endif 224 ldr r0, [sp, #S_PSR] @ irqs are already disabled 225 msr spsr_cxsf, r0 226#ifdef CONFIG_TRACE_IRQFLAGS 227 tst r0, #PSR_I_BIT 228 bleq trace_hardirqs_on 229#endif 230 ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr 231ENDPROC(__irq_svc) 232 233 .ltorg 234 235#ifdef CONFIG_PREEMPT 236svc_preempt: 237 mov r8, lr 2381: bl preempt_schedule_irq @ irq en/disable is done inside 239 ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS 240 tst r0, #_TIF_NEED_RESCHED 241 moveq pc, r8 @ go again 242 b 1b 243#endif 244 245 .align 5 246__und_svc: 247#ifdef CONFIG_KPROBES 248 @ If a kprobe is about to simulate a "stmdb sp..." instruction, 249 @ it obviously needs free stack space which then will belong to 250 @ the saved context. 251 svc_entry 64 252#else 253 svc_entry 254#endif 255 256 @ 257 @ call emulation code, which returns using r9 if it has emulated 258 @ the instruction, or the more conventional lr if we are to treat 259 @ this as a real undefined instruction 260 @ 261 @ r0 - instruction 262 @ 263 ldr r0, [r2, #-4] 264 adr r9, 1f 265 bl call_fpe 266 267 mov r0, sp @ struct pt_regs *regs 268 bl do_undefinstr 269 270 @ 271 @ IRQs off again before pulling preserved data off the stack 272 @ 2731: disable_irq 274 275 @ 276 @ restore SPSR and restart the instruction 277 @ 278 ldr lr, [sp, #S_PSR] @ Get SVC cpsr 279 msr spsr_cxsf, lr 280 ldmia sp, {r0 - pc}^ @ Restore SVC registers 281ENDPROC(__und_svc) 282 283 .align 5 284__pabt_svc: 285 svc_entry 286 287 @ 288 @ re-enable interrupts if appropriate 289 @ 290 mrs r9, cpsr 291 tst r3, #PSR_I_BIT 292 biceq r9, r9, #PSR_I_BIT 293 294 @ 295 @ set args, then call main handler 296 @ 297 @ r0 - address of faulting instruction 298 @ r1 - pointer to registers on stack 299 @ 300#ifdef MULTI_PABORT 301 mov r0, r2 @ pass address of aborted instruction. 302 ldr r4, .LCprocfns 303 mov lr, pc 304 ldr pc, [r4, #PROCESSOR_PABT_FUNC] 305#else 306 CPU_PABORT_HANDLER(r0, r2) 307#endif 308 msr cpsr_c, r9 @ Maybe enable interrupts 309 mov r1, sp @ regs 310 bl do_PrefetchAbort @ call abort handler 311 312 @ 313 @ IRQs off again before pulling preserved data off the stack 314 @ 315 disable_irq 316 317 @ 318 @ restore SPSR and restart the instruction 319 @ 320 ldr r0, [sp, #S_PSR] 321 msr spsr_cxsf, r0 322 ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr 323ENDPROC(__pabt_svc) 324 325 .align 5 326.LCcralign: 327 .word cr_alignment 328#ifdef MULTI_DABORT 329.LCprocfns: 330 .word processor 331#endif 332.LCfp: 333 .word fp_enter 334 335/* 336 * User mode handlers 337 * 338 * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE 339 */ 340 341#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7) 342#error "sizeof(struct pt_regs) must be a multiple of 8" 343#endif 344 345 .macro usr_entry 346 sub sp, sp, #S_FRAME_SIZE 347 stmib sp, {r1 - r12} 348 349 ldmia r0, {r1 - r3} 350 add r0, sp, #S_PC @ here for interlock avoidance 351 mov r4, #-1 @ "" "" "" "" 352 353 str r1, [sp] @ save the "real" r0 copied 354 @ from the exception stack 355 356 @ 357 @ We are now ready to fill in the remaining blanks on the stack: 358 @ 359 @ r2 - lr_<exception>, already fixed up for correct return/restart 360 @ r3 - spsr_<exception> 361 @ r4 - orig_r0 (see pt_regs definition in ptrace.h) 362 @ 363 @ Also, separately save sp_usr and lr_usr 364 @ 365 stmia r0, {r2 - r4} 366 stmdb r0, {sp, lr}^ 367 368 @ 369 @ Enable the alignment trap while in kernel mode 370 @ 371 alignment_trap r0 372 373 @ 374 @ Clear FP to mark the first stack frame 375 @ 376 zero_fp 377 .endm 378 379 .macro kuser_cmpxchg_check 380#if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) 381#ifndef CONFIG_MMU 382#warning "NPTL on non MMU needs fixing" 383#else 384 @ Make sure our user space atomic helper is restarted 385 @ if it was interrupted in a critical region. Here we 386 @ perform a quick test inline since it should be false 387 @ 99.9999% of the time. The rest is done out of line. 388 cmp r2, #TASK_SIZE 389 blhs kuser_cmpxchg_fixup 390#endif 391#endif 392 .endm 393 394 .align 5 395__dabt_usr: 396 usr_entry 397 kuser_cmpxchg_check 398 399 @ 400 @ Call the processor-specific abort handler: 401 @ 402 @ r2 - aborted context pc 403 @ r3 - aborted context cpsr 404 @ 405 @ The abort handler must return the aborted address in r0, and 406 @ the fault status register in r1. 407 @ 408#ifdef MULTI_DABORT 409 ldr r4, .LCprocfns 410 mov lr, pc 411 ldr pc, [r4, #PROCESSOR_DABT_FUNC] 412#else 413 bl CPU_DABORT_HANDLER 414#endif 415 416 @ 417 @ IRQs on, then call the main handler 418 @ 419 enable_irq 420 mov r2, sp 421 adr lr, ret_from_exception 422 b do_DataAbort 423ENDPROC(__dabt_usr) 424 425 .align 5 426__irq_usr: 427 usr_entry 428 kuser_cmpxchg_check 429 430#ifdef CONFIG_TRACE_IRQFLAGS 431 bl trace_hardirqs_off 432#endif 433 get_thread_info tsk 434#ifdef CONFIG_PREEMPT 435 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count 436 add r7, r8, #1 @ increment it 437 str r7, [tsk, #TI_PREEMPT] 438#endif 439 440 irq_handler 441#ifdef CONFIG_PREEMPT 442 ldr r0, [tsk, #TI_PREEMPT] 443 str r8, [tsk, #TI_PREEMPT] 444 teq r0, r7 445 strne r0, [r0, -r0] 446#endif 447#ifdef CONFIG_TRACE_IRQFLAGS 448 bl trace_hardirqs_on 449#endif 450 451 mov why, #0 452 b ret_to_user 453ENDPROC(__irq_usr) 454 455 .ltorg 456 457 .align 5 458__und_usr: 459 usr_entry 460 461 @ 462 @ fall through to the emulation code, which returns using r9 if 463 @ it has emulated the instruction, or the more conventional lr 464 @ if we are to treat this as a real undefined instruction 465 @ 466 @ r0 - instruction 467 @ 468 adr r9, ret_from_exception 469 adr lr, __und_usr_unknown 470 tst r3, #PSR_T_BIT @ Thumb mode? 471 subeq r4, r2, #4 @ ARM instr at LR - 4 472 subne r4, r2, #2 @ Thumb instr at LR - 2 4731: ldreqt r0, [r4] 474 beq call_fpe 475 @ Thumb instruction 476#if __LINUX_ARM_ARCH__ >= 7 4772: ldrht r5, [r4], #2 478 and r0, r5, #0xf800 @ mask bits 111x x... .... .... 479 cmp r0, #0xe800 @ 32bit instruction if xx != 0 480 blo __und_usr_unknown 4813: ldrht r0, [r4] 482 add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 483 orr r0, r0, r5, lsl #16 484#else 485 b __und_usr_unknown 486#endif 487ENDPROC(__und_usr) 488 489 @ 490 @ fallthrough to call_fpe 491 @ 492 493/* 494 * The out of line fixup for the ldrt above. 495 */ 496 .section .fixup, "ax" 4974: mov pc, r9 498 .previous 499 .section __ex_table,"a" 500 .long 1b, 4b 501#if __LINUX_ARM_ARCH__ >= 7 502 .long 2b, 4b 503 .long 3b, 4b 504#endif 505 .previous 506 507/* 508 * Check whether the instruction is a co-processor instruction. 509 * If yes, we need to call the relevant co-processor handler. 510 * 511 * Note that we don't do a full check here for the co-processor 512 * instructions; all instructions with bit 27 set are well 513 * defined. The only instructions that should fault are the 514 * co-processor instructions. However, we have to watch out 515 * for the ARM6/ARM7 SWI bug. 516 * 517 * NEON is a special case that has to be handled here. Not all 518 * NEON instructions are co-processor instructions, so we have 519 * to make a special case of checking for them. Plus, there's 520 * five groups of them, so we have a table of mask/opcode pairs 521 * to check against, and if any match then we branch off into the 522 * NEON handler code. 523 * 524 * Emulators may wish to make use of the following registers: 525 * r0 = instruction opcode. 526 * r2 = PC+4 527 * r9 = normal "successful" return address 528 * r10 = this threads thread_info structure. 529 * lr = unrecognised instruction return address 530 */ 531 @ 532 @ Fall-through from Thumb-2 __und_usr 533 @ 534#ifdef CONFIG_NEON 535 adr r6, .LCneon_thumb_opcodes 536 b 2f 537#endif 538call_fpe: 539#ifdef CONFIG_NEON 540 adr r6, .LCneon_arm_opcodes 5412: 542 ldr r7, [r6], #4 @ mask value 543 cmp r7, #0 @ end mask? 544 beq 1f 545 and r8, r0, r7 546 ldr r7, [r6], #4 @ opcode bits matching in mask 547 cmp r8, r7 @ NEON instruction? 548 bne 2b 549 get_thread_info r10 550 mov r7, #1 551 strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used 552 strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used 553 b do_vfp @ let VFP handler handle this 5541: 555#endif 556 tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 557 tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 558#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710) 559 and r8, r0, #0x0f000000 @ mask out op-code bits 560 teqne r8, #0x0f000000 @ SWI (ARM6/7 bug)? 561#endif 562 moveq pc, lr 563 get_thread_info r10 @ get current thread 564 and r8, r0, #0x00000f00 @ mask out CP number 565 mov r7, #1 566 add r6, r10, #TI_USED_CP 567 strb r7, [r6, r8, lsr #8] @ set appropriate used_cp[] 568#ifdef CONFIG_IWMMXT 569 @ Test if we need to give access to iWMMXt coprocessors 570 ldr r5, [r10, #TI_FLAGS] 571 rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only 572 movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1) 573 bcs iwmmxt_task_enable 574#endif 575 add pc, pc, r8, lsr #6 576 mov r0, r0 577 578 mov pc, lr @ CP#0 579 b do_fpe @ CP#1 (FPE) 580 b do_fpe @ CP#2 (FPE) 581 mov pc, lr @ CP#3 582#ifdef CONFIG_CRUNCH 583 b crunch_task_enable @ CP#4 (MaverickCrunch) 584 b crunch_task_enable @ CP#5 (MaverickCrunch) 585 b crunch_task_enable @ CP#6 (MaverickCrunch) 586#else 587 mov pc, lr @ CP#4 588 mov pc, lr @ CP#5 589 mov pc, lr @ CP#6 590#endif 591 mov pc, lr @ CP#7 592 mov pc, lr @ CP#8 593 mov pc, lr @ CP#9 594#ifdef CONFIG_VFP 595 b do_vfp @ CP#10 (VFP) 596 b do_vfp @ CP#11 (VFP) 597#else 598 mov pc, lr @ CP#10 (VFP) 599 mov pc, lr @ CP#11 (VFP) 600#endif 601 mov pc, lr @ CP#12 602 mov pc, lr @ CP#13 603 mov pc, lr @ CP#14 (Debug) 604 mov pc, lr @ CP#15 (Control) 605 606#ifdef CONFIG_NEON 607 .align 6 608 609.LCneon_arm_opcodes: 610 .word 0xfe000000 @ mask 611 .word 0xf2000000 @ opcode 612 613 .word 0xff100000 @ mask 614 .word 0xf4000000 @ opcode 615 616 .word 0x00000000 @ mask 617 .word 0x00000000 @ opcode 618 619.LCneon_thumb_opcodes: 620 .word 0xef000000 @ mask 621 .word 0xef000000 @ opcode 622 623 .word 0xff100000 @ mask 624 .word 0xf9000000 @ opcode 625 626 .word 0x00000000 @ mask 627 .word 0x00000000 @ opcode 628#endif 629 630do_fpe: 631 enable_irq 632 ldr r4, .LCfp 633 add r10, r10, #TI_FPSTATE @ r10 = workspace 634 ldr pc, [r4] @ Call FP module USR entry point 635 636/* 637 * The FP module is called with these registers set: 638 * r0 = instruction 639 * r2 = PC+4 640 * r9 = normal "successful" return address 641 * r10 = FP workspace 642 * lr = unrecognised FP instruction return address 643 */ 644 645 .data 646ENTRY(fp_enter) 647 .word no_fp 648 .previous 649 650no_fp: mov pc, lr 651 652__und_usr_unknown: 653 mov r0, sp 654 adr lr, ret_from_exception 655 b do_undefinstr 656ENDPROC(__und_usr_unknown) 657 658 .align 5 659__pabt_usr: 660 usr_entry 661 662#ifdef MULTI_PABORT 663 mov r0, r2 @ pass address of aborted instruction. 664 ldr r4, .LCprocfns 665 mov lr, pc 666 ldr pc, [r4, #PROCESSOR_PABT_FUNC] 667#else 668 CPU_PABORT_HANDLER(r0, r2) 669#endif 670 enable_irq @ Enable interrupts 671 mov r1, sp @ regs 672 bl do_PrefetchAbort @ call abort handler 673 /* fall through */ 674/* 675 * This is the return code to user mode for abort handlers 676 */ 677ENTRY(ret_from_exception) 678 get_thread_info tsk 679 mov why, #0 680 b ret_to_user 681ENDPROC(__pabt_usr) 682ENDPROC(ret_from_exception) 683 684/* 685 * Register switch for ARMv3 and ARMv4 processors 686 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info 687 * previous and next are guaranteed not to be the same. 688 */ 689ENTRY(__switch_to) 690 add ip, r1, #TI_CPU_SAVE 691 ldr r3, [r2, #TI_TP_VALUE] 692 stmia ip!, {r4 - sl, fp, sp, lr} @ Store most regs on stack 693#ifdef CONFIG_MMU 694 ldr r6, [r2, #TI_CPU_DOMAIN] 695#endif 696#if __LINUX_ARM_ARCH__ >= 6 697#ifdef CONFIG_CPU_32v6K 698 clrex 699#else 700 strex r5, r4, [ip] @ Clear exclusive monitor 701#endif 702#endif 703#if defined(CONFIG_HAS_TLS_REG) 704 mcr p15, 0, r3, c13, c0, 3 @ set TLS register 705#elif !defined(CONFIG_TLS_REG_EMUL) 706 mov r4, #0xffff0fff 707 str r3, [r4, #-15] @ TLS val at 0xffff0ff0 708#endif 709#ifdef CONFIG_MMU 710 mcr p15, 0, r6, c3, c0, 0 @ Set domain register 711#endif 712 mov r5, r0 713 add r4, r2, #TI_CPU_SAVE 714 ldr r0, =thread_notify_head 715 mov r1, #THREAD_NOTIFY_SWITCH 716 bl atomic_notifier_call_chain 717 mov r0, r5 718 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously 719ENDPROC(__switch_to) 720 721 __INIT 722 723/* 724 * User helpers. 725 * 726 * These are segment of kernel provided user code reachable from user space 727 * at a fixed address in kernel memory. This is used to provide user space 728 * with some operations which require kernel help because of unimplemented 729 * native feature and/or instructions in many ARM CPUs. The idea is for 730 * this code to be executed directly in user mode for best efficiency but 731 * which is too intimate with the kernel counter part to be left to user 732 * libraries. In fact this code might even differ from one CPU to another 733 * depending on the available instruction set and restrictions like on 734 * SMP systems. In other words, the kernel reserves the right to change 735 * this code as needed without warning. Only the entry points and their 736 * results are guaranteed to be stable. 737 * 738 * Each segment is 32-byte aligned and will be moved to the top of the high 739 * vector page. New segments (if ever needed) must be added in front of 740 * existing ones. This mechanism should be used only for things that are 741 * really small and justified, and not be abused freely. 742 * 743 * User space is expected to implement those things inline when optimizing 744 * for a processor that has the necessary native support, but only if such 745 * resulting binaries are already to be incompatible with earlier ARM 746 * processors due to the use of unsupported instructions other than what 747 * is provided here. In other words don't make binaries unable to run on 748 * earlier processors just for the sake of not using these kernel helpers 749 * if your compiled code is not going to use the new instructions for other 750 * purpose. 751 */ 752 753 .macro usr_ret, reg 754#ifdef CONFIG_ARM_THUMB 755 bx \reg 756#else 757 mov pc, \reg 758#endif 759 .endm 760 761 .align 5 762 .globl __kuser_helper_start 763__kuser_helper_start: 764 765/* 766 * Reference prototype: 767 * 768 * void __kernel_memory_barrier(void) 769 * 770 * Input: 771 * 772 * lr = return address 773 * 774 * Output: 775 * 776 * none 777 * 778 * Clobbered: 779 * 780 * none 781 * 782 * Definition and user space usage example: 783 * 784 * typedef void (__kernel_dmb_t)(void); 785 * #define __kernel_dmb (*(__kernel_dmb_t *)0xffff0fa0) 786 * 787 * Apply any needed memory barrier to preserve consistency with data modified 788 * manually and __kuser_cmpxchg usage. 789 * 790 * This could be used as follows: 791 * 792 * #define __kernel_dmb() \ 793 * asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \ 794 * : : : "r0", "lr","cc" ) 795 */ 796 797__kuser_memory_barrier: @ 0xffff0fa0 798 799#if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_SMP) 800 mcr p15, 0, r0, c7, c10, 5 @ dmb 801#endif 802 usr_ret lr 803 804 .align 5 805 806/* 807 * Reference prototype: 808 * 809 * int __kernel_cmpxchg(int oldval, int newval, int *ptr) 810 * 811 * Input: 812 * 813 * r0 = oldval 814 * r1 = newval 815 * r2 = ptr 816 * lr = return address 817 * 818 * Output: 819 * 820 * r0 = returned value (zero or non-zero) 821 * C flag = set if r0 == 0, clear if r0 != 0 822 * 823 * Clobbered: 824 * 825 * r3, ip, flags 826 * 827 * Definition and user space usage example: 828 * 829 * typedef int (__kernel_cmpxchg_t)(int oldval, int newval, int *ptr); 830 * #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *)0xffff0fc0) 831 * 832 * Atomically store newval in *ptr if *ptr is equal to oldval for user space. 833 * Return zero if *ptr was changed or non-zero if no exchange happened. 834 * The C flag is also set if *ptr was changed to allow for assembly 835 * optimization in the calling code. 836 * 837 * Notes: 838 * 839 * - This routine already includes memory barriers as needed. 840 * 841 * For example, a user space atomic_add implementation could look like this: 842 * 843 * #define atomic_add(ptr, val) \ 844 * ({ register unsigned int *__ptr asm("r2") = (ptr); \ 845 * register unsigned int __result asm("r1"); \ 846 * asm volatile ( \ 847 * "1: @ atomic_add\n\t" \ 848 * "ldr r0, [r2]\n\t" \ 849 * "mov r3, #0xffff0fff\n\t" \ 850 * "add lr, pc, #4\n\t" \ 851 * "add r1, r0, %2\n\t" \ 852 * "add pc, r3, #(0xffff0fc0 - 0xffff0fff)\n\t" \ 853 * "bcc 1b" \ 854 * : "=&r" (__result) \ 855 * : "r" (__ptr), "rIL" (val) \ 856 * : "r0","r3","ip","lr","cc","memory" ); \ 857 * __result; }) 858 */ 859 860__kuser_cmpxchg: @ 0xffff0fc0 861 862#if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) 863 864 /* 865 * Poor you. No fast solution possible... 866 * The kernel itself must perform the operation. 867 * A special ghost syscall is used for that (see traps.c). 868 */ 869 stmfd sp!, {r7, lr} 870 mov r7, #0xff00 @ 0xfff0 into r7 for EABI 871 orr r7, r7, #0xf0 872 swi #0x9ffff0 873 ldmfd sp!, {r7, pc} 874 875#elif __LINUX_ARM_ARCH__ < 6 876 877#ifdef CONFIG_MMU 878 879 /* 880 * The only thing that can break atomicity in this cmpxchg 881 * implementation is either an IRQ or a data abort exception 882 * causing another process/thread to be scheduled in the middle 883 * of the critical sequence. To prevent this, code is added to 884 * the IRQ and data abort exception handlers to set the pc back 885 * to the beginning of the critical section if it is found to be 886 * within that critical section (see kuser_cmpxchg_fixup). 887 */ 8881: ldr r3, [r2] @ load current val 889 subs r3, r3, r0 @ compare with oldval 8902: streq r1, [r2] @ store newval if eq 891 rsbs r0, r3, #0 @ set return val and C flag 892 usr_ret lr 893 894 .text 895kuser_cmpxchg_fixup: 896 @ Called from kuser_cmpxchg_check macro. 897 @ r2 = address of interrupted insn (must be preserved). 898 @ sp = saved regs. r7 and r8 are clobbered. 899 @ 1b = first critical insn, 2b = last critical insn. 900 @ If r2 >= 1b and r2 <= 2b then saved pc_usr is set to 1b. 901 mov r7, #0xffff0fff 902 sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) 903 subs r8, r2, r7 904 rsbcss r8, r8, #(2b - 1b) 905 strcs r7, [sp, #S_PC] 906 mov pc, lr 907 .previous 908 909#else 910#warning "NPTL on non MMU needs fixing" 911 mov r0, #-1 912 adds r0, r0, #0 913 usr_ret lr 914#endif 915 916#else 917 918#ifdef CONFIG_SMP 919 mcr p15, 0, r0, c7, c10, 5 @ dmb 920#endif 9211: ldrex r3, [r2] 922 subs r3, r3, r0 923 strexeq r3, r1, [r2] 924 teqeq r3, #1 925 beq 1b 926 rsbs r0, r3, #0 927 /* beware -- each __kuser slot must be 8 instructions max */ 928#ifdef CONFIG_SMP 929 b __kuser_memory_barrier 930#else 931 usr_ret lr 932#endif 933 934#endif 935 936 .align 5 937 938/* 939 * Reference prototype: 940 * 941 * int __kernel_get_tls(void) 942 * 943 * Input: 944 * 945 * lr = return address 946 * 947 * Output: 948 * 949 * r0 = TLS value 950 * 951 * Clobbered: 952 * 953 * none 954 * 955 * Definition and user space usage example: 956 * 957 * typedef int (__kernel_get_tls_t)(void); 958 * #define __kernel_get_tls (*(__kernel_get_tls_t *)0xffff0fe0) 959 * 960 * Get the TLS value as previously set via the __ARM_NR_set_tls syscall. 961 * 962 * This could be used as follows: 963 * 964 * #define __kernel_get_tls() \ 965 * ({ register unsigned int __val asm("r0"); \ 966 * asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \ 967 * : "=r" (__val) : : "lr","cc" ); \ 968 * __val; }) 969 */ 970 971__kuser_get_tls: @ 0xffff0fe0 972 973#if !defined(CONFIG_HAS_TLS_REG) && !defined(CONFIG_TLS_REG_EMUL) 974 ldr r0, [pc, #(16 - 8)] @ TLS stored at 0xffff0ff0 975#else 976 mrc p15, 0, r0, c13, c0, 3 @ read TLS register 977#endif 978 usr_ret lr 979 980 .rep 5 981 .word 0 @ pad up to __kuser_helper_version 982 .endr 983 984/* 985 * Reference declaration: 986 * 987 * extern unsigned int __kernel_helper_version; 988 * 989 * Definition and user space usage example: 990 * 991 * #define __kernel_helper_version (*(unsigned int *)0xffff0ffc) 992 * 993 * User space may read this to determine the curent number of helpers 994 * available. 995 */ 996 997__kuser_helper_version: @ 0xffff0ffc 998 .word ((__kuser_helper_end - __kuser_helper_start) >> 5) 999 1000 .globl __kuser_helper_end 1001__kuser_helper_end: 1002 1003 1004/* 1005 * Vector stubs. 1006 * 1007 * This code is copied to 0xffff0200 so we can use branches in the 1008 * vectors, rather than ldr's. Note that this code must not 1009 * exceed 0x300 bytes. 1010 * 1011 * Common stub entry macro: 1012 * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 1013 * 1014 * SP points to a minimal amount of processor-private memory, the address 1015 * of which is copied into r0 for the mode specific abort handler. 1016 */ 1017 .macro vector_stub, name, mode, correction=0 1018 .align 5 1019 1020vector_\name: 1021 .if \correction 1022 sub lr, lr, #\correction 1023 .endif 1024 1025 @ 1026 @ Save r0, lr_<exception> (parent PC) and spsr_<exception> 1027 @ (parent CPSR) 1028 @ 1029 stmia sp, {r0, lr} @ save r0, lr 1030 mrs lr, spsr 1031 str lr, [sp, #8] @ save spsr 1032 1033 @ 1034 @ Prepare for SVC32 mode. IRQs remain disabled. 1035 @ 1036 mrs r0, cpsr 1037 eor r0, r0, #(\mode ^ SVC_MODE) 1038 msr spsr_cxsf, r0 1039 1040 @ 1041 @ the branch table must immediately follow this code 1042 @ 1043 and lr, lr, #0x0f 1044 mov r0, sp 1045 ldr lr, [pc, lr, lsl #2] 1046 movs pc, lr @ branch to handler in SVC mode 1047ENDPROC(vector_\name) 1048 .endm 1049 1050 .globl __stubs_start 1051__stubs_start: 1052/* 1053 * Interrupt dispatcher 1054 */ 1055 vector_stub irq, IRQ_MODE, 4 1056 1057 .long __irq_usr @ 0 (USR_26 / USR_32) 1058 .long __irq_invalid @ 1 (FIQ_26 / FIQ_32) 1059 .long __irq_invalid @ 2 (IRQ_26 / IRQ_32) 1060 .long __irq_svc @ 3 (SVC_26 / SVC_32) 1061 .long __irq_invalid @ 4 1062 .long __irq_invalid @ 5 1063 .long __irq_invalid @ 6 1064 .long __irq_invalid @ 7 1065 .long __irq_invalid @ 8 1066 .long __irq_invalid @ 9 1067 .long __irq_invalid @ a 1068 .long __irq_invalid @ b 1069 .long __irq_invalid @ c 1070 .long __irq_invalid @ d 1071 .long __irq_invalid @ e 1072 .long __irq_invalid @ f 1073 1074/* 1075 * Data abort dispatcher 1076 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 1077 */ 1078 vector_stub dabt, ABT_MODE, 8 1079 1080 .long __dabt_usr @ 0 (USR_26 / USR_32) 1081 .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32) 1082 .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32) 1083 .long __dabt_svc @ 3 (SVC_26 / SVC_32) 1084 .long __dabt_invalid @ 4 1085 .long __dabt_invalid @ 5 1086 .long __dabt_invalid @ 6 1087 .long __dabt_invalid @ 7 1088 .long __dabt_invalid @ 8 1089 .long __dabt_invalid @ 9 1090 .long __dabt_invalid @ a 1091 .long __dabt_invalid @ b 1092 .long __dabt_invalid @ c 1093 .long __dabt_invalid @ d 1094 .long __dabt_invalid @ e 1095 .long __dabt_invalid @ f 1096 1097/* 1098 * Prefetch abort dispatcher 1099 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC 1100 */ 1101 vector_stub pabt, ABT_MODE, 4 1102 1103 .long __pabt_usr @ 0 (USR_26 / USR_32) 1104 .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32) 1105 .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32) 1106 .long __pabt_svc @ 3 (SVC_26 / SVC_32) 1107 .long __pabt_invalid @ 4 1108 .long __pabt_invalid @ 5 1109 .long __pabt_invalid @ 6 1110 .long __pabt_invalid @ 7 1111 .long __pabt_invalid @ 8 1112 .long __pabt_invalid @ 9 1113 .long __pabt_invalid @ a 1114 .long __pabt_invalid @ b 1115 .long __pabt_invalid @ c 1116 .long __pabt_invalid @ d 1117 .long __pabt_invalid @ e 1118 .long __pabt_invalid @ f 1119 1120/* 1121 * Undef instr entry dispatcher 1122 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC 1123 */ 1124 vector_stub und, UND_MODE 1125 1126 .long __und_usr @ 0 (USR_26 / USR_32) 1127 .long __und_invalid @ 1 (FIQ_26 / FIQ_32) 1128 .long __und_invalid @ 2 (IRQ_26 / IRQ_32) 1129 .long __und_svc @ 3 (SVC_26 / SVC_32) 1130 .long __und_invalid @ 4 1131 .long __und_invalid @ 5 1132 .long __und_invalid @ 6 1133 .long __und_invalid @ 7 1134 .long __und_invalid @ 8 1135 .long __und_invalid @ 9 1136 .long __und_invalid @ a 1137 .long __und_invalid @ b 1138 .long __und_invalid @ c 1139 .long __und_invalid @ d 1140 .long __und_invalid @ e 1141 .long __und_invalid @ f 1142 1143 .align 5 1144 1145/*============================================================================= 1146 * Undefined FIQs 1147 *----------------------------------------------------------------------------- 1148 * Enter in FIQ mode, spsr = ANY CPSR, lr = ANY PC 1149 * MUST PRESERVE SVC SPSR, but need to switch to SVC mode to show our msg. 1150 * Basically to switch modes, we *HAVE* to clobber one register... brain 1151 * damage alert! I don't think that we can execute any code in here in any 1152 * other mode than FIQ... Ok you can switch to another mode, but you can't 1153 * get out of that mode without clobbering one register. 1154 */ 1155vector_fiq: 1156 disable_fiq 1157 subs pc, lr, #4 1158 1159/*============================================================================= 1160 * Address exception handler 1161 *----------------------------------------------------------------------------- 1162 * These aren't too critical. 1163 * (they're not supposed to happen, and won't happen in 32-bit data mode). 1164 */ 1165 1166vector_addrexcptn: 1167 b vector_addrexcptn 1168 1169/* 1170 * We group all the following data together to optimise 1171 * for CPUs with separate I & D caches. 1172 */ 1173 .align 5 1174 1175.LCvswi: 1176 .word vector_swi 1177 1178 .globl __stubs_end 1179__stubs_end: 1180 1181 .equ stubs_offset, __vectors_start + 0x200 - __stubs_start 1182 1183 .globl __vectors_start 1184__vectors_start: 1185 swi SYS_ERROR0 1186 b vector_und + stubs_offset 1187 ldr pc, .LCvswi + stubs_offset 1188 b vector_pabt + stubs_offset 1189 b vector_dabt + stubs_offset 1190 b vector_addrexcptn + stubs_offset 1191 b vector_irq + stubs_offset 1192 b vector_fiq + stubs_offset 1193 1194 .globl __vectors_end 1195__vectors_end: 1196 1197 .data 1198 1199 .globl cr_alignment 1200 .globl cr_no_alignment 1201cr_alignment: 1202 .space 4 1203cr_no_alignment: 1204 .space 4 1205