Lines Matching refs:regs

40 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)  in __enter_from_kernel_mode()  argument
42 regs->exit_rcu = false; in __enter_from_kernel_mode()
49 regs->exit_rcu = true; in __enter_from_kernel_mode()
58 static void noinstr enter_from_kernel_mode(struct pt_regs *regs) in enter_from_kernel_mode() argument
60 __enter_from_kernel_mode(regs); in enter_from_kernel_mode()
73 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) in __exit_to_kernel_mode() argument
77 if (interrupts_enabled(regs)) { in __exit_to_kernel_mode()
78 if (regs->exit_rcu) { in __exit_to_kernel_mode()
88 if (regs->exit_rcu) in __exit_to_kernel_mode()
93 static void noinstr exit_to_kernel_mode(struct pt_regs *regs) in exit_to_kernel_mode() argument
96 __exit_to_kernel_mode(regs); in exit_to_kernel_mode()
113 static __always_inline void enter_from_user_mode(struct pt_regs *regs) in enter_from_user_mode() argument
131 static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) in do_notify_resume() argument
140 uprobe_notify_resume(regs); in do_notify_resume()
152 do_signal(regs); in do_notify_resume()
155 resume_user_mode_work(regs); in do_notify_resume()
165 static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs) in exit_to_user_mode_prepare() argument
173 do_notify_resume(regs, flags); in exit_to_user_mode_prepare()
180 static __always_inline void exit_to_user_mode(struct pt_regs *regs) in exit_to_user_mode() argument
182 exit_to_user_mode_prepare(regs); in exit_to_user_mode()
187 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) in asm_exit_to_user_mode() argument
189 exit_to_user_mode(regs); in asm_exit_to_user_mode()
197 static void noinstr arm64_enter_nmi(struct pt_regs *regs) in arm64_enter_nmi() argument
199 regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); in arm64_enter_nmi()
215 static void noinstr arm64_exit_nmi(struct pt_regs *regs) in arm64_exit_nmi() argument
217 bool restore = regs->lockdep_hardirqs; in arm64_exit_nmi()
237 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) in arm64_enter_el1_dbg() argument
239 regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); in arm64_enter_el1_dbg()
252 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) in arm64_exit_el1_dbg() argument
254 bool restore = regs->lockdep_hardirqs; in arm64_exit_el1_dbg()
308 static void do_interrupt_handler(struct pt_regs *regs, in do_interrupt_handler() argument
311 struct pt_regs *old_regs = set_irq_regs(regs); in do_interrupt_handler()
314 call_on_irq_stack(regs, handler); in do_interrupt_handler()
316 handler(regs); in do_interrupt_handler()
324 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, in __panic_unhandled() argument
327 arm64_enter_nmi(regs); in __panic_unhandled()
335 __show_regs(regs); in __panic_unhandled()
340 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
343 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
373 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) in cortex_a76_erratum_1463225_debug_handler() argument
385 regs->pstate |= PSR_D_BIT; in cortex_a76_erratum_1463225_debug_handler()
390 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) in cortex_a76_erratum_1463225_debug_handler() argument
455 static void debug_exception_enter(struct pt_regs *regs) in debug_exception_enter() argument
464 static void debug_exception_exit(struct pt_regs *regs) in debug_exception_exit() argument
475 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) in el1_abort() argument
479 enter_from_kernel_mode(regs); in el1_abort()
480 local_daif_inherit(regs); in el1_abort()
481 do_mem_abort(far, esr, regs); in el1_abort()
483 exit_to_kernel_mode(regs); in el1_abort()
486 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) in el1_pc() argument
490 enter_from_kernel_mode(regs); in el1_pc()
491 local_daif_inherit(regs); in el1_pc()
492 do_sp_pc_abort(far, esr, regs); in el1_pc()
494 exit_to_kernel_mode(regs); in el1_pc()
497 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) in el1_undef() argument
499 enter_from_kernel_mode(regs); in el1_undef()
500 local_daif_inherit(regs); in el1_undef()
501 do_el1_undef(regs, esr); in el1_undef()
503 exit_to_kernel_mode(regs); in el1_undef()
506 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) in el1_bti() argument
508 enter_from_kernel_mode(regs); in el1_bti()
509 local_daif_inherit(regs); in el1_bti()
510 do_el1_bti(regs, esr); in el1_bti()
512 exit_to_kernel_mode(regs); in el1_bti()
515 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) in el1_gcs() argument
517 enter_from_kernel_mode(regs); in el1_gcs()
518 local_daif_inherit(regs); in el1_gcs()
519 do_el1_gcs(regs, esr); in el1_gcs()
521 exit_to_kernel_mode(regs); in el1_gcs()
524 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) in el1_mops() argument
526 enter_from_kernel_mode(regs); in el1_mops()
527 local_daif_inherit(regs); in el1_mops()
528 do_el1_mops(regs, esr); in el1_mops()
530 exit_to_kernel_mode(regs); in el1_mops()
533 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) in el1_breakpt() argument
535 arm64_enter_el1_dbg(regs); in el1_breakpt()
536 debug_exception_enter(regs); in el1_breakpt()
537 do_breakpoint(esr, regs); in el1_breakpt()
538 debug_exception_exit(regs); in el1_breakpt()
539 arm64_exit_el1_dbg(regs); in el1_breakpt()
542 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) in el1_softstp() argument
544 arm64_enter_el1_dbg(regs); in el1_softstp()
545 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { in el1_softstp()
546 debug_exception_enter(regs); in el1_softstp()
553 if (!try_step_suspended_breakpoints(regs)) in el1_softstp()
554 do_el1_softstep(esr, regs); in el1_softstp()
555 debug_exception_exit(regs); in el1_softstp()
557 arm64_exit_el1_dbg(regs); in el1_softstp()
560 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) in el1_watchpt() argument
565 arm64_enter_el1_dbg(regs); in el1_watchpt()
566 debug_exception_enter(regs); in el1_watchpt()
567 do_watchpoint(far, esr, regs); in el1_watchpt()
568 debug_exception_exit(regs); in el1_watchpt()
569 arm64_exit_el1_dbg(regs); in el1_watchpt()
572 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) in el1_brk64() argument
574 arm64_enter_el1_dbg(regs); in el1_brk64()
575 debug_exception_enter(regs); in el1_brk64()
576 do_el1_brk64(esr, regs); in el1_brk64()
577 debug_exception_exit(regs); in el1_brk64()
578 arm64_exit_el1_dbg(regs); in el1_brk64()
581 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) in el1_fpac() argument
583 enter_from_kernel_mode(regs); in el1_fpac()
584 local_daif_inherit(regs); in el1_fpac()
585 do_el1_fpac(regs, esr); in el1_fpac()
587 exit_to_kernel_mode(regs); in el1_fpac()
590 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) in el1h_64_sync_handler() argument
597 el1_abort(regs, esr); in el1h_64_sync_handler()
604 el1_pc(regs, esr); in el1h_64_sync_handler()
608 el1_undef(regs, esr); in el1h_64_sync_handler()
611 el1_bti(regs, esr); in el1h_64_sync_handler()
614 el1_gcs(regs, esr); in el1h_64_sync_handler()
617 el1_mops(regs, esr); in el1h_64_sync_handler()
620 el1_breakpt(regs, esr); in el1h_64_sync_handler()
623 el1_softstp(regs, esr); in el1h_64_sync_handler()
626 el1_watchpt(regs, esr); in el1h_64_sync_handler()
629 el1_brk64(regs, esr); in el1h_64_sync_handler()
632 el1_fpac(regs, esr); in el1h_64_sync_handler()
635 __panic_unhandled(regs, "64-bit el1h sync", esr); in el1h_64_sync_handler()
639 static __always_inline void __el1_pnmi(struct pt_regs *regs, in __el1_pnmi() argument
642 arm64_enter_nmi(regs); in __el1_pnmi()
643 do_interrupt_handler(regs, handler); in __el1_pnmi()
644 arm64_exit_nmi(regs); in __el1_pnmi()
647 static __always_inline void __el1_irq(struct pt_regs *regs, in __el1_irq() argument
650 enter_from_kernel_mode(regs); in __el1_irq()
653 do_interrupt_handler(regs, handler); in __el1_irq()
658 exit_to_kernel_mode(regs); in __el1_irq()
660 static void noinstr el1_interrupt(struct pt_regs *regs, in el1_interrupt() argument
665 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) in el1_interrupt()
666 __el1_pnmi(regs, handler); in el1_interrupt()
668 __el1_irq(regs, handler); in el1_interrupt()
671 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) in el1h_64_irq_handler() argument
673 el1_interrupt(regs, handle_arch_irq); in el1h_64_irq_handler()
676 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) in el1h_64_fiq_handler() argument
678 el1_interrupt(regs, handle_arch_fiq); in el1h_64_fiq_handler()
681 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) in el1h_64_error_handler() argument
686 arm64_enter_nmi(regs); in el1h_64_error_handler()
687 do_serror(regs, esr); in el1h_64_error_handler()
688 arm64_exit_nmi(regs); in el1h_64_error_handler()
691 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) in el0_da() argument
695 enter_from_user_mode(regs); in el0_da()
697 do_mem_abort(far, esr, regs); in el0_da()
698 exit_to_user_mode(regs); in el0_da()
701 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) in el0_ia() argument
713 enter_from_user_mode(regs); in el0_ia()
715 do_mem_abort(far, esr, regs); in el0_ia()
716 exit_to_user_mode(regs); in el0_ia()
719 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) in el0_fpsimd_acc() argument
721 enter_from_user_mode(regs); in el0_fpsimd_acc()
723 do_fpsimd_acc(esr, regs); in el0_fpsimd_acc()
724 exit_to_user_mode(regs); in el0_fpsimd_acc()
727 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) in el0_sve_acc() argument
729 enter_from_user_mode(regs); in el0_sve_acc()
731 do_sve_acc(esr, regs); in el0_sve_acc()
732 exit_to_user_mode(regs); in el0_sve_acc()
735 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) in el0_sme_acc() argument
737 enter_from_user_mode(regs); in el0_sme_acc()
739 do_sme_acc(esr, regs); in el0_sme_acc()
740 exit_to_user_mode(regs); in el0_sme_acc()
743 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) in el0_fpsimd_exc() argument
745 enter_from_user_mode(regs); in el0_fpsimd_exc()
747 do_fpsimd_exc(esr, regs); in el0_fpsimd_exc()
748 exit_to_user_mode(regs); in el0_fpsimd_exc()
751 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) in el0_sys() argument
753 enter_from_user_mode(regs); in el0_sys()
755 do_el0_sys(esr, regs); in el0_sys()
756 exit_to_user_mode(regs); in el0_sys()
759 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) in el0_pc() argument
763 if (!is_ttbr0_addr(instruction_pointer(regs))) in el0_pc()
766 enter_from_user_mode(regs); in el0_pc()
768 do_sp_pc_abort(far, esr, regs); in el0_pc()
769 exit_to_user_mode(regs); in el0_pc()
772 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) in el0_sp() argument
774 enter_from_user_mode(regs); in el0_sp()
776 do_sp_pc_abort(regs->sp, esr, regs); in el0_sp()
777 exit_to_user_mode(regs); in el0_sp()
780 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) in el0_undef() argument
782 enter_from_user_mode(regs); in el0_undef()
784 do_el0_undef(regs, esr); in el0_undef()
785 exit_to_user_mode(regs); in el0_undef()
788 static void noinstr el0_bti(struct pt_regs *regs) in el0_bti() argument
790 enter_from_user_mode(regs); in el0_bti()
792 do_el0_bti(regs); in el0_bti()
793 exit_to_user_mode(regs); in el0_bti()
796 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) in el0_mops() argument
798 enter_from_user_mode(regs); in el0_mops()
800 do_el0_mops(regs, esr); in el0_mops()
801 exit_to_user_mode(regs); in el0_mops()
804 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) in el0_gcs() argument
806 enter_from_user_mode(regs); in el0_gcs()
808 do_el0_gcs(regs, esr); in el0_gcs()
809 exit_to_user_mode(regs); in el0_gcs()
812 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) in el0_inv() argument
814 enter_from_user_mode(regs); in el0_inv()
816 bad_el0_sync(regs, 0, esr); in el0_inv()
817 exit_to_user_mode(regs); in el0_inv()
820 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) in el0_breakpt() argument
822 if (!is_ttbr0_addr(regs->pc)) in el0_breakpt()
825 enter_from_user_mode(regs); in el0_breakpt()
826 debug_exception_enter(regs); in el0_breakpt()
827 do_breakpoint(esr, regs); in el0_breakpt()
828 debug_exception_exit(regs); in el0_breakpt()
830 exit_to_user_mode(regs); in el0_breakpt()
833 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) in el0_softstp() argument
835 if (!is_ttbr0_addr(regs->pc)) in el0_softstp()
838 enter_from_user_mode(regs); in el0_softstp()
845 if (!try_step_suspended_breakpoints(regs)) { in el0_softstp()
847 do_el0_softstep(esr, regs); in el0_softstp()
849 exit_to_user_mode(regs); in el0_softstp()
852 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) in el0_watchpt() argument
857 enter_from_user_mode(regs); in el0_watchpt()
858 debug_exception_enter(regs); in el0_watchpt()
859 do_watchpoint(far, esr, regs); in el0_watchpt()
860 debug_exception_exit(regs); in el0_watchpt()
862 exit_to_user_mode(regs); in el0_watchpt()
865 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) in el0_brk64() argument
867 enter_from_user_mode(regs); in el0_brk64()
869 do_el0_brk64(esr, regs); in el0_brk64()
870 exit_to_user_mode(regs); in el0_brk64()
873 static void noinstr el0_svc(struct pt_regs *regs) in el0_svc() argument
875 enter_from_user_mode(regs); in el0_svc()
879 do_el0_svc(regs); in el0_svc()
880 exit_to_user_mode(regs); in el0_svc()
884 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) in el0_fpac() argument
886 enter_from_user_mode(regs); in el0_fpac()
888 do_el0_fpac(regs, esr); in el0_fpac()
889 exit_to_user_mode(regs); in el0_fpac()
892 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) in el0t_64_sync_handler() argument
898 el0_svc(regs); in el0t_64_sync_handler()
901 el0_da(regs, esr); in el0t_64_sync_handler()
904 el0_ia(regs, esr); in el0t_64_sync_handler()
907 el0_fpsimd_acc(regs, esr); in el0t_64_sync_handler()
910 el0_sve_acc(regs, esr); in el0t_64_sync_handler()
913 el0_sme_acc(regs, esr); in el0t_64_sync_handler()
916 el0_fpsimd_exc(regs, esr); in el0t_64_sync_handler()
920 el0_sys(regs, esr); in el0t_64_sync_handler()
923 el0_sp(regs, esr); in el0t_64_sync_handler()
926 el0_pc(regs, esr); in el0t_64_sync_handler()
929 el0_undef(regs, esr); in el0t_64_sync_handler()
932 el0_bti(regs); in el0t_64_sync_handler()
935 el0_mops(regs, esr); in el0t_64_sync_handler()
938 el0_gcs(regs, esr); in el0t_64_sync_handler()
941 el0_breakpt(regs, esr); in el0t_64_sync_handler()
944 el0_softstp(regs, esr); in el0t_64_sync_handler()
947 el0_watchpt(regs, esr); in el0t_64_sync_handler()
950 el0_brk64(regs, esr); in el0t_64_sync_handler()
953 el0_fpac(regs, esr); in el0t_64_sync_handler()
956 el0_inv(regs, esr); in el0t_64_sync_handler()
960 static void noinstr el0_interrupt(struct pt_regs *regs, in el0_interrupt() argument
963 enter_from_user_mode(regs); in el0_interrupt()
967 if (regs->pc & BIT(55)) in el0_interrupt()
971 do_interrupt_handler(regs, handler); in el0_interrupt()
974 exit_to_user_mode(regs); in el0_interrupt()
977 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) in __el0_irq_handler_common() argument
979 el0_interrupt(regs, handle_arch_irq); in __el0_irq_handler_common()
982 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) in el0t_64_irq_handler() argument
984 __el0_irq_handler_common(regs); in el0t_64_irq_handler()
987 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) in __el0_fiq_handler_common() argument
989 el0_interrupt(regs, handle_arch_fiq); in __el0_fiq_handler_common()
992 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) in el0t_64_fiq_handler() argument
994 __el0_fiq_handler_common(regs); in el0t_64_fiq_handler()
997 static void noinstr __el0_error_handler_common(struct pt_regs *regs) in __el0_error_handler_common() argument
1001 enter_from_user_mode(regs); in __el0_error_handler_common()
1003 arm64_enter_nmi(regs); in __el0_error_handler_common()
1004 do_serror(regs, esr); in __el0_error_handler_common()
1005 arm64_exit_nmi(regs); in __el0_error_handler_common()
1007 exit_to_user_mode(regs); in __el0_error_handler_common()
1010 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) in el0t_64_error_handler() argument
1012 __el0_error_handler_common(regs); in el0t_64_error_handler()
1016 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) in el0_cp15() argument
1018 enter_from_user_mode(regs); in el0_cp15()
1020 do_el0_cp15(esr, regs); in el0_cp15()
1021 exit_to_user_mode(regs); in el0_cp15()
1024 static void noinstr el0_svc_compat(struct pt_regs *regs) in el0_svc_compat() argument
1026 enter_from_user_mode(regs); in el0_svc_compat()
1029 do_el0_svc_compat(regs); in el0_svc_compat()
1030 exit_to_user_mode(regs); in el0_svc_compat()
1033 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) in el0_bkpt32() argument
1035 enter_from_user_mode(regs); in el0_bkpt32()
1037 do_bkpt32(esr, regs); in el0_bkpt32()
1038 exit_to_user_mode(regs); in el0_bkpt32()
1041 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) in el0t_32_sync_handler() argument
1047 el0_svc_compat(regs); in el0t_32_sync_handler()
1050 el0_da(regs, esr); in el0t_32_sync_handler()
1053 el0_ia(regs, esr); in el0t_32_sync_handler()
1056 el0_fpsimd_acc(regs, esr); in el0t_32_sync_handler()
1059 el0_fpsimd_exc(regs, esr); in el0t_32_sync_handler()
1062 el0_pc(regs, esr); in el0t_32_sync_handler()
1068 el0_undef(regs, esr); in el0t_32_sync_handler()
1072 el0_cp15(regs, esr); in el0t_32_sync_handler()
1075 el0_breakpt(regs, esr); in el0t_32_sync_handler()
1078 el0_softstp(regs, esr); in el0t_32_sync_handler()
1081 el0_watchpt(regs, esr); in el0t_32_sync_handler()
1084 el0_bkpt32(regs, esr); in el0t_32_sync_handler()
1087 el0_inv(regs, esr); in el0t_32_sync_handler()
1091 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) in el0t_32_irq_handler() argument
1093 __el0_irq_handler_common(regs); in el0t_32_irq_handler()
1096 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) in el0t_32_fiq_handler() argument
1098 __el0_fiq_handler_common(regs); in el0t_32_fiq_handler()
1101 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) in el0t_32_error_handler() argument
1103 __el0_error_handler_common(regs); in el0t_32_error_handler()
1112 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) in handle_bad_stack() argument
1117 arm64_enter_nmi(regs); in handle_bad_stack()
1118 panic_bad_stack(regs, esr, far); in handle_bad_stack()
1123 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) in __sdei_handler() argument
1149 arm64_enter_nmi(regs); in __sdei_handler()
1150 ret = do_sdei_event(regs, arg); in __sdei_handler()
1151 arm64_exit_nmi(regs); in __sdei_handler()