1*a737737cSHeiko Carstens /* SPDX-License-Identifier: GPL-2.0 */ 2*a737737cSHeiko Carstens #ifndef ARCH_S390_ENTRY_PERCPU_H 3*a737737cSHeiko Carstens #define ARCH_S390_ENTRY_PERCPU_H 4*a737737cSHeiko Carstens 5*a737737cSHeiko Carstens #include <linux/kprobes.h> 6*a737737cSHeiko Carstens #include <linux/percpu.h> 7*a737737cSHeiko Carstens #include <asm/lowcore.h> 8*a737737cSHeiko Carstens #include <asm/ptrace.h> 9*a737737cSHeiko Carstens #include <asm/asm-offsets.h> 10*a737737cSHeiko Carstens 11*a737737cSHeiko Carstens static __always_inline void percpu_entry(struct pt_regs *regs) 12*a737737cSHeiko Carstens { 13*a737737cSHeiko Carstens struct lowcore *lc = get_lowcore(); 14*a737737cSHeiko Carstens 15*a737737cSHeiko Carstens if (user_mode(regs)) 16*a737737cSHeiko Carstens return; 17*a737737cSHeiko Carstens regs->cpu = lc->cpu_nr; 18*a737737cSHeiko Carstens regs->percpu_register = lc->percpu_register; 19*a737737cSHeiko Carstens lc->percpu_register = 0; 20*a737737cSHeiko Carstens } 21*a737737cSHeiko Carstens 22*a737737cSHeiko Carstens static __always_inline bool percpu_code_check(struct pt_regs *regs) 23*a737737cSHeiko Carstens { 24*a737737cSHeiko Carstens unsigned int insn, disp; 25*a737737cSHeiko Carstens struct kprobe *p; 26*a737737cSHeiko Carstens 27*a737737cSHeiko Carstens if (likely(user_mode(regs) || !regs->percpu_register)) 28*a737737cSHeiko Carstens return false; 29*a737737cSHeiko Carstens /* 30*a737737cSHeiko Carstens * Within a percpu code section - check if the percpu base register 31*a737737cSHeiko Carstens * needs to be updated. This is the case if the PSW does not point to 32*a737737cSHeiko Carstens * the ADD instruction within the section. 33*a737737cSHeiko Carstens * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) 34*a737737cSHeiko Carstens * which adds the percpu offset to the percpu base register. 35*a737737cSHeiko Carstens */ 36*a737737cSHeiko Carstens lockdep_assert_preemption_disabled(); 37*a737737cSHeiko Carstens again: 38*a737737cSHeiko Carstens insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); 39*a737737cSHeiko Carstens if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { 40*a737737cSHeiko Carstens p = get_kprobe((void *)psw_bits(regs->psw).ia); 41*a737737cSHeiko Carstens /* 42*a737737cSHeiko Carstens * If the kprobe is concurrently removed on a different CPU 43*a737737cSHeiko Carstens * it might not be found anymore. However text must have 44*a737737cSHeiko Carstens * been restored - try again. 45*a737737cSHeiko Carstens */ 46*a737737cSHeiko Carstens if (!p) 47*a737737cSHeiko Carstens goto again; 48*a737737cSHeiko Carstens insn = p->opcode; 49*a737737cSHeiko Carstens } 50*a737737cSHeiko Carstens if ((insn & 0xff0f) != 0xe300) 51*a737737cSHeiko Carstens return true; 52*a737737cSHeiko Carstens disp = offsetof(struct lowcore, percpu_offset); 53*a737737cSHeiko Carstens if (machine_has_relocated_lowcore()) 54*a737737cSHeiko Carstens disp += LOWCORE_ALT_ADDRESS; 55*a737737cSHeiko Carstens insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; 56*a737737cSHeiko Carstens if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) 57*a737737cSHeiko Carstens return true; 58*a737737cSHeiko Carstens return false; 59*a737737cSHeiko Carstens } 60*a737737cSHeiko Carstens 61*a737737cSHeiko Carstens static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) 62*a737737cSHeiko Carstens { 63*a737737cSHeiko Carstens struct lowcore *lc = get_lowcore(); 64*a737737cSHeiko Carstens unsigned char reg; 65*a737737cSHeiko Carstens 66*a737737cSHeiko Carstens if (user_mode(regs)) 67*a737737cSHeiko Carstens return; 68*a737737cSHeiko Carstens reg = regs->percpu_register; 69*a737737cSHeiko Carstens lc->percpu_register = reg; 70*a737737cSHeiko Carstens if (likely(!needs_fixup)) 71*a737737cSHeiko Carstens return; 72*a737737cSHeiko Carstens /* Check if process has been migrated to a different CPU. */ 73*a737737cSHeiko Carstens if (regs->cpu == lc->cpu_nr) 74*a737737cSHeiko Carstens return; 75*a737737cSHeiko Carstens /* Fixup percpu base register */ 76*a737737cSHeiko Carstens regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; 77*a737737cSHeiko Carstens regs->gprs[reg] += lc->percpu_offset; 78*a737737cSHeiko Carstens } 79*a737737cSHeiko Carstens 80*a737737cSHeiko Carstens #endif 81