1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef ARCH_S390_ENTRY_PERCPU_H 3 #define ARCH_S390_ENTRY_PERCPU_H 4 5 #include <linux/kprobes.h> 6 #include <linux/percpu.h> 7 #include <asm/lowcore.h> 8 #include <asm/ptrace.h> 9 #include <asm/asm-offsets.h> 10 11 static __always_inline void percpu_entry(struct pt_regs *regs) 12 { 13 struct lowcore *lc = get_lowcore(); 14 15 if (user_mode(regs)) 16 return; 17 regs->cpu = lc->cpu_nr; 18 regs->percpu_register = lc->percpu_register; 19 lc->percpu_register = 0; 20 } 21 22 static __always_inline bool percpu_code_check(struct pt_regs *regs) 23 { 24 unsigned int insn, disp; 25 struct kprobe *p; 26 27 if (likely(user_mode(regs) || !regs->percpu_register)) 28 return false; 29 /* 30 * Within a percpu code section - check if the percpu base register 31 * needs to be updated. This is the case if the PSW does not point to 32 * the ADD instruction within the section. 33 * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) 34 * which adds the percpu offset to the percpu base register. 35 */ 36 lockdep_assert_preemption_disabled(); 37 again: 38 insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); 39 if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { 40 p = get_kprobe((void *)psw_bits(regs->psw).ia); 41 /* 42 * If the kprobe is concurrently removed on a different CPU 43 * it might not be found anymore. However text must have 44 * been restored - try again. 45 */ 46 if (!p) 47 goto again; 48 insn = p->opcode; 49 } 50 if ((insn & 0xff0f) != 0xe300) 51 return true; 52 disp = offsetof(struct lowcore, percpu_offset); 53 if (machine_has_relocated_lowcore()) 54 disp += LOWCORE_ALT_ADDRESS; 55 insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; 56 if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) 57 return true; 58 return false; 59 } 60 61 static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) 62 { 63 struct lowcore *lc = get_lowcore(); 64 unsigned char reg; 65 66 if (user_mode(regs)) 67 return; 68 reg = regs->percpu_register; 69 lc->percpu_register = reg; 70 if (likely(!needs_fixup)) 71 return; 72 /* Check if process has been migrated to a different CPU. */ 73 if (regs->cpu == lc->cpu_nr) 74 return; 75 /* Fixup percpu base register */ 76 regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; 77 regs->gprs[reg] += lc->percpu_offset; 78 } 79 80 #endif 81