xref: /linux/arch/x86/kernel/uprobes.c (revision 827634added7f38b7d724cab1dccdb2b004c13c3)
1 /*
2  * User-space Probes (UProbes) for x86
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17  *
18  * Copyright (C) IBM Corporation, 2008-2011
19  * Authors:
20  *	Srikar Dronamraju
21  *	Jim Keniston
22  */
23 #include <linux/kernel.h>
24 #include <linux/sched.h>
25 #include <linux/ptrace.h>
26 #include <linux/uprobes.h>
27 #include <linux/uaccess.h>
28 
29 #include <linux/kdebug.h>
30 #include <asm/processor.h>
31 #include <asm/insn.h>
32 
33 /* Post-execution fixups. */
34 
35 /* Adjust IP back to vicinity of actual insn */
36 #define UPROBE_FIX_IP		0x01
37 
38 /* Adjust the return address of a call insn */
39 #define UPROBE_FIX_CALL		0x02
40 
41 /* Instruction will modify TF, don't change it */
42 #define UPROBE_FIX_SETF		0x04
43 
44 #define UPROBE_FIX_RIP_SI	0x08
45 #define UPROBE_FIX_RIP_DI	0x10
46 #define UPROBE_FIX_RIP_BX	0x20
47 #define UPROBE_FIX_RIP_MASK	\
48 	(UPROBE_FIX_RIP_SI | UPROBE_FIX_RIP_DI | UPROBE_FIX_RIP_BX)
49 
50 #define	UPROBE_TRAP_NR		UINT_MAX
51 
52 /* Adaptations for mhiramat x86 decoder v14. */
53 #define OPCODE1(insn)		((insn)->opcode.bytes[0])
54 #define OPCODE2(insn)		((insn)->opcode.bytes[1])
55 #define OPCODE3(insn)		((insn)->opcode.bytes[2])
56 #define MODRM_REG(insn)		X86_MODRM_REG((insn)->modrm.value)
57 
58 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
59 	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
60 	  (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) |   \
61 	  (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) |   \
62 	  (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf))    \
63 	 << (row % 32))
64 
65 /*
66  * Good-instruction tables for 32-bit apps.  This is non-const and volatile
67  * to keep gcc from statically optimizing it out, as variable_test_bit makes
68  * some versions of gcc to think only *(unsigned long*) is used.
69  *
70  * Opcodes we'll probably never support:
71  * 6c-6f - ins,outs. SEGVs if used in userspace
72  * e4-e7 - in,out imm. SEGVs if used in userspace
73  * ec-ef - in,out acc. SEGVs if used in userspace
74  * cc - int3. SIGTRAP if used in userspace
75  * ce - into. Not used in userspace - no kernel support to make it useful. SEGVs
76  *	(why we support bound (62) then? it's similar, and similarly unused...)
77  * f1 - int1. SIGTRAP if used in userspace
78  * f4 - hlt. SEGVs if used in userspace
79  * fa - cli. SEGVs if used in userspace
80  * fb - sti. SEGVs if used in userspace
81  *
82  * Opcodes which need some work to be supported:
83  * 07,17,1f - pop es/ss/ds
84  *	Normally not used in userspace, but would execute if used.
85  *	Can cause GP or stack exception if tries to load wrong segment descriptor.
86  *	We hesitate to run them under single step since kernel's handling
87  *	of userspace single-stepping (TF flag) is fragile.
88  *	We can easily refuse to support push es/cs/ss/ds (06/0e/16/1e)
89  *	on the same grounds that they are never used.
90  * cd - int N.
91  *	Used by userspace for "int 80" syscall entry. (Other "int N"
92  *	cause GP -> SEGV since their IDT gates don't allow calls from CPL 3).
93  *	Not supported since kernel's handling of userspace single-stepping
94  *	(TF flag) is fragile.
95  * cf - iret. Normally not used in userspace. Doesn't SEGV unless arguments are bad
96  */
97 #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
98 static volatile u32 good_insns_32[256 / 32] = {
99 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f         */
100 	/*      ----------------------------------------------         */
101 	W(0x00, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 00 */
102 	W(0x10, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) , /* 10 */
103 	W(0x20, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
104 	W(0x30, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 30 */
105 	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
106 	W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
107 	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
108 	W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
109 	W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
110 	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
111 	W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
112 	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
113 	W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
114 	W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
115 	W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* e0 */
116 	W(0xf0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1)   /* f0 */
117 	/*      ----------------------------------------------         */
118 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f         */
119 };
120 #else
121 #define good_insns_32	NULL
122 #endif
123 
124 /* Good-instruction tables for 64-bit apps.
125  *
126  * Genuinely invalid opcodes:
127  * 06,07 - formerly push/pop es
128  * 0e - formerly push cs
129  * 16,17 - formerly push/pop ss
130  * 1e,1f - formerly push/pop ds
131  * 27,2f,37,3f - formerly daa/das/aaa/aas
132  * 60,61 - formerly pusha/popa
133  * 62 - formerly bound. EVEX prefix for AVX512 (not yet supported)
134  * 82 - formerly redundant encoding of Group1
135  * 9a - formerly call seg:ofs
136  * ce - formerly into
137  * d4,d5 - formerly aam/aad
138  * d6 - formerly undocumented salc
139  * ea - formerly jmp seg:ofs
140  *
141  * Opcodes we'll probably never support:
142  * 6c-6f - ins,outs. SEGVs if used in userspace
143  * e4-e7 - in,out imm. SEGVs if used in userspace
144  * ec-ef - in,out acc. SEGVs if used in userspace
145  * cc - int3. SIGTRAP if used in userspace
146  * f1 - int1. SIGTRAP if used in userspace
147  * f4 - hlt. SEGVs if used in userspace
148  * fa - cli. SEGVs if used in userspace
149  * fb - sti. SEGVs if used in userspace
150  *
151  * Opcodes which need some work to be supported:
152  * cd - int N.
153  *	Used by userspace for "int 80" syscall entry. (Other "int N"
154  *	cause GP -> SEGV since their IDT gates don't allow calls from CPL 3).
155  *	Not supported since kernel's handling of userspace single-stepping
156  *	(TF flag) is fragile.
157  * cf - iret. Normally not used in userspace. Doesn't SEGV unless arguments are bad
158  */
159 #if defined(CONFIG_X86_64)
160 static volatile u32 good_insns_64[256 / 32] = {
161 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f         */
162 	/*      ----------------------------------------------         */
163 	W(0x00, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1) | /* 00 */
164 	W(0x10, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0) , /* 10 */
165 	W(0x20, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) | /* 20 */
166 	W(0x30, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) , /* 30 */
167 	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
168 	W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
169 	W(0x60, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
170 	W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
171 	W(0x80, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
172 	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1) , /* 90 */
173 	W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
174 	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
175 	W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
176 	W(0xd0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
177 	W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0) | /* e0 */
178 	W(0xf0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1)   /* f0 */
179 	/*      ----------------------------------------------         */
180 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f         */
181 };
182 #else
183 #define good_insns_64	NULL
184 #endif
185 
186 /* Using this for both 64-bit and 32-bit apps.
187  * Opcodes we don't support:
188  * 0f 00 - SLDT/STR/LLDT/LTR/VERR/VERW/-/- group. System insns
189  * 0f 01 - SGDT/SIDT/LGDT/LIDT/SMSW/-/LMSW/INVLPG group.
190  *	Also encodes tons of other system insns if mod=11.
191  *	Some are in fact non-system: xend, xtest, rdtscp, maybe more
192  * 0f 05 - syscall
193  * 0f 06 - clts (CPL0 insn)
194  * 0f 07 - sysret
195  * 0f 08 - invd (CPL0 insn)
196  * 0f 09 - wbinvd (CPL0 insn)
197  * 0f 0b - ud2
198  * 0f 30 - wrmsr (CPL0 insn) (then why rdmsr is allowed, it's also CPL0 insn?)
199  * 0f 34 - sysenter
200  * 0f 35 - sysexit
201  * 0f 37 - getsec
202  * 0f 78 - vmread (Intel VMX. CPL0 insn)
203  * 0f 79 - vmwrite (Intel VMX. CPL0 insn)
204  *	Note: with prefixes, these two opcodes are
205  *	extrq/insertq/AVX512 convert vector ops.
206  * 0f ae - group15: [f]xsave,[f]xrstor,[v]{ld,st}mxcsr,clflush[opt],
207  *	{rd,wr}{fs,gs}base,{s,l,m}fence.
208  *	Why? They are all user-executable.
209  */
210 static volatile u32 good_2byte_insns[256 / 32] = {
211 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f         */
212 	/*      ----------------------------------------------         */
213 	W(0x00, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1) | /* 00 */
214 	W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 10 */
215 	W(0x20, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
216 	W(0x30, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) , /* 30 */
217 	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
218 	W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
219 	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 60 */
220 	W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1) , /* 70 */
221 	W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
222 	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
223 	W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1) | /* a0 */
224 	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
225 	W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
226 	W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
227 	W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* e0 */
228 	W(0xf0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)   /* f0 */
229 	/*      ----------------------------------------------         */
230 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f         */
231 };
232 #undef W
233 
234 /*
235  * opcodes we may need to refine support for:
236  *
237  *  0f - 2-byte instructions: For many of these instructions, the validity
238  *  depends on the prefix and/or the reg field.  On such instructions, we
239  *  just consider the opcode combination valid if it corresponds to any
240  *  valid instruction.
241  *
242  *  8f - Group 1 - only reg = 0 is OK
243  *  c6-c7 - Group 11 - only reg = 0 is OK
244  *  d9-df - fpu insns with some illegal encodings
245  *  f2, f3 - repnz, repz prefixes.  These are also the first byte for
246  *  certain floating-point instructions, such as addsd.
247  *
248  *  fe - Group 4 - only reg = 0 or 1 is OK
249  *  ff - Group 5 - only reg = 0-6 is OK
250  *
251  * others -- Do we need to support these?
252  *
253  *  0f - (floating-point?) prefetch instructions
254  *  07, 17, 1f - pop es, pop ss, pop ds
255  *  26, 2e, 36, 3e - es:, cs:, ss:, ds: segment prefixes --
256  *	but 64 and 65 (fs: and gs:) seem to be used, so we support them
257  *  67 - addr16 prefix
258  *  ce - into
259  *  f0 - lock prefix
260  */
261 
262 /*
263  * TODO:
264  * - Where necessary, examine the modrm byte and allow only valid instructions
265  * in the different Groups and fpu instructions.
266  */
267 
268 static bool is_prefix_bad(struct insn *insn)
269 {
270 	int i;
271 
272 	for (i = 0; i < insn->prefixes.nbytes; i++) {
273 		switch (insn->prefixes.bytes[i]) {
274 		case 0x26:	/* INAT_PFX_ES   */
275 		case 0x2E:	/* INAT_PFX_CS   */
276 		case 0x36:	/* INAT_PFX_DS   */
277 		case 0x3E:	/* INAT_PFX_SS   */
278 		case 0xF0:	/* INAT_PFX_LOCK */
279 			return true;
280 		}
281 	}
282 	return false;
283 }
284 
285 static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)
286 {
287 	u32 volatile *good_insns;
288 
289 	insn_init(insn, auprobe->insn, sizeof(auprobe->insn), x86_64);
290 	/* has the side-effect of processing the entire instruction */
291 	insn_get_length(insn);
292 	if (WARN_ON_ONCE(!insn_complete(insn)))
293 		return -ENOEXEC;
294 
295 	if (is_prefix_bad(insn))
296 		return -ENOTSUPP;
297 
298 	if (x86_64)
299 		good_insns = good_insns_64;
300 	else
301 		good_insns = good_insns_32;
302 
303 	if (test_bit(OPCODE1(insn), (unsigned long *)good_insns))
304 		return 0;
305 
306 	if (insn->opcode.nbytes == 2) {
307 		if (test_bit(OPCODE2(insn), (unsigned long *)good_2byte_insns))
308 			return 0;
309 	}
310 
311 	return -ENOTSUPP;
312 }
313 
314 #ifdef CONFIG_X86_64
315 static inline bool is_64bit_mm(struct mm_struct *mm)
316 {
317 	return	!config_enabled(CONFIG_IA32_EMULATION) ||
318 		!(mm->context.ia32_compat == TIF_IA32);
319 }
320 /*
321  * If arch_uprobe->insn doesn't use rip-relative addressing, return
322  * immediately.  Otherwise, rewrite the instruction so that it accesses
323  * its memory operand indirectly through a scratch register.  Set
324  * defparam->fixups accordingly. (The contents of the scratch register
325  * will be saved before we single-step the modified instruction,
326  * and restored afterward).
327  *
328  * We do this because a rip-relative instruction can access only a
329  * relatively small area (+/- 2 GB from the instruction), and the XOL
330  * area typically lies beyond that area.  At least for instructions
331  * that store to memory, we can't execute the original instruction
332  * and "fix things up" later, because the misdirected store could be
333  * disastrous.
334  *
335  * Some useful facts about rip-relative instructions:
336  *
337  *  - There's always a modrm byte with bit layout "00 reg 101".
338  *  - There's never a SIB byte.
339  *  - The displacement is always 4 bytes.
340  *  - REX.B=1 bit in REX prefix, which normally extends r/m field,
341  *    has no effect on rip-relative mode. It doesn't make modrm byte
342  *    with r/m=101 refer to register 1101 = R13.
343  */
344 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
345 {
346 	u8 *cursor;
347 	u8 reg;
348 	u8 reg2;
349 
350 	if (!insn_rip_relative(insn))
351 		return;
352 
353 	/*
354 	 * insn_rip_relative() would have decoded rex_prefix, vex_prefix, modrm.
355 	 * Clear REX.b bit (extension of MODRM.rm field):
356 	 * we want to encode low numbered reg, not r8+.
357 	 */
358 	if (insn->rex_prefix.nbytes) {
359 		cursor = auprobe->insn + insn_offset_rex_prefix(insn);
360 		/* REX byte has 0100wrxb layout, clearing REX.b bit */
361 		*cursor &= 0xfe;
362 	}
363 	/*
364 	 * Similar treatment for VEX3 prefix.
365 	 * TODO: add XOP/EVEX treatment when insn decoder supports them
366 	 */
367 	if (insn->vex_prefix.nbytes == 3) {
368 		/*
369 		 * vex2:     c5    rvvvvLpp   (has no b bit)
370 		 * vex3/xop: c4/8f rxbmmmmm wvvvvLpp
371 		 * evex:     62    rxbR00mm wvvvv1pp zllBVaaa
372 		 *   (evex will need setting of both b and x since
373 		 *   in non-sib encoding evex.x is 4th bit of MODRM.rm)
374 		 * Setting VEX3.b (setting because it has inverted meaning):
375 		 */
376 		cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;
377 		*cursor |= 0x20;
378 	}
379 
380 	/*
381 	 * Convert from rip-relative addressing to register-relative addressing
382 	 * via a scratch register.
383 	 *
384 	 * This is tricky since there are insns with modrm byte
385 	 * which also use registers not encoded in modrm byte:
386 	 * [i]div/[i]mul: implicitly use dx:ax
387 	 * shift ops: implicitly use cx
388 	 * cmpxchg: implicitly uses ax
389 	 * cmpxchg8/16b: implicitly uses dx:ax and bx:cx
390 	 *   Encoding: 0f c7/1 modrm
391 	 *   The code below thinks that reg=1 (cx), chooses si as scratch.
392 	 * mulx: implicitly uses dx: mulx r/m,r1,r2 does r1:r2 = dx * r/m.
393 	 *   First appeared in Haswell (BMI2 insn). It is vex-encoded.
394 	 *   Example where none of bx,cx,dx can be used as scratch reg:
395 	 *   c4 e2 63 f6 0d disp32   mulx disp32(%rip),%ebx,%ecx
396 	 * [v]pcmpistri: implicitly uses cx, xmm0
397 	 * [v]pcmpistrm: implicitly uses xmm0
398 	 * [v]pcmpestri: implicitly uses ax, dx, cx, xmm0
399 	 * [v]pcmpestrm: implicitly uses ax, dx, xmm0
400 	 *   Evil SSE4.2 string comparison ops from hell.
401 	 * maskmovq/[v]maskmovdqu: implicitly uses (ds:rdi) as destination.
402 	 *   Encoding: 0f f7 modrm, 66 0f f7 modrm, vex-encoded: c5 f9 f7 modrm.
403 	 *   Store op1, byte-masked by op2 msb's in each byte, to (ds:rdi).
404 	 *   AMD says it has no 3-operand form (vex.vvvv must be 1111)
405 	 *   and that it can have only register operands, not mem
406 	 *   (its modrm byte must have mode=11).
407 	 *   If these restrictions will ever be lifted,
408 	 *   we'll need code to prevent selection of di as scratch reg!
409 	 *
410 	 * Summary: I don't know any insns with modrm byte which
411 	 * use SI register implicitly. DI register is used only
412 	 * by one insn (maskmovq) and BX register is used
413 	 * only by one too (cmpxchg8b).
414 	 * BP is stack-segment based (may be a problem?).
415 	 * AX, DX, CX are off-limits (many implicit users).
416 	 * SP is unusable (it's stack pointer - think about "pop mem";
417 	 * also, rsp+disp32 needs sib encoding -> insn length change).
418 	 */
419 
420 	reg = MODRM_REG(insn);	/* Fetch modrm.reg */
421 	reg2 = 0xff;		/* Fetch vex.vvvv */
422 	if (insn->vex_prefix.nbytes == 2)
423 		reg2 = insn->vex_prefix.bytes[1];
424 	else if (insn->vex_prefix.nbytes == 3)
425 		reg2 = insn->vex_prefix.bytes[2];
426 	/*
427 	 * TODO: add XOP, EXEV vvvv reading.
428 	 *
429 	 * vex.vvvv field is in bits 6-3, bits are inverted.
430 	 * But in 32-bit mode, high-order bit may be ignored.
431 	 * Therefore, let's consider only 3 low-order bits.
432 	 */
433 	reg2 = ((reg2 >> 3) & 0x7) ^ 0x7;
434 	/*
435 	 * Register numbering is ax,cx,dx,bx, sp,bp,si,di, r8..r15.
436 	 *
437 	 * Choose scratch reg. Order is important: must not select bx
438 	 * if we can use si (cmpxchg8b case!)
439 	 */
440 	if (reg != 6 && reg2 != 6) {
441 		reg2 = 6;
442 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;
443 	} else if (reg != 7 && reg2 != 7) {
444 		reg2 = 7;
445 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;
446 		/* TODO (paranoia): force maskmovq to not use di */
447 	} else {
448 		reg2 = 3;
449 		auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;
450 	}
451 	/*
452 	 * Point cursor at the modrm byte.  The next 4 bytes are the
453 	 * displacement.  Beyond the displacement, for some instructions,
454 	 * is the immediate operand.
455 	 */
456 	cursor = auprobe->insn + insn_offset_modrm(insn);
457 	/*
458 	 * Change modrm from "00 reg 101" to "10 reg reg2". Example:
459 	 * 89 05 disp32  mov %eax,disp32(%rip) becomes
460 	 * 89 86 disp32  mov %eax,disp32(%rsi)
461 	 */
462 	*cursor = 0x80 | (reg << 3) | reg2;
463 }
464 
465 static inline unsigned long *
466 scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)
467 {
468 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)
469 		return &regs->si;
470 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)
471 		return &regs->di;
472 	return &regs->bx;
473 }
474 
475 /*
476  * If we're emulating a rip-relative instruction, save the contents
477  * of the scratch register and store the target address in that register.
478  */
479 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
480 {
481 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
482 		struct uprobe_task *utask = current->utask;
483 		unsigned long *sr = scratch_reg(auprobe, regs);
484 
485 		utask->autask.saved_scratch_register = *sr;
486 		*sr = utask->vaddr + auprobe->defparam.ilen;
487 	}
488 }
489 
490 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
491 {
492 	if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
493 		struct uprobe_task *utask = current->utask;
494 		unsigned long *sr = scratch_reg(auprobe, regs);
495 
496 		*sr = utask->autask.saved_scratch_register;
497 	}
498 }
499 #else /* 32-bit: */
500 static inline bool is_64bit_mm(struct mm_struct *mm)
501 {
502 	return false;
503 }
504 /*
505  * No RIP-relative addressing on 32-bit
506  */
507 static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
508 {
509 }
510 static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
511 {
512 }
513 static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
514 {
515 }
516 #endif /* CONFIG_X86_64 */
517 
518 struct uprobe_xol_ops {
519 	bool	(*emulate)(struct arch_uprobe *, struct pt_regs *);
520 	int	(*pre_xol)(struct arch_uprobe *, struct pt_regs *);
521 	int	(*post_xol)(struct arch_uprobe *, struct pt_regs *);
522 	void	(*abort)(struct arch_uprobe *, struct pt_regs *);
523 };
524 
525 static inline int sizeof_long(void)
526 {
527 	return is_ia32_task() ? 4 : 8;
528 }
529 
530 static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
531 {
532 	riprel_pre_xol(auprobe, regs);
533 	return 0;
534 }
535 
536 static int push_ret_address(struct pt_regs *regs, unsigned long ip)
537 {
538 	unsigned long new_sp = regs->sp - sizeof_long();
539 
540 	if (copy_to_user((void __user *)new_sp, &ip, sizeof_long()))
541 		return -EFAULT;
542 
543 	regs->sp = new_sp;
544 	return 0;
545 }
546 
547 /*
548  * We have to fix things up as follows:
549  *
550  * Typically, the new ip is relative to the copied instruction.  We need
551  * to make it relative to the original instruction (FIX_IP).  Exceptions
552  * are return instructions and absolute or indirect jump or call instructions.
553  *
554  * If the single-stepped instruction was a call, the return address that
555  * is atop the stack is the address following the copied instruction.  We
556  * need to make it the address following the original instruction (FIX_CALL).
557  *
558  * If the original instruction was a rip-relative instruction such as
559  * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
560  * instruction using a scratch register -- e.g., "movl %edx,0xnnnn(%rsi)".
561  * We need to restore the contents of the scratch register
562  * (FIX_RIP_reg).
563  */
564 static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
565 {
566 	struct uprobe_task *utask = current->utask;
567 
568 	riprel_post_xol(auprobe, regs);
569 	if (auprobe->defparam.fixups & UPROBE_FIX_IP) {
570 		long correction = utask->vaddr - utask->xol_vaddr;
571 		regs->ip += correction;
572 	} else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {
573 		regs->sp += sizeof_long(); /* Pop incorrect return address */
574 		if (push_ret_address(regs, utask->vaddr + auprobe->defparam.ilen))
575 			return -ERESTART;
576 	}
577 	/* popf; tell the caller to not touch TF */
578 	if (auprobe->defparam.fixups & UPROBE_FIX_SETF)
579 		utask->autask.saved_tf = true;
580 
581 	return 0;
582 }
583 
584 static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
585 {
586 	riprel_post_xol(auprobe, regs);
587 }
588 
589 static struct uprobe_xol_ops default_xol_ops = {
590 	.pre_xol  = default_pre_xol_op,
591 	.post_xol = default_post_xol_op,
592 	.abort	  = default_abort_op,
593 };
594 
595 static bool branch_is_call(struct arch_uprobe *auprobe)
596 {
597 	return auprobe->branch.opc1 == 0xe8;
598 }
599 
600 #define CASE_COND					\
601 	COND(70, 71, XF(OF))				\
602 	COND(72, 73, XF(CF))				\
603 	COND(74, 75, XF(ZF))				\
604 	COND(78, 79, XF(SF))				\
605 	COND(7a, 7b, XF(PF))				\
606 	COND(76, 77, XF(CF) || XF(ZF))			\
607 	COND(7c, 7d, XF(SF) != XF(OF))			\
608 	COND(7e, 7f, XF(ZF) || XF(SF) != XF(OF))
609 
610 #define COND(op_y, op_n, expr)				\
611 	case 0x ## op_y: DO((expr) != 0)		\
612 	case 0x ## op_n: DO((expr) == 0)
613 
614 #define XF(xf)	(!!(flags & X86_EFLAGS_ ## xf))
615 
616 static bool is_cond_jmp_opcode(u8 opcode)
617 {
618 	switch (opcode) {
619 	#define DO(expr)	\
620 		return true;
621 	CASE_COND
622 	#undef	DO
623 
624 	default:
625 		return false;
626 	}
627 }
628 
629 static bool check_jmp_cond(struct arch_uprobe *auprobe, struct pt_regs *regs)
630 {
631 	unsigned long flags = regs->flags;
632 
633 	switch (auprobe->branch.opc1) {
634 	#define DO(expr)	\
635 		return expr;
636 	CASE_COND
637 	#undef	DO
638 
639 	default:	/* not a conditional jmp */
640 		return true;
641 	}
642 }
643 
644 #undef	XF
645 #undef	COND
646 #undef	CASE_COND
647 
648 static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
649 {
650 	unsigned long new_ip = regs->ip += auprobe->branch.ilen;
651 	unsigned long offs = (long)auprobe->branch.offs;
652 
653 	if (branch_is_call(auprobe)) {
654 		/*
655 		 * If it fails we execute this (mangled, see the comment in
656 		 * branch_clear_offset) insn out-of-line. In the likely case
657 		 * this should trigger the trap, and the probed application
658 		 * should die or restart the same insn after it handles the
659 		 * signal, arch_uprobe_post_xol() won't be even called.
660 		 *
661 		 * But there is corner case, see the comment in ->post_xol().
662 		 */
663 		if (push_ret_address(regs, new_ip))
664 			return false;
665 	} else if (!check_jmp_cond(auprobe, regs)) {
666 		offs = 0;
667 	}
668 
669 	regs->ip = new_ip + offs;
670 	return true;
671 }
672 
673 static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
674 {
675 	BUG_ON(!branch_is_call(auprobe));
676 	/*
677 	 * We can only get here if branch_emulate_op() failed to push the ret
678 	 * address _and_ another thread expanded our stack before the (mangled)
679 	 * "call" insn was executed out-of-line. Just restore ->sp and restart.
680 	 * We could also restore ->ip and try to call branch_emulate_op() again.
681 	 */
682 	regs->sp += sizeof_long();
683 	return -ERESTART;
684 }
685 
686 static void branch_clear_offset(struct arch_uprobe *auprobe, struct insn *insn)
687 {
688 	/*
689 	 * Turn this insn into "call 1f; 1:", this is what we will execute
690 	 * out-of-line if ->emulate() fails. We only need this to generate
691 	 * a trap, so that the probed task receives the correct signal with
692 	 * the properly filled siginfo.
693 	 *
694 	 * But see the comment in ->post_xol(), in the unlikely case it can
695 	 * succeed. So we need to ensure that the new ->ip can not fall into
696 	 * the non-canonical area and trigger #GP.
697 	 *
698 	 * We could turn it into (say) "pushf", but then we would need to
699 	 * divorce ->insn[] and ->ixol[]. We need to preserve the 1st byte
700 	 * of ->insn[] for set_orig_insn().
701 	 */
702 	memset(auprobe->insn + insn_offset_immediate(insn),
703 		0, insn->immediate.nbytes);
704 }
705 
706 static struct uprobe_xol_ops branch_xol_ops = {
707 	.emulate  = branch_emulate_op,
708 	.post_xol = branch_post_xol_op,
709 };
710 
711 /* Returns -ENOSYS if branch_xol_ops doesn't handle this insn */
712 static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
713 {
714 	u8 opc1 = OPCODE1(insn);
715 	int i;
716 
717 	switch (opc1) {
718 	case 0xeb:	/* jmp 8 */
719 	case 0xe9:	/* jmp 32 */
720 	case 0x90:	/* prefix* + nop; same as jmp with .offs = 0 */
721 		break;
722 
723 	case 0xe8:	/* call relative */
724 		branch_clear_offset(auprobe, insn);
725 		break;
726 
727 	case 0x0f:
728 		if (insn->opcode.nbytes != 2)
729 			return -ENOSYS;
730 		/*
731 		 * If it is a "near" conditional jmp, OPCODE2() - 0x10 matches
732 		 * OPCODE1() of the "short" jmp which checks the same condition.
733 		 */
734 		opc1 = OPCODE2(insn) - 0x10;
735 	default:
736 		if (!is_cond_jmp_opcode(opc1))
737 			return -ENOSYS;
738 	}
739 
740 	/*
741 	 * 16-bit overrides such as CALLW (66 e8 nn nn) are not supported.
742 	 * Intel and AMD behavior differ in 64-bit mode: Intel ignores 66 prefix.
743 	 * No one uses these insns, reject any branch insns with such prefix.
744 	 */
745 	for (i = 0; i < insn->prefixes.nbytes; i++) {
746 		if (insn->prefixes.bytes[i] == 0x66)
747 			return -ENOTSUPP;
748 	}
749 
750 	auprobe->branch.opc1 = opc1;
751 	auprobe->branch.ilen = insn->length;
752 	auprobe->branch.offs = insn->immediate.value;
753 
754 	auprobe->ops = &branch_xol_ops;
755 	return 0;
756 }
757 
758 /**
759  * arch_uprobe_analyze_insn - instruction analysis including validity and fixups.
760  * @mm: the probed address space.
761  * @arch_uprobe: the probepoint information.
762  * @addr: virtual address at which to install the probepoint
763  * Return 0 on success or a -ve number on error.
764  */
765 int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)
766 {
767 	struct insn insn;
768 	u8 fix_ip_or_call = UPROBE_FIX_IP;
769 	int ret;
770 
771 	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
772 	if (ret)
773 		return ret;
774 
775 	ret = branch_setup_xol_ops(auprobe, &insn);
776 	if (ret != -ENOSYS)
777 		return ret;
778 
779 	/*
780 	 * Figure out which fixups default_post_xol_op() will need to perform,
781 	 * and annotate defparam->fixups accordingly.
782 	 */
783 	switch (OPCODE1(&insn)) {
784 	case 0x9d:		/* popf */
785 		auprobe->defparam.fixups |= UPROBE_FIX_SETF;
786 		break;
787 	case 0xc3:		/* ret or lret -- ip is correct */
788 	case 0xcb:
789 	case 0xc2:
790 	case 0xca:
791 	case 0xea:		/* jmp absolute -- ip is correct */
792 		fix_ip_or_call = 0;
793 		break;
794 	case 0x9a:		/* call absolute - Fix return addr, not ip */
795 		fix_ip_or_call = UPROBE_FIX_CALL;
796 		break;
797 	case 0xff:
798 		switch (MODRM_REG(&insn)) {
799 		case 2: case 3:			/* call or lcall, indirect */
800 			fix_ip_or_call = UPROBE_FIX_CALL;
801 			break;
802 		case 4: case 5:			/* jmp or ljmp, indirect */
803 			fix_ip_or_call = 0;
804 			break;
805 		}
806 		/* fall through */
807 	default:
808 		riprel_analyze(auprobe, &insn);
809 	}
810 
811 	auprobe->defparam.ilen = insn.length;
812 	auprobe->defparam.fixups |= fix_ip_or_call;
813 
814 	auprobe->ops = &default_xol_ops;
815 	return 0;
816 }
817 
818 /*
819  * arch_uprobe_pre_xol - prepare to execute out of line.
820  * @auprobe: the probepoint information.
821  * @regs: reflects the saved user state of current task.
822  */
823 int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
824 {
825 	struct uprobe_task *utask = current->utask;
826 
827 	if (auprobe->ops->pre_xol) {
828 		int err = auprobe->ops->pre_xol(auprobe, regs);
829 		if (err)
830 			return err;
831 	}
832 
833 	regs->ip = utask->xol_vaddr;
834 	utask->autask.saved_trap_nr = current->thread.trap_nr;
835 	current->thread.trap_nr = UPROBE_TRAP_NR;
836 
837 	utask->autask.saved_tf = !!(regs->flags & X86_EFLAGS_TF);
838 	regs->flags |= X86_EFLAGS_TF;
839 	if (test_tsk_thread_flag(current, TIF_BLOCKSTEP))
840 		set_task_blockstep(current, false);
841 
842 	return 0;
843 }
844 
845 /*
846  * If xol insn itself traps and generates a signal(Say,
847  * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped
848  * instruction jumps back to its own address. It is assumed that anything
849  * like do_page_fault/do_trap/etc sets thread.trap_nr != -1.
850  *
851  * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr,
852  * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to
853  * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol().
854  */
855 bool arch_uprobe_xol_was_trapped(struct task_struct *t)
856 {
857 	if (t->thread.trap_nr != UPROBE_TRAP_NR)
858 		return true;
859 
860 	return false;
861 }
862 
863 /*
864  * Called after single-stepping. To avoid the SMP problems that can
865  * occur when we temporarily put back the original opcode to
866  * single-step, we single-stepped a copy of the instruction.
867  *
868  * This function prepares to resume execution after the single-step.
869  */
870 int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
871 {
872 	struct uprobe_task *utask = current->utask;
873 	bool send_sigtrap = utask->autask.saved_tf;
874 	int err = 0;
875 
876 	WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR);
877 	current->thread.trap_nr = utask->autask.saved_trap_nr;
878 
879 	if (auprobe->ops->post_xol) {
880 		err = auprobe->ops->post_xol(auprobe, regs);
881 		if (err) {
882 			/*
883 			 * Restore ->ip for restart or post mortem analysis.
884 			 * ->post_xol() must not return -ERESTART unless this
885 			 * is really possible.
886 			 */
887 			regs->ip = utask->vaddr;
888 			if (err == -ERESTART)
889 				err = 0;
890 			send_sigtrap = false;
891 		}
892 	}
893 	/*
894 	 * arch_uprobe_pre_xol() doesn't save the state of TIF_BLOCKSTEP
895 	 * so we can get an extra SIGTRAP if we do not clear TF. We need
896 	 * to examine the opcode to make it right.
897 	 */
898 	if (send_sigtrap)
899 		send_sig(SIGTRAP, current, 0);
900 
901 	if (!utask->autask.saved_tf)
902 		regs->flags &= ~X86_EFLAGS_TF;
903 
904 	return err;
905 }
906 
907 /* callback routine for handling exceptions. */
908 int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data)
909 {
910 	struct die_args *args = data;
911 	struct pt_regs *regs = args->regs;
912 	int ret = NOTIFY_DONE;
913 
914 	/* We are only interested in userspace traps */
915 	if (regs && !user_mode(regs))
916 		return NOTIFY_DONE;
917 
918 	switch (val) {
919 	case DIE_INT3:
920 		if (uprobe_pre_sstep_notifier(regs))
921 			ret = NOTIFY_STOP;
922 
923 		break;
924 
925 	case DIE_DEBUG:
926 		if (uprobe_post_sstep_notifier(regs))
927 			ret = NOTIFY_STOP;
928 
929 	default:
930 		break;
931 	}
932 
933 	return ret;
934 }
935 
936 /*
937  * This function gets called when XOL instruction either gets trapped or
938  * the thread has a fatal signal. Reset the instruction pointer to its
939  * probed address for the potential restart or for post mortem analysis.
940  */
941 void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
942 {
943 	struct uprobe_task *utask = current->utask;
944 
945 	if (auprobe->ops->abort)
946 		auprobe->ops->abort(auprobe, regs);
947 
948 	current->thread.trap_nr = utask->autask.saved_trap_nr;
949 	regs->ip = utask->vaddr;
950 	/* clear TF if it was set by us in arch_uprobe_pre_xol() */
951 	if (!utask->autask.saved_tf)
952 		regs->flags &= ~X86_EFLAGS_TF;
953 }
954 
955 static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
956 {
957 	if (auprobe->ops->emulate)
958 		return auprobe->ops->emulate(auprobe, regs);
959 	return false;
960 }
961 
962 bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
963 {
964 	bool ret = __skip_sstep(auprobe, regs);
965 	if (ret && (regs->flags & X86_EFLAGS_TF))
966 		send_sig(SIGTRAP, current, 0);
967 	return ret;
968 }
969 
970 unsigned long
971 arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs *regs)
972 {
973 	int rasize = sizeof_long(), nleft;
974 	unsigned long orig_ret_vaddr = 0; /* clear high bits for 32-bit apps */
975 
976 	if (copy_from_user(&orig_ret_vaddr, (void __user *)regs->sp, rasize))
977 		return -1;
978 
979 	/* check whether address has been already hijacked */
980 	if (orig_ret_vaddr == trampoline_vaddr)
981 		return orig_ret_vaddr;
982 
983 	nleft = copy_to_user((void __user *)regs->sp, &trampoline_vaddr, rasize);
984 	if (likely(!nleft))
985 		return orig_ret_vaddr;
986 
987 	if (nleft != rasize) {
988 		pr_err("uprobe: return address clobbered: pid=%d, %%sp=%#lx, "
989 			"%%ip=%#lx\n", current->pid, regs->sp, regs->ip);
990 
991 		force_sig_info(SIGSEGV, SEND_SIG_FORCED, current);
992 	}
993 
994 	return -1;
995 }
996