xref: /freebsd/sys/amd64/amd64/bpf_jit_machdep.h (revision 95ee2897e98f5d444f26ed2334cc7c439f9c16c6)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
5  * Copyright (C) 2005-2016 Jung-uk Kim <jkim@FreeBSD.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the distribution.
17  * 3. Neither the name of the Politecnico di Torino nor the names of its
18  * contributors may be used to endorse or promote products derived from
19  * this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #ifndef _BPF_JIT_MACHDEP_H_
35 #define _BPF_JIT_MACHDEP_H_
36 
37 /*
38  * Registers
39  */
40 #define RAX	0
41 #define RCX	1
42 #define RDX	2
43 #define RBX	3
44 #define RSP	4
45 #define RBP	5
46 #define RSI	6
47 #define RDI	7
48 #define R8	0
49 #define R9	1
50 #define R10	2
51 #define R11	3
52 #define R12	4
53 #define R13	5
54 #define R14	6
55 #define R15	7
56 
57 #define EAX	0
58 #define ECX	1
59 #define EDX	2
60 #define EBX	3
61 #define ESP	4
62 #define EBP	5
63 #define ESI	6
64 #define EDI	7
65 #define R8D	0
66 #define R9D	1
67 #define R10D	2
68 #define R11D	3
69 #define R12D	4
70 #define R13D	5
71 #define R14D	6
72 #define R15D	7
73 
74 #define AX	0
75 #define CX	1
76 #define DX	2
77 #define BX	3
78 #define SP	4
79 #define BP	5
80 #define SI	6
81 #define DI	7
82 
83 #define AL	0
84 #define CL	1
85 #define DL	2
86 #define BL	3
87 
88 /* Optimization flags */
89 #define	BPF_JIT_FRET	0x01
90 #define	BPF_JIT_FPKT	0x02
91 #define	BPF_JIT_FMEM	0x04
92 #define	BPF_JIT_FJMP	0x08
93 #define	BPF_JIT_FLEN	0x10
94 
95 #define	BPF_JIT_FLAG_ALL	\
96     (BPF_JIT_FPKT | BPF_JIT_FMEM | BPF_JIT_FJMP | BPF_JIT_FLEN)
97 
98 /* A stream of native binary code */
99 typedef struct bpf_bin_stream {
100 	/* Current native instruction pointer. */
101 	int		cur_ip;
102 
103 	/*
104 	 * Current BPF instruction pointer, i.e. position in
105 	 * the BPF program reached by the jitter.
106 	 */
107 	int		bpf_pc;
108 
109 	/* Instruction buffer, contains the generated native code. */
110 	char		*ibuf;
111 
112 	/* Jumps reference table. */
113 	u_int		*refs;
114 } bpf_bin_stream;
115 
116 /*
117  * Prototype of the emit functions.
118  *
119  * Different emit functions are used to create the reference table and
120  * to generate the actual filtering code. This allows to have simpler
121  * instruction macros.
122  * The first parameter is the stream that will receive the data.
123  * The second one is a variable containing the data.
124  * The third one is the length, that can be 1, 2, or 4 since it is possible
125  * to emit a byte, a short, or a word at a time.
126  */
127 typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
128 
129 /*
130  * Native instruction macros
131  */
132 
133 /* movl i32,r32 */
134 #define MOVid(i32, r32) do {						\
135 	emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1);		\
136 	emitm(&stream, i32, 4);						\
137 } while (0)
138 
139 /* movq i64,r64 */
140 #define MOViq(i64, r64) do {						\
141 	emitm(&stream, 0x48, 1);					\
142 	emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1);		\
143 	emitm(&stream, i64, 4);						\
144 	emitm(&stream, (i64 >> 32), 4);					\
145 } while (0)
146 
147 /* movl sr32,dr32 */
148 #define MOVrd(sr32, dr32) do {						\
149 	emitm(&stream, 0x89, 1);					\
150 	emitm(&stream,							\
151 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
152 } while (0)
153 
154 /* movl sr32,dr32 (dr32 = %r8-15d) */
155 #define MOVrd2(sr32, dr32) do {						\
156 	emitm(&stream, 0x8941, 2);					\
157 	emitm(&stream,							\
158 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
159 } while (0)
160 
161 /* movl sr32,dr32 (sr32 = %r8-15d) */
162 #define MOVrd3(sr32, dr32) do {						\
163 	emitm(&stream, 0x8944, 2);					\
164 	emitm(&stream,							\
165 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
166 } while (0)
167 
168 /* movq sr64,dr64 */
169 #define MOVrq(sr64, dr64) do {						\
170 	emitm(&stream, 0x8948, 2);					\
171 	emitm(&stream,							\
172 	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
173 } while (0)
174 
175 /* movq sr64,dr64 (dr64 = %r8-15) */
176 #define MOVrq2(sr64, dr64) do {						\
177 	emitm(&stream, 0x8949, 2);					\
178 	emitm(&stream,							\
179 	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
180 } while (0)
181 
182 /* movq sr64,dr64 (sr64 = %r8-15) */
183 #define MOVrq3(sr64, dr64) do {						\
184 	emitm(&stream, 0x894c, 2);					\
185 	emitm(&stream,							\
186 	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
187 } while (0)
188 
189 /* movl (sr64,or64,1),dr32 */
190 #define MOVobd(sr64, or64, dr32) do {					\
191 	emitm(&stream, 0x8b, 1);					\
192 	emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);			\
193 	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
194 } while (0)
195 
196 /* movw (sr64,or64,1),dr16 */
197 #define MOVobw(sr64, or64, dr16) do {					\
198 	emitm(&stream, 0x8b66, 2);					\
199 	emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1);			\
200 	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
201 } while (0)
202 
203 /* movb (sr64,or64,1),dr8 */
204 #define MOVobb(sr64, or64, dr8) do {					\
205 	emitm(&stream, 0x8a, 1);					\
206 	emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1);			\
207 	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
208 } while (0)
209 
210 /* movl sr32,(dr64,or64,1) */
211 #define MOVomd(sr32, dr64, or64) do {					\
212 	emitm(&stream, 0x89, 1);					\
213 	emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1);			\
214 	emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
215 } while (0)
216 
217 /* bswapl dr32 */
218 #define BSWAP(dr32) do {						\
219 	emitm(&stream, 0xf, 1);						\
220 	emitm(&stream, (0x19 << 3) | dr32, 1);				\
221 } while (0)
222 
223 /* xchgb %al,%ah */
224 #define SWAP_AX() do {							\
225 	emitm(&stream, 0xc486, 2);					\
226 } while (0)
227 
228 /* pushq r64 */
229 #define PUSH(r64) do {							\
230 	emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1);		\
231 } while (0)
232 
233 /* leaveq */
234 #define LEAVE() do {							\
235 	emitm(&stream, 0xc9, 1);					\
236 } while (0)
237 
238 /* retq */
239 #define RET() do {							\
240 	emitm(&stream, 0xc3, 1);					\
241 } while (0)
242 
243 /* addl sr32,dr32 */
244 #define ADDrd(sr32, dr32) do {						\
245 	emitm(&stream, 0x01, 1);					\
246 	emitm(&stream,							\
247 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
248 } while (0)
249 
250 /* addl i32,%eax */
251 #define ADD_EAXi(i32) do {						\
252 	emitm(&stream, 0x05, 1);					\
253 	emitm(&stream, i32, 4);						\
254 } while (0)
255 
256 /* addl i8,r32 */
257 #define ADDib(i8, r32) do {						\
258 	emitm(&stream, 0x83, 1);					\
259 	emitm(&stream, (24 << 3) | r32, 1);				\
260 	emitm(&stream, i8, 1);						\
261 } while (0)
262 
263 /* subl sr32,dr32 */
264 #define SUBrd(sr32, dr32) do {						\
265 	emitm(&stream, 0x29, 1);					\
266 	emitm(&stream,							\
267 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
268 } while (0)
269 
270 /* subl i32,%eax */
271 #define SUB_EAXi(i32) do {						\
272 	emitm(&stream, 0x2d, 1);					\
273 	emitm(&stream, i32, 4);						\
274 } while (0)
275 
276 /* subq i8,r64 */
277 #define SUBib(i8, r64) do {						\
278 	emitm(&stream, 0x8348, 2);					\
279 	emitm(&stream, (29 << 3) | (r64 & 0x7), 1);			\
280 	emitm(&stream, i8, 1);						\
281 } while (0)
282 
283 /* mull r32 */
284 #define MULrd(r32) do {							\
285 	emitm(&stream, 0xf7, 1);					\
286 	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
287 } while (0)
288 
289 /* divl r32 */
290 #define DIVrd(r32) do {							\
291 	emitm(&stream, 0xf7, 1);					\
292 	emitm(&stream, (15 << 4) | (r32 & 0x7), 1);			\
293 } while (0)
294 
295 /* andb i8,r8 */
296 #define ANDib(i8, r8) do {						\
297 	if (r8 == AL) {							\
298 		emitm(&stream, 0x24, 1);				\
299 	} else {							\
300 		emitm(&stream, 0x80, 1);				\
301 		emitm(&stream, (7 << 5) | r8, 1);			\
302 	}								\
303 	emitm(&stream, i8, 1);						\
304 } while (0)
305 
306 /* andl i32,r32 */
307 #define ANDid(i32, r32) do {						\
308 	if (r32 == EAX) {						\
309 		emitm(&stream, 0x25, 1);				\
310 	} else {							\
311 		emitm(&stream, 0x81, 1);				\
312 		emitm(&stream, (7 << 5) | r32, 1);			\
313 	}								\
314 	emitm(&stream, i32, 4);						\
315 } while (0)
316 
317 /* andl sr32,dr32 */
318 #define ANDrd(sr32, dr32) do {						\
319 	emitm(&stream, 0x21, 1);					\
320 	emitm(&stream,							\
321 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
322 } while (0)
323 
324 /* testl i32,r32 */
325 #define TESTid(i32, r32) do {						\
326 	if (r32 == EAX) {						\
327 		emitm(&stream, 0xa9, 1);				\
328 	} else {							\
329 		emitm(&stream, 0xf7, 1);				\
330 		emitm(&stream, (3 << 6) | r32, 1);			\
331 	}								\
332 	emitm(&stream, i32, 4);						\
333 } while (0)
334 
335 /* testl sr32,dr32 */
336 #define TESTrd(sr32, dr32) do {						\
337 	emitm(&stream, 0x85, 1);					\
338 	emitm(&stream,							\
339 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
340 } while (0)
341 
342 /* orl sr32,dr32 */
343 #define ORrd(sr32, dr32) do {						\
344 	emitm(&stream, 0x09, 1);					\
345 	emitm(&stream,							\
346 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
347 } while (0)
348 
349 /* orl i32,r32 */
350 #define ORid(i32, r32) do {						\
351 	if (r32 == EAX) {						\
352 		emitm(&stream, 0x0d, 1);				\
353 	} else {							\
354 		emitm(&stream, 0x81, 1);				\
355 		emitm(&stream, (25 << 3) | r32, 1);			\
356 	}								\
357 	emitm(&stream, i32, 4);						\
358 } while (0)
359 
360 /* xorl sr32,dr32 */
361 #define XORrd(sr32, dr32) do {						\
362 	emitm(&stream, 0x31, 1);					\
363 	emitm(&stream,							\
364 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
365 } while (0)
366 
367 /* xorl i32,r32 */
368 #define XORid(i32, r32) do {						\
369 	if (r32 == EAX) {						\
370 		emitm(&stream, 0x35, 1);				\
371 	} else {							\
372 		emitm(&stream, 0x81, 1);				\
373 		emitm(&stream, (25 << 3) | r32, 1);			\
374 	}								\
375 	emitm(&stream, i32, 4);						\
376 } while (0)
377 
378 /* shll i8,r32 */
379 #define SHLib(i8, r32) do {						\
380 	emitm(&stream, 0xc1, 1);					\
381 	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
382 	emitm(&stream, i8, 1);						\
383 } while (0)
384 
385 /* shll %cl,dr32 */
386 #define SHL_CLrb(dr32) do {						\
387 	emitm(&stream, 0xd3, 1);					\
388 	emitm(&stream, (7 << 5) | (dr32 & 0x7), 1);			\
389 } while (0)
390 
391 /* shrl i8,r32 */
392 #define SHRib(i8, r32) do {						\
393 	emitm(&stream, 0xc1, 1);					\
394 	emitm(&stream, (29 << 3) | (r32 & 0x7), 1);			\
395 	emitm(&stream, i8, 1);						\
396 } while (0)
397 
398 /* shrl %cl,dr32 */
399 #define SHR_CLrb(dr32) do {						\
400 	emitm(&stream, 0xd3, 1);					\
401 	emitm(&stream, (29 << 3) | (dr32 & 0x7), 1);			\
402 } while (0)
403 
404 /* negl r32 */
405 #define NEGd(r32) do {							\
406 	emitm(&stream, 0xf7, 1);					\
407 	emitm(&stream, (27 << 3) | (r32 & 0x7), 1);			\
408 } while (0)
409 
410 /* cmpl sr32,dr32 */
411 #define CMPrd(sr32, dr32) do {						\
412 	emitm(&stream, 0x39, 1);					\
413 	emitm(&stream,							\
414 	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
415 } while (0)
416 
417 /* cmpl i32,dr32 */
418 #define CMPid(i32, dr32) do {						\
419 	if (dr32 == EAX){						\
420 		emitm(&stream, 0x3d, 1);				\
421 		emitm(&stream, i32, 4);					\
422 	} else {							\
423 		emitm(&stream, 0x81, 1);				\
424 		emitm(&stream, (0x1f << 3) | (dr32 & 0x7), 1);		\
425 		emitm(&stream, i32, 4);					\
426 	}								\
427 } while (0)
428 
429 /* jb off8 */
430 #define JBb(off8) do {							\
431 	emitm(&stream, 0x72, 1);					\
432 	emitm(&stream, off8, 1);					\
433 } while (0)
434 
435 /* jae off8 */
436 #define JAEb(off8) do {							\
437 	emitm(&stream, 0x73, 1);					\
438 	emitm(&stream, off8, 1);					\
439 } while (0)
440 
441 /* jne off8 */
442 #define JNEb(off8) do {							\
443 	emitm(&stream, 0x75, 1);					\
444 	emitm(&stream, off8, 1);					\
445 } while (0)
446 
447 /* ja off8 */
448 #define JAb(off8) do {							\
449 	emitm(&stream, 0x77, 1);					\
450 	emitm(&stream, off8, 1);					\
451 } while (0)
452 
453 /* jmp off32 */
454 #define JMP(off32) do {							\
455 	emitm(&stream, 0xe9, 1);					\
456 	emitm(&stream, off32, 4);					\
457 } while (0)
458 
459 /* xorl r32,r32 */
460 #define ZEROrd(r32) do {						\
461 	emitm(&stream, 0x31, 1);					\
462 	emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1);	\
463 } while (0)
464 
465 /*
466  * Conditional long jumps
467  */
468 #define	JB	0x82
469 #define	JAE	0x83
470 #define	JE	0x84
471 #define	JNE	0x85
472 #define	JBE	0x86
473 #define	JA	0x87
474 
475 #define	JCC(t, f) do {							\
476 	if (ins->jt != 0 && ins->jf != 0) {				\
477 		/* 5 is the size of the following jmp */		\
478 		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
479 		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
480 		    stream.refs[stream.bpf_pc] + 5, 4);			\
481 		JMP(stream.refs[stream.bpf_pc + ins->jf] -		\
482 		    stream.refs[stream.bpf_pc]);			\
483 	} else if (ins->jt != 0) {					\
484 		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
485 		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
486 		    stream.refs[stream.bpf_pc], 4);			\
487 	} else {							\
488 		emitm(&stream, ((f) << 8) | 0x0f, 2);			\
489 		emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -	\
490 		    stream.refs[stream.bpf_pc], 4);			\
491 	}								\
492 } while (0)
493 
494 #define	JUMP(off) do {							\
495 	if ((off) != 0)							\
496 		JMP(stream.refs[stream.bpf_pc + (off)] -		\
497 		    stream.refs[stream.bpf_pc]);			\
498 } while (0)
499 
500 #endif	/* _BPF_JIT_MACHDEP_H_ */
501