xref: /linux/arch/arm/net/bpf_jit_32.c (revision bfd5bb6f90af092aa345b15cd78143956a13c2a8)
1 /*
2  * Just-In-Time compiler for eBPF filters on 32bit ARM
3  *
4  * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
5  * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms of the GNU General Public License as published by the
9  * Free Software Foundation; version 2 of the License.
10  */
11 
12 #include <linux/bpf.h>
13 #include <linux/bitops.h>
14 #include <linux/compiler.h>
15 #include <linux/errno.h>
16 #include <linux/filter.h>
17 #include <linux/netdevice.h>
18 #include <linux/string.h>
19 #include <linux/slab.h>
20 #include <linux/if_vlan.h>
21 
22 #include <asm/cacheflush.h>
23 #include <asm/hwcap.h>
24 #include <asm/opcodes.h>
25 
26 #include "bpf_jit_32.h"
27 
28 /*
29  * eBPF prog stack layout:
30  *
31  *                         high
32  * original ARM_SP =>     +-----+
33  *                        |     | callee saved registers
34  *                        +-----+ <= (BPF_FP + SCRATCH_SIZE)
35  *                        | ... | eBPF JIT scratch space
36  * eBPF fp register =>    +-----+
37  *   (BPF_FP)             | ... | eBPF prog stack
38  *                        +-----+
39  *                        |RSVD | JIT scratchpad
40  * current ARM_SP =>      +-----+ <= (BPF_FP - STACK_SIZE + SCRATCH_SIZE)
41  *                        |     |
42  *                        | ... | Function call stack
43  *                        |     |
44  *                        +-----+
45  *                          low
46  *
47  * The callee saved registers depends on whether frame pointers are enabled.
48  * With frame pointers (to be compliant with the ABI):
49  *
50  *                                high
51  * original ARM_SP =>     +------------------+ \
52  *                        |        pc        | |
53  * current ARM_FP =>      +------------------+ } callee saved registers
54  *                        |r4-r8,r10,fp,ip,lr| |
55  *                        +------------------+ /
56  *                                low
57  *
58  * Without frame pointers:
59  *
60  *                                high
61  * original ARM_SP =>     +------------------+
62  *                        | r4-r8,r10,fp,lr  | callee saved registers
63  * current ARM_FP =>      +------------------+
64  *                                low
65  *
66  * When popping registers off the stack at the end of a BPF function, we
67  * reference them via the current ARM_FP register.
68  */
69 #define CALLEE_MASK	(1 << ARM_R4 | 1 << ARM_R5 | 1 << ARM_R6 | \
70 			 1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R10 | \
71 			 1 << ARM_FP)
72 #define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR)
73 #define CALLEE_POP_MASK  (CALLEE_MASK | 1 << ARM_PC)
74 
75 #define STACK_OFFSET(k)	(k)
76 #define TMP_REG_1	(MAX_BPF_JIT_REG + 0)	/* TEMP Register 1 */
77 #define TMP_REG_2	(MAX_BPF_JIT_REG + 1)	/* TEMP Register 2 */
78 #define TCALL_CNT	(MAX_BPF_JIT_REG + 2)	/* Tail Call Count */
79 
80 #define FLAG_IMM_OVERFLOW	(1 << 0)
81 
82 /*
83  * Map eBPF registers to ARM 32bit registers or stack scratch space.
84  *
85  * 1. First argument is passed using the arm 32bit registers and rest of the
86  * arguments are passed on stack scratch space.
87  * 2. First callee-saved argument is mapped to arm 32 bit registers and rest
88  * arguments are mapped to scratch space on stack.
89  * 3. We need two 64 bit temp registers to do complex operations on eBPF
90  * registers.
91  *
92  * As the eBPF registers are all 64 bit registers and arm has only 32 bit
93  * registers, we have to map each eBPF registers with two arm 32 bit regs or
94  * scratch memory space and we have to build eBPF 64 bit register from those.
95  *
96  */
97 static const u8 bpf2a32[][2] = {
98 	/* return value from in-kernel function, and exit value from eBPF */
99 	[BPF_REG_0] = {ARM_R1, ARM_R0},
100 	/* arguments from eBPF program to in-kernel function */
101 	[BPF_REG_1] = {ARM_R3, ARM_R2},
102 	/* Stored on stack scratch space */
103 	[BPF_REG_2] = {STACK_OFFSET(0), STACK_OFFSET(4)},
104 	[BPF_REG_3] = {STACK_OFFSET(8), STACK_OFFSET(12)},
105 	[BPF_REG_4] = {STACK_OFFSET(16), STACK_OFFSET(20)},
106 	[BPF_REG_5] = {STACK_OFFSET(24), STACK_OFFSET(28)},
107 	/* callee saved registers that in-kernel function will preserve */
108 	[BPF_REG_6] = {ARM_R5, ARM_R4},
109 	/* Stored on stack scratch space */
110 	[BPF_REG_7] = {STACK_OFFSET(32), STACK_OFFSET(36)},
111 	[BPF_REG_8] = {STACK_OFFSET(40), STACK_OFFSET(44)},
112 	[BPF_REG_9] = {STACK_OFFSET(48), STACK_OFFSET(52)},
113 	/* Read only Frame Pointer to access Stack */
114 	[BPF_REG_FP] = {STACK_OFFSET(56), STACK_OFFSET(60)},
115 	/* Temporary Register for internal BPF JIT, can be used
116 	 * for constant blindings and others.
117 	 */
118 	[TMP_REG_1] = {ARM_R7, ARM_R6},
119 	[TMP_REG_2] = {ARM_R10, ARM_R8},
120 	/* Tail call count. Stored on stack scratch space. */
121 	[TCALL_CNT] = {STACK_OFFSET(64), STACK_OFFSET(68)},
122 	/* temporary register for blinding constants.
123 	 * Stored on stack scratch space.
124 	 */
125 	[BPF_REG_AX] = {STACK_OFFSET(72), STACK_OFFSET(76)},
126 };
127 
128 #define	dst_lo	dst[1]
129 #define dst_hi	dst[0]
130 #define src_lo	src[1]
131 #define src_hi	src[0]
132 
133 /*
134  * JIT Context:
135  *
136  * prog			:	bpf_prog
137  * idx			:	index of current last JITed instruction.
138  * prologue_bytes	:	bytes used in prologue.
139  * epilogue_offset	:	offset of epilogue starting.
140  * offsets		:	array of eBPF instruction offsets in
141  *				JITed code.
142  * target		:	final JITed code.
143  * epilogue_bytes	:	no of bytes used in epilogue.
144  * imm_count		:	no of immediate counts used for global
145  *				variables.
146  * imms			:	array of global variable addresses.
147  */
148 
149 struct jit_ctx {
150 	const struct bpf_prog *prog;
151 	unsigned int idx;
152 	unsigned int prologue_bytes;
153 	unsigned int epilogue_offset;
154 	u32 flags;
155 	u32 *offsets;
156 	u32 *target;
157 	u32 stack_size;
158 #if __LINUX_ARM_ARCH__ < 7
159 	u16 epilogue_bytes;
160 	u16 imm_count;
161 	u32 *imms;
162 #endif
163 };
164 
165 /*
166  * Wrappers which handle both OABI and EABI and assures Thumb2 interworking
167  * (where the assembly routines like __aeabi_uidiv could cause problems).
168  */
169 static u32 jit_udiv32(u32 dividend, u32 divisor)
170 {
171 	return dividend / divisor;
172 }
173 
174 static u32 jit_mod32(u32 dividend, u32 divisor)
175 {
176 	return dividend % divisor;
177 }
178 
179 static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
180 {
181 	inst |= (cond << 28);
182 	inst = __opcode_to_mem_arm(inst);
183 
184 	if (ctx->target != NULL)
185 		ctx->target[ctx->idx] = inst;
186 
187 	ctx->idx++;
188 }
189 
190 /*
191  * Emit an instruction that will be executed unconditionally.
192  */
193 static inline void emit(u32 inst, struct jit_ctx *ctx)
194 {
195 	_emit(ARM_COND_AL, inst, ctx);
196 }
197 
198 /*
199  * Checks if immediate value can be converted to imm12(12 bits) value.
200  */
201 static int16_t imm8m(u32 x)
202 {
203 	u32 rot;
204 
205 	for (rot = 0; rot < 16; rot++)
206 		if ((x & ~ror32(0xff, 2 * rot)) == 0)
207 			return rol32(x, 2 * rot) | (rot << 8);
208 	return -1;
209 }
210 
211 /*
212  * Initializes the JIT space with undefined instructions.
213  */
214 static void jit_fill_hole(void *area, unsigned int size)
215 {
216 	u32 *ptr;
217 	/* We are guaranteed to have aligned memory. */
218 	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
219 		*ptr++ = __opcode_to_mem_arm(ARM_INST_UDF);
220 }
221 
222 #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
223 /* EABI requires the stack to be aligned to 64-bit boundaries */
224 #define STACK_ALIGNMENT	8
225 #else
226 /* Stack must be aligned to 32-bit boundaries */
227 #define STACK_ALIGNMENT	4
228 #endif
229 
230 /* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4,
231  * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9,
232  * BPF_REG_FP and Tail call counts.
233  */
234 #define SCRATCH_SIZE 80
235 
236 /* total stack size used in JITed code */
237 #define _STACK_SIZE	(ctx->prog->aux->stack_depth + SCRATCH_SIZE)
238 #define STACK_SIZE	ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
239 
240 /* Get the offset of eBPF REGISTERs stored on scratch space. */
241 #define STACK_VAR(off) (STACK_SIZE - off)
242 
243 #if __LINUX_ARM_ARCH__ < 7
244 
245 static u16 imm_offset(u32 k, struct jit_ctx *ctx)
246 {
247 	unsigned int i = 0, offset;
248 	u16 imm;
249 
250 	/* on the "fake" run we just count them (duplicates included) */
251 	if (ctx->target == NULL) {
252 		ctx->imm_count++;
253 		return 0;
254 	}
255 
256 	while ((i < ctx->imm_count) && ctx->imms[i]) {
257 		if (ctx->imms[i] == k)
258 			break;
259 		i++;
260 	}
261 
262 	if (ctx->imms[i] == 0)
263 		ctx->imms[i] = k;
264 
265 	/* constants go just after the epilogue */
266 	offset =  ctx->offsets[ctx->prog->len - 1] * 4;
267 	offset += ctx->prologue_bytes;
268 	offset += ctx->epilogue_bytes;
269 	offset += i * 4;
270 
271 	ctx->target[offset / 4] = k;
272 
273 	/* PC in ARM mode == address of the instruction + 8 */
274 	imm = offset - (8 + ctx->idx * 4);
275 
276 	if (imm & ~0xfff) {
277 		/*
278 		 * literal pool is too far, signal it into flags. we
279 		 * can only detect it on the second pass unfortunately.
280 		 */
281 		ctx->flags |= FLAG_IMM_OVERFLOW;
282 		return 0;
283 	}
284 
285 	return imm;
286 }
287 
288 #endif /* __LINUX_ARM_ARCH__ */
289 
290 static inline int bpf2a32_offset(int bpf_to, int bpf_from,
291 				 const struct jit_ctx *ctx) {
292 	int to, from;
293 
294 	if (ctx->target == NULL)
295 		return 0;
296 	to = ctx->offsets[bpf_to];
297 	from = ctx->offsets[bpf_from];
298 
299 	return to - from - 1;
300 }
301 
302 /*
303  * Move an immediate that's not an imm8m to a core register.
304  */
305 static inline void emit_mov_i_no8m(const u8 rd, u32 val, struct jit_ctx *ctx)
306 {
307 #if __LINUX_ARM_ARCH__ < 7
308 	emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx);
309 #else
310 	emit(ARM_MOVW(rd, val & 0xffff), ctx);
311 	if (val > 0xffff)
312 		emit(ARM_MOVT(rd, val >> 16), ctx);
313 #endif
314 }
315 
316 static inline void emit_mov_i(const u8 rd, u32 val, struct jit_ctx *ctx)
317 {
318 	int imm12 = imm8m(val);
319 
320 	if (imm12 >= 0)
321 		emit(ARM_MOV_I(rd, imm12), ctx);
322 	else
323 		emit_mov_i_no8m(rd, val, ctx);
324 }
325 
326 static void emit_bx_r(u8 tgt_reg, struct jit_ctx *ctx)
327 {
328 	if (elf_hwcap & HWCAP_THUMB)
329 		emit(ARM_BX(tgt_reg), ctx);
330 	else
331 		emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx);
332 }
333 
334 static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx)
335 {
336 #if __LINUX_ARM_ARCH__ < 5
337 	emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx);
338 	emit_bx_r(tgt_reg, ctx);
339 #else
340 	emit(ARM_BLX_R(tgt_reg), ctx);
341 #endif
342 }
343 
344 static inline int epilogue_offset(const struct jit_ctx *ctx)
345 {
346 	int to, from;
347 	/* No need for 1st dummy run */
348 	if (ctx->target == NULL)
349 		return 0;
350 	to = ctx->epilogue_offset;
351 	from = ctx->idx;
352 
353 	return to - from - 2;
354 }
355 
356 static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op)
357 {
358 	const u8 *tmp = bpf2a32[TMP_REG_1];
359 
360 #if __LINUX_ARM_ARCH__ == 7
361 	if (elf_hwcap & HWCAP_IDIVA) {
362 		if (op == BPF_DIV)
363 			emit(ARM_UDIV(rd, rm, rn), ctx);
364 		else {
365 			emit(ARM_UDIV(ARM_IP, rm, rn), ctx);
366 			emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx);
367 		}
368 		return;
369 	}
370 #endif
371 
372 	/*
373 	 * For BPF_ALU | BPF_DIV | BPF_K instructions
374 	 * As ARM_R1 and ARM_R0 contains 1st argument of bpf
375 	 * function, we need to save it on caller side to save
376 	 * it from getting destroyed within callee.
377 	 * After the return from the callee, we restore ARM_R0
378 	 * ARM_R1.
379 	 */
380 	if (rn != ARM_R1) {
381 		emit(ARM_MOV_R(tmp[0], ARM_R1), ctx);
382 		emit(ARM_MOV_R(ARM_R1, rn), ctx);
383 	}
384 	if (rm != ARM_R0) {
385 		emit(ARM_MOV_R(tmp[1], ARM_R0), ctx);
386 		emit(ARM_MOV_R(ARM_R0, rm), ctx);
387 	}
388 
389 	/* Call appropriate function */
390 	emit_mov_i(ARM_IP, op == BPF_DIV ?
391 		   (u32)jit_udiv32 : (u32)jit_mod32, ctx);
392 	emit_blx_r(ARM_IP, ctx);
393 
394 	/* Save return value */
395 	if (rd != ARM_R0)
396 		emit(ARM_MOV_R(rd, ARM_R0), ctx);
397 
398 	/* Restore ARM_R0 and ARM_R1 */
399 	if (rn != ARM_R1)
400 		emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx);
401 	if (rm != ARM_R0)
402 		emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx);
403 }
404 
405 /* Checks whether BPF register is on scratch stack space or not. */
406 static inline bool is_on_stack(u8 bpf_reg)
407 {
408 	static u8 stack_regs[] = {BPF_REG_AX, BPF_REG_3, BPF_REG_4, BPF_REG_5,
409 				BPF_REG_7, BPF_REG_8, BPF_REG_9, TCALL_CNT,
410 				BPF_REG_2, BPF_REG_FP};
411 	int i, reg_len = sizeof(stack_regs);
412 
413 	for (i = 0 ; i < reg_len ; i++) {
414 		if (bpf_reg == stack_regs[i])
415 			return true;
416 	}
417 	return false;
418 }
419 
420 static inline void emit_a32_mov_i(const u8 dst, const u32 val,
421 				  bool dstk, struct jit_ctx *ctx)
422 {
423 	const u8 *tmp = bpf2a32[TMP_REG_1];
424 
425 	if (dstk) {
426 		emit_mov_i(tmp[1], val, ctx);
427 		emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(dst)), ctx);
428 	} else {
429 		emit_mov_i(dst, val, ctx);
430 	}
431 }
432 
433 /* Sign extended move */
434 static inline void emit_a32_mov_i64(const bool is64, const u8 dst[],
435 				  const u32 val, bool dstk,
436 				  struct jit_ctx *ctx) {
437 	u32 hi = 0;
438 
439 	if (is64 && (val & (1<<31)))
440 		hi = (u32)~0;
441 	emit_a32_mov_i(dst_lo, val, dstk, ctx);
442 	emit_a32_mov_i(dst_hi, hi, dstk, ctx);
443 }
444 
445 static inline void emit_a32_add_r(const u8 dst, const u8 src,
446 			      const bool is64, const bool hi,
447 			      struct jit_ctx *ctx) {
448 	/* 64 bit :
449 	 *	adds dst_lo, dst_lo, src_lo
450 	 *	adc dst_hi, dst_hi, src_hi
451 	 * 32 bit :
452 	 *	add dst_lo, dst_lo, src_lo
453 	 */
454 	if (!hi && is64)
455 		emit(ARM_ADDS_R(dst, dst, src), ctx);
456 	else if (hi && is64)
457 		emit(ARM_ADC_R(dst, dst, src), ctx);
458 	else
459 		emit(ARM_ADD_R(dst, dst, src), ctx);
460 }
461 
462 static inline void emit_a32_sub_r(const u8 dst, const u8 src,
463 				  const bool is64, const bool hi,
464 				  struct jit_ctx *ctx) {
465 	/* 64 bit :
466 	 *	subs dst_lo, dst_lo, src_lo
467 	 *	sbc dst_hi, dst_hi, src_hi
468 	 * 32 bit :
469 	 *	sub dst_lo, dst_lo, src_lo
470 	 */
471 	if (!hi && is64)
472 		emit(ARM_SUBS_R(dst, dst, src), ctx);
473 	else if (hi && is64)
474 		emit(ARM_SBC_R(dst, dst, src), ctx);
475 	else
476 		emit(ARM_SUB_R(dst, dst, src), ctx);
477 }
478 
479 static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64,
480 			      const bool hi, const u8 op, struct jit_ctx *ctx){
481 	switch (BPF_OP(op)) {
482 	/* dst = dst + src */
483 	case BPF_ADD:
484 		emit_a32_add_r(dst, src, is64, hi, ctx);
485 		break;
486 	/* dst = dst - src */
487 	case BPF_SUB:
488 		emit_a32_sub_r(dst, src, is64, hi, ctx);
489 		break;
490 	/* dst = dst | src */
491 	case BPF_OR:
492 		emit(ARM_ORR_R(dst, dst, src), ctx);
493 		break;
494 	/* dst = dst & src */
495 	case BPF_AND:
496 		emit(ARM_AND_R(dst, dst, src), ctx);
497 		break;
498 	/* dst = dst ^ src */
499 	case BPF_XOR:
500 		emit(ARM_EOR_R(dst, dst, src), ctx);
501 		break;
502 	/* dst = dst * src */
503 	case BPF_MUL:
504 		emit(ARM_MUL(dst, dst, src), ctx);
505 		break;
506 	/* dst = dst << src */
507 	case BPF_LSH:
508 		emit(ARM_LSL_R(dst, dst, src), ctx);
509 		break;
510 	/* dst = dst >> src */
511 	case BPF_RSH:
512 		emit(ARM_LSR_R(dst, dst, src), ctx);
513 		break;
514 	/* dst = dst >> src (signed)*/
515 	case BPF_ARSH:
516 		emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx);
517 		break;
518 	}
519 }
520 
521 /* ALU operation (32 bit)
522  * dst = dst (op) src
523  */
524 static inline void emit_a32_alu_r(const u8 dst, const u8 src,
525 				  bool dstk, bool sstk,
526 				  struct jit_ctx *ctx, const bool is64,
527 				  const bool hi, const u8 op) {
528 	const u8 *tmp = bpf2a32[TMP_REG_1];
529 	u8 rn = sstk ? tmp[1] : src;
530 
531 	if (sstk)
532 		emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx);
533 
534 	/* ALU operation */
535 	if (dstk) {
536 		emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
537 		emit_alu_r(tmp[0], rn, is64, hi, op, ctx);
538 		emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
539 	} else {
540 		emit_alu_r(dst, rn, is64, hi, op, ctx);
541 	}
542 }
543 
544 /* ALU operation (64 bit) */
545 static inline void emit_a32_alu_r64(const bool is64, const u8 dst[],
546 				  const u8 src[], bool dstk,
547 				  bool sstk, struct jit_ctx *ctx,
548 				  const u8 op) {
549 	emit_a32_alu_r(dst_lo, src_lo, dstk, sstk, ctx, is64, false, op);
550 	if (is64)
551 		emit_a32_alu_r(dst_hi, src_hi, dstk, sstk, ctx, is64, true, op);
552 	else
553 		emit_a32_mov_i(dst_hi, 0, dstk, ctx);
554 }
555 
556 /* dst = imm (4 bytes)*/
557 static inline void emit_a32_mov_r(const u8 dst, const u8 src,
558 				  bool dstk, bool sstk,
559 				  struct jit_ctx *ctx) {
560 	const u8 *tmp = bpf2a32[TMP_REG_1];
561 	u8 rt = sstk ? tmp[0] : src;
562 
563 	if (sstk)
564 		emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx);
565 	if (dstk)
566 		emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst)), ctx);
567 	else
568 		emit(ARM_MOV_R(dst, rt), ctx);
569 }
570 
571 /* dst = src */
572 static inline void emit_a32_mov_r64(const bool is64, const u8 dst[],
573 				  const u8 src[], bool dstk,
574 				  bool sstk, struct jit_ctx *ctx) {
575 	emit_a32_mov_r(dst_lo, src_lo, dstk, sstk, ctx);
576 	if (is64) {
577 		/* complete 8 byte move */
578 		emit_a32_mov_r(dst_hi, src_hi, dstk, sstk, ctx);
579 	} else {
580 		/* Zero out high 4 bytes */
581 		emit_a32_mov_i(dst_hi, 0, dstk, ctx);
582 	}
583 }
584 
585 /* Shift operations */
586 static inline void emit_a32_alu_i(const u8 dst, const u32 val, bool dstk,
587 				struct jit_ctx *ctx, const u8 op) {
588 	const u8 *tmp = bpf2a32[TMP_REG_1];
589 	u8 rd = dstk ? tmp[0] : dst;
590 
591 	if (dstk)
592 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
593 
594 	/* Do shift operation */
595 	switch (op) {
596 	case BPF_LSH:
597 		emit(ARM_LSL_I(rd, rd, val), ctx);
598 		break;
599 	case BPF_RSH:
600 		emit(ARM_LSR_I(rd, rd, val), ctx);
601 		break;
602 	case BPF_NEG:
603 		emit(ARM_RSB_I(rd, rd, val), ctx);
604 		break;
605 	}
606 
607 	if (dstk)
608 		emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
609 }
610 
611 /* dst = ~dst (64 bit) */
612 static inline void emit_a32_neg64(const u8 dst[], bool dstk,
613 				struct jit_ctx *ctx){
614 	const u8 *tmp = bpf2a32[TMP_REG_1];
615 	u8 rd = dstk ? tmp[1] : dst[1];
616 	u8 rm = dstk ? tmp[0] : dst[0];
617 
618 	/* Setup Operand */
619 	if (dstk) {
620 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
621 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
622 	}
623 
624 	/* Do Negate Operation */
625 	emit(ARM_RSBS_I(rd, rd, 0), ctx);
626 	emit(ARM_RSC_I(rm, rm, 0), ctx);
627 
628 	if (dstk) {
629 		emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
630 		emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
631 	}
632 }
633 
634 /* dst = dst << src */
635 static inline void emit_a32_lsh_r64(const u8 dst[], const u8 src[], bool dstk,
636 				    bool sstk, struct jit_ctx *ctx) {
637 	const u8 *tmp = bpf2a32[TMP_REG_1];
638 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
639 
640 	/* Setup Operands */
641 	u8 rt = sstk ? tmp2[1] : src_lo;
642 	u8 rd = dstk ? tmp[1] : dst_lo;
643 	u8 rm = dstk ? tmp[0] : dst_hi;
644 
645 	if (sstk)
646 		emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
647 	if (dstk) {
648 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
649 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
650 	}
651 
652 	/* Do LSH operation */
653 	emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);
654 	emit(ARM_RSB_I(tmp2[0], rt, 32), ctx);
655 	emit(ARM_MOV_SR(ARM_LR, rm, SRTYPE_ASL, rt), ctx);
656 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd, SRTYPE_ASL, ARM_IP), ctx);
657 	emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx);
658 	emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx);
659 
660 	if (dstk) {
661 		emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
662 		emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
663 	} else {
664 		emit(ARM_MOV_R(rd, ARM_LR), ctx);
665 		emit(ARM_MOV_R(rm, ARM_IP), ctx);
666 	}
667 }
668 
669 /* dst = dst >> src (signed)*/
670 static inline void emit_a32_arsh_r64(const u8 dst[], const u8 src[], bool dstk,
671 				    bool sstk, struct jit_ctx *ctx) {
672 	const u8 *tmp = bpf2a32[TMP_REG_1];
673 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
674 	/* Setup Operands */
675 	u8 rt = sstk ? tmp2[1] : src_lo;
676 	u8 rd = dstk ? tmp[1] : dst_lo;
677 	u8 rm = dstk ? tmp[0] : dst_hi;
678 
679 	if (sstk)
680 		emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
681 	if (dstk) {
682 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
683 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
684 	}
685 
686 	/* Do the ARSH operation */
687 	emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
688 	emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
689 	emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx);
690 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
691 	_emit(ARM_COND_MI, ARM_B(0), ctx);
692 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx);
693 	emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx);
694 	if (dstk) {
695 		emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
696 		emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
697 	} else {
698 		emit(ARM_MOV_R(rd, ARM_LR), ctx);
699 		emit(ARM_MOV_R(rm, ARM_IP), ctx);
700 	}
701 }
702 
703 /* dst = dst >> src */
704 static inline void emit_a32_rsh_r64(const u8 dst[], const u8 src[], bool dstk,
705 				     bool sstk, struct jit_ctx *ctx) {
706 	const u8 *tmp = bpf2a32[TMP_REG_1];
707 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
708 	/* Setup Operands */
709 	u8 rt = sstk ? tmp2[1] : src_lo;
710 	u8 rd = dstk ? tmp[1] : dst_lo;
711 	u8 rm = dstk ? tmp[0] : dst_hi;
712 
713 	if (sstk)
714 		emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
715 	if (dstk) {
716 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
717 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
718 	}
719 
720 	/* Do RSH operation */
721 	emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
722 	emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
723 	emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx);
724 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
725 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx);
726 	emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx);
727 	if (dstk) {
728 		emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
729 		emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
730 	} else {
731 		emit(ARM_MOV_R(rd, ARM_LR), ctx);
732 		emit(ARM_MOV_R(rm, ARM_IP), ctx);
733 	}
734 }
735 
736 /* dst = dst << val */
737 static inline void emit_a32_lsh_i64(const u8 dst[], bool dstk,
738 				     const u32 val, struct jit_ctx *ctx){
739 	const u8 *tmp = bpf2a32[TMP_REG_1];
740 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
741 	/* Setup operands */
742 	u8 rd = dstk ? tmp[1] : dst_lo;
743 	u8 rm = dstk ? tmp[0] : dst_hi;
744 
745 	if (dstk) {
746 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
747 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
748 	}
749 
750 	/* Do LSH operation */
751 	if (val < 32) {
752 		emit(ARM_MOV_SI(tmp2[0], rm, SRTYPE_ASL, val), ctx);
753 		emit(ARM_ORR_SI(rm, tmp2[0], rd, SRTYPE_LSR, 32 - val), ctx);
754 		emit(ARM_MOV_SI(rd, rd, SRTYPE_ASL, val), ctx);
755 	} else {
756 		if (val == 32)
757 			emit(ARM_MOV_R(rm, rd), ctx);
758 		else
759 			emit(ARM_MOV_SI(rm, rd, SRTYPE_ASL, val - 32), ctx);
760 		emit(ARM_EOR_R(rd, rd, rd), ctx);
761 	}
762 
763 	if (dstk) {
764 		emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
765 		emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
766 	}
767 }
768 
769 /* dst = dst >> val */
770 static inline void emit_a32_rsh_i64(const u8 dst[], bool dstk,
771 				    const u32 val, struct jit_ctx *ctx) {
772 	const u8 *tmp = bpf2a32[TMP_REG_1];
773 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
774 	/* Setup operands */
775 	u8 rd = dstk ? tmp[1] : dst_lo;
776 	u8 rm = dstk ? tmp[0] : dst_hi;
777 
778 	if (dstk) {
779 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
780 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
781 	}
782 
783 	/* Do LSR operation */
784 	if (val < 32) {
785 		emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx);
786 		emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx);
787 		emit(ARM_MOV_SI(rm, rm, SRTYPE_LSR, val), ctx);
788 	} else if (val == 32) {
789 		emit(ARM_MOV_R(rd, rm), ctx);
790 		emit(ARM_MOV_I(rm, 0), ctx);
791 	} else {
792 		emit(ARM_MOV_SI(rd, rm, SRTYPE_LSR, val - 32), ctx);
793 		emit(ARM_MOV_I(rm, 0), ctx);
794 	}
795 
796 	if (dstk) {
797 		emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
798 		emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
799 	}
800 }
801 
802 /* dst = dst >> val (signed) */
803 static inline void emit_a32_arsh_i64(const u8 dst[], bool dstk,
804 				     const u32 val, struct jit_ctx *ctx){
805 	const u8 *tmp = bpf2a32[TMP_REG_1];
806 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
807 	 /* Setup operands */
808 	u8 rd = dstk ? tmp[1] : dst_lo;
809 	u8 rm = dstk ? tmp[0] : dst_hi;
810 
811 	if (dstk) {
812 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
813 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
814 	}
815 
816 	/* Do ARSH operation */
817 	if (val < 32) {
818 		emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx);
819 		emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx);
820 		emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, val), ctx);
821 	} else if (val == 32) {
822 		emit(ARM_MOV_R(rd, rm), ctx);
823 		emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
824 	} else {
825 		emit(ARM_MOV_SI(rd, rm, SRTYPE_ASR, val - 32), ctx);
826 		emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
827 	}
828 
829 	if (dstk) {
830 		emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
831 		emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
832 	}
833 }
834 
835 static inline void emit_a32_mul_r64(const u8 dst[], const u8 src[], bool dstk,
836 				    bool sstk, struct jit_ctx *ctx) {
837 	const u8 *tmp = bpf2a32[TMP_REG_1];
838 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
839 	/* Setup operands for multiplication */
840 	u8 rd = dstk ? tmp[1] : dst_lo;
841 	u8 rm = dstk ? tmp[0] : dst_hi;
842 	u8 rt = sstk ? tmp2[1] : src_lo;
843 	u8 rn = sstk ? tmp2[0] : src_hi;
844 
845 	if (dstk) {
846 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
847 		emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
848 	}
849 	if (sstk) {
850 		emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
851 		emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_hi)), ctx);
852 	}
853 
854 	/* Do Multiplication */
855 	emit(ARM_MUL(ARM_IP, rd, rn), ctx);
856 	emit(ARM_MUL(ARM_LR, rm, rt), ctx);
857 	emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx);
858 
859 	emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx);
860 	emit(ARM_ADD_R(rm, ARM_LR, rm), ctx);
861 	if (dstk) {
862 		emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_lo)), ctx);
863 		emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
864 	} else {
865 		emit(ARM_MOV_R(rd, ARM_IP), ctx);
866 	}
867 }
868 
869 /* *(size *)(dst + off) = src */
870 static inline void emit_str_r(const u8 dst, const u8 src, bool dstk,
871 			      const s32 off, struct jit_ctx *ctx, const u8 sz){
872 	const u8 *tmp = bpf2a32[TMP_REG_1];
873 	u8 rd = dstk ? tmp[1] : dst;
874 
875 	if (dstk)
876 		emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
877 	if (off) {
878 		emit_a32_mov_i(tmp[0], off, false, ctx);
879 		emit(ARM_ADD_R(tmp[0], rd, tmp[0]), ctx);
880 		rd = tmp[0];
881 	}
882 	switch (sz) {
883 	case BPF_W:
884 		/* Store a Word */
885 		emit(ARM_STR_I(src, rd, 0), ctx);
886 		break;
887 	case BPF_H:
888 		/* Store a HalfWord */
889 		emit(ARM_STRH_I(src, rd, 0), ctx);
890 		break;
891 	case BPF_B:
892 		/* Store a Byte */
893 		emit(ARM_STRB_I(src, rd, 0), ctx);
894 		break;
895 	}
896 }
897 
898 /* dst = *(size*)(src + off) */
899 static inline void emit_ldx_r(const u8 dst[], const u8 src, bool dstk,
900 			      s32 off, struct jit_ctx *ctx, const u8 sz){
901 	const u8 *tmp = bpf2a32[TMP_REG_1];
902 	const u8 *rd = dstk ? tmp : dst;
903 	u8 rm = src;
904 	s32 off_max;
905 
906 	if (sz == BPF_H)
907 		off_max = 0xff;
908 	else
909 		off_max = 0xfff;
910 
911 	if (off < 0 || off > off_max) {
912 		emit_a32_mov_i(tmp[0], off, false, ctx);
913 		emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);
914 		rm = tmp[0];
915 		off = 0;
916 	} else if (rd[1] == rm) {
917 		emit(ARM_MOV_R(tmp[0], rm), ctx);
918 		rm = tmp[0];
919 	}
920 	switch (sz) {
921 	case BPF_B:
922 		/* Load a Byte */
923 		emit(ARM_LDRB_I(rd[1], rm, off), ctx);
924 		emit_a32_mov_i(dst[0], 0, dstk, ctx);
925 		break;
926 	case BPF_H:
927 		/* Load a HalfWord */
928 		emit(ARM_LDRH_I(rd[1], rm, off), ctx);
929 		emit_a32_mov_i(dst[0], 0, dstk, ctx);
930 		break;
931 	case BPF_W:
932 		/* Load a Word */
933 		emit(ARM_LDR_I(rd[1], rm, off), ctx);
934 		emit_a32_mov_i(dst[0], 0, dstk, ctx);
935 		break;
936 	case BPF_DW:
937 		/* Load a Double Word */
938 		emit(ARM_LDR_I(rd[1], rm, off), ctx);
939 		emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
940 		break;
941 	}
942 	if (dstk)
943 		emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst[1])), ctx);
944 	if (dstk && sz == BPF_DW)
945 		emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst[0])), ctx);
946 }
947 
948 /* Arithmatic Operation */
949 static inline void emit_ar_r(const u8 rd, const u8 rt, const u8 rm,
950 			     const u8 rn, struct jit_ctx *ctx, u8 op) {
951 	switch (op) {
952 	case BPF_JSET:
953 		emit(ARM_AND_R(ARM_IP, rt, rn), ctx);
954 		emit(ARM_AND_R(ARM_LR, rd, rm), ctx);
955 		emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);
956 		break;
957 	case BPF_JEQ:
958 	case BPF_JNE:
959 	case BPF_JGT:
960 	case BPF_JGE:
961 	case BPF_JLE:
962 	case BPF_JLT:
963 		emit(ARM_CMP_R(rd, rm), ctx);
964 		_emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx);
965 		break;
966 	case BPF_JSLE:
967 	case BPF_JSGT:
968 		emit(ARM_CMP_R(rn, rt), ctx);
969 		emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);
970 		break;
971 	case BPF_JSLT:
972 	case BPF_JSGE:
973 		emit(ARM_CMP_R(rt, rn), ctx);
974 		emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);
975 		break;
976 	}
977 }
978 
979 static int out_offset = -1; /* initialized on the first pass of build_body() */
980 static int emit_bpf_tail_call(struct jit_ctx *ctx)
981 {
982 
983 	/* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */
984 	const u8 *r2 = bpf2a32[BPF_REG_2];
985 	const u8 *r3 = bpf2a32[BPF_REG_3];
986 	const u8 *tmp = bpf2a32[TMP_REG_1];
987 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
988 	const u8 *tcc = bpf2a32[TCALL_CNT];
989 	const int idx0 = ctx->idx;
990 #define cur_offset (ctx->idx - idx0)
991 #define jmp_offset (out_offset - (cur_offset) - 2)
992 	u32 off, lo, hi;
993 
994 	/* if (index >= array->map.max_entries)
995 	 *	goto out;
996 	 */
997 	off = offsetof(struct bpf_array, map.max_entries);
998 	/* array->map.max_entries */
999 	emit_a32_mov_i(tmp[1], off, false, ctx);
1000 	emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
1001 	emit(ARM_LDR_R(tmp[1], tmp2[1], tmp[1]), ctx);
1002 	/* index is 32-bit for arrays */
1003 	emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx);
1004 	/* index >= array->map.max_entries */
1005 	emit(ARM_CMP_R(tmp2[1], tmp[1]), ctx);
1006 	_emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1007 
1008 	/* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
1009 	 *	goto out;
1010 	 * tail_call_cnt++;
1011 	 */
1012 	lo = (u32)MAX_TAIL_CALL_CNT;
1013 	hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32);
1014 	emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx);
1015 	emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx);
1016 	emit(ARM_CMP_I(tmp[0], hi), ctx);
1017 	_emit(ARM_COND_EQ, ARM_CMP_I(tmp[1], lo), ctx);
1018 	_emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
1019 	emit(ARM_ADDS_I(tmp[1], tmp[1], 1), ctx);
1020 	emit(ARM_ADC_I(tmp[0], tmp[0], 0), ctx);
1021 	emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx);
1022 	emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx);
1023 
1024 	/* prog = array->ptrs[index]
1025 	 * if (prog == NULL)
1026 	 *	goto out;
1027 	 */
1028 	off = offsetof(struct bpf_array, ptrs);
1029 	emit_a32_mov_i(tmp[1], off, false, ctx);
1030 	emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
1031 	emit(ARM_ADD_R(tmp[1], tmp2[1], tmp[1]), ctx);
1032 	emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx);
1033 	emit(ARM_MOV_SI(tmp[0], tmp2[1], SRTYPE_ASL, 2), ctx);
1034 	emit(ARM_LDR_R(tmp[1], tmp[1], tmp[0]), ctx);
1035 	emit(ARM_CMP_I(tmp[1], 0), ctx);
1036 	_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1037 
1038 	/* goto *(prog->bpf_func + prologue_size); */
1039 	off = offsetof(struct bpf_prog, bpf_func);
1040 	emit_a32_mov_i(tmp2[1], off, false, ctx);
1041 	emit(ARM_LDR_R(tmp[1], tmp[1], tmp2[1]), ctx);
1042 	emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);
1043 	emit_bx_r(tmp[1], ctx);
1044 
1045 	/* out: */
1046 	if (out_offset == -1)
1047 		out_offset = cur_offset;
1048 	if (cur_offset != out_offset) {
1049 		pr_err_once("tail_call out_offset = %d, expected %d!\n",
1050 			    cur_offset, out_offset);
1051 		return -1;
1052 	}
1053 	return 0;
1054 #undef cur_offset
1055 #undef jmp_offset
1056 }
1057 
1058 /* 0xabcd => 0xcdab */
1059 static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1060 {
1061 #if __LINUX_ARM_ARCH__ < 6
1062 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
1063 
1064 	emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1065 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx);
1066 	emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1067 	emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx);
1068 #else /* ARMv6+ */
1069 	emit(ARM_REV16(rd, rn), ctx);
1070 #endif
1071 }
1072 
1073 /* 0xabcdefgh => 0xghefcdab */
1074 static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1075 {
1076 #if __LINUX_ARM_ARCH__ < 6
1077 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
1078 
1079 	emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1080 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx);
1081 	emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx);
1082 
1083 	emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx);
1084 	emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx);
1085 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx);
1086 	emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1087 	emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx);
1088 	emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx);
1089 	emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx);
1090 
1091 #else /* ARMv6+ */
1092 	emit(ARM_REV(rd, rn), ctx);
1093 #endif
1094 }
1095 
1096 // push the scratch stack register on top of the stack
1097 static inline void emit_push_r64(const u8 src[], const u8 shift,
1098 		struct jit_ctx *ctx)
1099 {
1100 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
1101 	u16 reg_set = 0;
1102 
1103 	emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(src[1]+shift)), ctx);
1104 	emit(ARM_LDR_I(tmp2[0], ARM_SP, STACK_VAR(src[0]+shift)), ctx);
1105 
1106 	reg_set = (1 << tmp2[1]) | (1 << tmp2[0]);
1107 	emit(ARM_PUSH(reg_set), ctx);
1108 }
1109 
1110 static void build_prologue(struct jit_ctx *ctx)
1111 {
1112 	const u8 r0 = bpf2a32[BPF_REG_0][1];
1113 	const u8 r2 = bpf2a32[BPF_REG_1][1];
1114 	const u8 r3 = bpf2a32[BPF_REG_1][0];
1115 	const u8 r4 = bpf2a32[BPF_REG_6][1];
1116 	const u8 fplo = bpf2a32[BPF_REG_FP][1];
1117 	const u8 fphi = bpf2a32[BPF_REG_FP][0];
1118 	const u8 *tcc = bpf2a32[TCALL_CNT];
1119 
1120 	/* Save callee saved registers. */
1121 #ifdef CONFIG_FRAME_POINTER
1122 	u16 reg_set = CALLEE_PUSH_MASK | 1 << ARM_IP | 1 << ARM_PC;
1123 	emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx);
1124 	emit(ARM_PUSH(reg_set), ctx);
1125 	emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx);
1126 #else
1127 	emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx);
1128 	emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx);
1129 #endif
1130 	/* Save frame pointer for later */
1131 	emit(ARM_SUB_I(ARM_IP, ARM_SP, SCRATCH_SIZE), ctx);
1132 
1133 	ctx->stack_size = imm8m(STACK_SIZE);
1134 
1135 	/* Set up function call stack */
1136 	emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);
1137 
1138 	/* Set up BPF prog stack base register */
1139 	emit_a32_mov_r(fplo, ARM_IP, true, false, ctx);
1140 	emit_a32_mov_i(fphi, 0, true, ctx);
1141 
1142 	/* mov r4, 0 */
1143 	emit(ARM_MOV_I(r4, 0), ctx);
1144 
1145 	/* Move BPF_CTX to BPF_R1 */
1146 	emit(ARM_MOV_R(r3, r4), ctx);
1147 	emit(ARM_MOV_R(r2, r0), ctx);
1148 	/* Initialize Tail Count */
1149 	emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[0])), ctx);
1150 	emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[1])), ctx);
1151 	/* end of prologue */
1152 }
1153 
1154 /* restore callee saved registers. */
1155 static void build_epilogue(struct jit_ctx *ctx)
1156 {
1157 #ifdef CONFIG_FRAME_POINTER
1158 	/* When using frame pointers, some additional registers need to
1159 	 * be loaded. */
1160 	u16 reg_set = CALLEE_POP_MASK | 1 << ARM_SP;
1161 	emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx);
1162 	emit(ARM_LDM(ARM_SP, reg_set), ctx);
1163 #else
1164 	/* Restore callee saved registers. */
1165 	emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx);
1166 	emit(ARM_POP(CALLEE_POP_MASK), ctx);
1167 #endif
1168 }
1169 
1170 /*
1171  * Convert an eBPF instruction to native instruction, i.e
1172  * JITs an eBPF instruction.
1173  * Returns :
1174  *	0  - Successfully JITed an 8-byte eBPF instruction
1175  *	>0 - Successfully JITed a 16-byte eBPF instruction
1176  *	<0 - Failed to JIT.
1177  */
1178 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
1179 {
1180 	const u8 code = insn->code;
1181 	const u8 *dst = bpf2a32[insn->dst_reg];
1182 	const u8 *src = bpf2a32[insn->src_reg];
1183 	const u8 *tmp = bpf2a32[TMP_REG_1];
1184 	const u8 *tmp2 = bpf2a32[TMP_REG_2];
1185 	const s16 off = insn->off;
1186 	const s32 imm = insn->imm;
1187 	const int i = insn - ctx->prog->insnsi;
1188 	const bool is64 = BPF_CLASS(code) == BPF_ALU64;
1189 	const bool dstk = is_on_stack(insn->dst_reg);
1190 	const bool sstk = is_on_stack(insn->src_reg);
1191 	u8 rd, rt, rm, rn;
1192 	s32 jmp_offset;
1193 
1194 #define check_imm(bits, imm) do {				\
1195 	if ((imm) >= (1 << ((bits) - 1)) ||			\
1196 	    (imm) < -(1 << ((bits) - 1))) {			\
1197 		pr_info("[%2d] imm=%d(0x%x) out of range\n",	\
1198 			i, imm, imm);				\
1199 		return -EINVAL;					\
1200 	}							\
1201 } while (0)
1202 #define check_imm24(imm) check_imm(24, imm)
1203 
1204 	switch (code) {
1205 	/* ALU operations */
1206 
1207 	/* dst = src */
1208 	case BPF_ALU | BPF_MOV | BPF_K:
1209 	case BPF_ALU | BPF_MOV | BPF_X:
1210 	case BPF_ALU64 | BPF_MOV | BPF_K:
1211 	case BPF_ALU64 | BPF_MOV | BPF_X:
1212 		switch (BPF_SRC(code)) {
1213 		case BPF_X:
1214 			emit_a32_mov_r64(is64, dst, src, dstk, sstk, ctx);
1215 			break;
1216 		case BPF_K:
1217 			/* Sign-extend immediate value to destination reg */
1218 			emit_a32_mov_i64(is64, dst, imm, dstk, ctx);
1219 			break;
1220 		}
1221 		break;
1222 	/* dst = dst + src/imm */
1223 	/* dst = dst - src/imm */
1224 	/* dst = dst | src/imm */
1225 	/* dst = dst & src/imm */
1226 	/* dst = dst ^ src/imm */
1227 	/* dst = dst * src/imm */
1228 	/* dst = dst << src */
1229 	/* dst = dst >> src */
1230 	case BPF_ALU | BPF_ADD | BPF_K:
1231 	case BPF_ALU | BPF_ADD | BPF_X:
1232 	case BPF_ALU | BPF_SUB | BPF_K:
1233 	case BPF_ALU | BPF_SUB | BPF_X:
1234 	case BPF_ALU | BPF_OR | BPF_K:
1235 	case BPF_ALU | BPF_OR | BPF_X:
1236 	case BPF_ALU | BPF_AND | BPF_K:
1237 	case BPF_ALU | BPF_AND | BPF_X:
1238 	case BPF_ALU | BPF_XOR | BPF_K:
1239 	case BPF_ALU | BPF_XOR | BPF_X:
1240 	case BPF_ALU | BPF_MUL | BPF_K:
1241 	case BPF_ALU | BPF_MUL | BPF_X:
1242 	case BPF_ALU | BPF_LSH | BPF_X:
1243 	case BPF_ALU | BPF_RSH | BPF_X:
1244 	case BPF_ALU | BPF_ARSH | BPF_K:
1245 	case BPF_ALU | BPF_ARSH | BPF_X:
1246 	case BPF_ALU64 | BPF_ADD | BPF_K:
1247 	case BPF_ALU64 | BPF_ADD | BPF_X:
1248 	case BPF_ALU64 | BPF_SUB | BPF_K:
1249 	case BPF_ALU64 | BPF_SUB | BPF_X:
1250 	case BPF_ALU64 | BPF_OR | BPF_K:
1251 	case BPF_ALU64 | BPF_OR | BPF_X:
1252 	case BPF_ALU64 | BPF_AND | BPF_K:
1253 	case BPF_ALU64 | BPF_AND | BPF_X:
1254 	case BPF_ALU64 | BPF_XOR | BPF_K:
1255 	case BPF_ALU64 | BPF_XOR | BPF_X:
1256 		switch (BPF_SRC(code)) {
1257 		case BPF_X:
1258 			emit_a32_alu_r64(is64, dst, src, dstk, sstk,
1259 					 ctx, BPF_OP(code));
1260 			break;
1261 		case BPF_K:
1262 			/* Move immediate value to the temporary register
1263 			 * and then do the ALU operation on the temporary
1264 			 * register as this will sign-extend the immediate
1265 			 * value into temporary reg and then it would be
1266 			 * safe to do the operation on it.
1267 			 */
1268 			emit_a32_mov_i64(is64, tmp2, imm, false, ctx);
1269 			emit_a32_alu_r64(is64, dst, tmp2, dstk, false,
1270 					 ctx, BPF_OP(code));
1271 			break;
1272 		}
1273 		break;
1274 	/* dst = dst / src(imm) */
1275 	/* dst = dst % src(imm) */
1276 	case BPF_ALU | BPF_DIV | BPF_K:
1277 	case BPF_ALU | BPF_DIV | BPF_X:
1278 	case BPF_ALU | BPF_MOD | BPF_K:
1279 	case BPF_ALU | BPF_MOD | BPF_X:
1280 		rt = src_lo;
1281 		rd = dstk ? tmp2[1] : dst_lo;
1282 		if (dstk)
1283 			emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
1284 		switch (BPF_SRC(code)) {
1285 		case BPF_X:
1286 			rt = sstk ? tmp2[0] : rt;
1287 			if (sstk)
1288 				emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)),
1289 				     ctx);
1290 			break;
1291 		case BPF_K:
1292 			rt = tmp2[0];
1293 			emit_a32_mov_i(rt, imm, false, ctx);
1294 			break;
1295 		}
1296 		emit_udivmod(rd, rd, rt, ctx, BPF_OP(code));
1297 		if (dstk)
1298 			emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
1299 		emit_a32_mov_i(dst_hi, 0, dstk, ctx);
1300 		break;
1301 	case BPF_ALU64 | BPF_DIV | BPF_K:
1302 	case BPF_ALU64 | BPF_DIV | BPF_X:
1303 	case BPF_ALU64 | BPF_MOD | BPF_K:
1304 	case BPF_ALU64 | BPF_MOD | BPF_X:
1305 		goto notyet;
1306 	/* dst = dst >> imm */
1307 	/* dst = dst << imm */
1308 	case BPF_ALU | BPF_RSH | BPF_K:
1309 	case BPF_ALU | BPF_LSH | BPF_K:
1310 		if (unlikely(imm > 31))
1311 			return -EINVAL;
1312 		if (imm)
1313 			emit_a32_alu_i(dst_lo, imm, dstk, ctx, BPF_OP(code));
1314 		emit_a32_mov_i(dst_hi, 0, dstk, ctx);
1315 		break;
1316 	/* dst = dst << imm */
1317 	case BPF_ALU64 | BPF_LSH | BPF_K:
1318 		if (unlikely(imm > 63))
1319 			return -EINVAL;
1320 		emit_a32_lsh_i64(dst, dstk, imm, ctx);
1321 		break;
1322 	/* dst = dst >> imm */
1323 	case BPF_ALU64 | BPF_RSH | BPF_K:
1324 		if (unlikely(imm > 63))
1325 			return -EINVAL;
1326 		emit_a32_rsh_i64(dst, dstk, imm, ctx);
1327 		break;
1328 	/* dst = dst << src */
1329 	case BPF_ALU64 | BPF_LSH | BPF_X:
1330 		emit_a32_lsh_r64(dst, src, dstk, sstk, ctx);
1331 		break;
1332 	/* dst = dst >> src */
1333 	case BPF_ALU64 | BPF_RSH | BPF_X:
1334 		emit_a32_rsh_r64(dst, src, dstk, sstk, ctx);
1335 		break;
1336 	/* dst = dst >> src (signed) */
1337 	case BPF_ALU64 | BPF_ARSH | BPF_X:
1338 		emit_a32_arsh_r64(dst, src, dstk, sstk, ctx);
1339 		break;
1340 	/* dst = dst >> imm (signed) */
1341 	case BPF_ALU64 | BPF_ARSH | BPF_K:
1342 		if (unlikely(imm > 63))
1343 			return -EINVAL;
1344 		emit_a32_arsh_i64(dst, dstk, imm, ctx);
1345 		break;
1346 	/* dst = ~dst */
1347 	case BPF_ALU | BPF_NEG:
1348 		emit_a32_alu_i(dst_lo, 0, dstk, ctx, BPF_OP(code));
1349 		emit_a32_mov_i(dst_hi, 0, dstk, ctx);
1350 		break;
1351 	/* dst = ~dst (64 bit) */
1352 	case BPF_ALU64 | BPF_NEG:
1353 		emit_a32_neg64(dst, dstk, ctx);
1354 		break;
1355 	/* dst = dst * src/imm */
1356 	case BPF_ALU64 | BPF_MUL | BPF_X:
1357 	case BPF_ALU64 | BPF_MUL | BPF_K:
1358 		switch (BPF_SRC(code)) {
1359 		case BPF_X:
1360 			emit_a32_mul_r64(dst, src, dstk, sstk, ctx);
1361 			break;
1362 		case BPF_K:
1363 			/* Move immediate value to the temporary register
1364 			 * and then do the multiplication on it as this
1365 			 * will sign-extend the immediate value into temp
1366 			 * reg then it would be safe to do the operation
1367 			 * on it.
1368 			 */
1369 			emit_a32_mov_i64(is64, tmp2, imm, false, ctx);
1370 			emit_a32_mul_r64(dst, tmp2, dstk, false, ctx);
1371 			break;
1372 		}
1373 		break;
1374 	/* dst = htole(dst) */
1375 	/* dst = htobe(dst) */
1376 	case BPF_ALU | BPF_END | BPF_FROM_LE:
1377 	case BPF_ALU | BPF_END | BPF_FROM_BE:
1378 		rd = dstk ? tmp[0] : dst_hi;
1379 		rt = dstk ? tmp[1] : dst_lo;
1380 		if (dstk) {
1381 			emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
1382 			emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
1383 		}
1384 		if (BPF_SRC(code) == BPF_FROM_LE)
1385 			goto emit_bswap_uxt;
1386 		switch (imm) {
1387 		case 16:
1388 			emit_rev16(rt, rt, ctx);
1389 			goto emit_bswap_uxt;
1390 		case 32:
1391 			emit_rev32(rt, rt, ctx);
1392 			goto emit_bswap_uxt;
1393 		case 64:
1394 			emit_rev32(ARM_LR, rt, ctx);
1395 			emit_rev32(rt, rd, ctx);
1396 			emit(ARM_MOV_R(rd, ARM_LR), ctx);
1397 			break;
1398 		}
1399 		goto exit;
1400 emit_bswap_uxt:
1401 		switch (imm) {
1402 		case 16:
1403 			/* zero-extend 16 bits into 64 bits */
1404 #if __LINUX_ARM_ARCH__ < 6
1405 			emit_a32_mov_i(tmp2[1], 0xffff, false, ctx);
1406 			emit(ARM_AND_R(rt, rt, tmp2[1]), ctx);
1407 #else /* ARMv6+ */
1408 			emit(ARM_UXTH(rt, rt), ctx);
1409 #endif
1410 			emit(ARM_EOR_R(rd, rd, rd), ctx);
1411 			break;
1412 		case 32:
1413 			/* zero-extend 32 bits into 64 bits */
1414 			emit(ARM_EOR_R(rd, rd, rd), ctx);
1415 			break;
1416 		case 64:
1417 			/* nop */
1418 			break;
1419 		}
1420 exit:
1421 		if (dstk) {
1422 			emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
1423 			emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
1424 		}
1425 		break;
1426 	/* dst = imm64 */
1427 	case BPF_LD | BPF_IMM | BPF_DW:
1428 	{
1429 		const struct bpf_insn insn1 = insn[1];
1430 		u32 hi, lo = imm;
1431 
1432 		hi = insn1.imm;
1433 		emit_a32_mov_i(dst_lo, lo, dstk, ctx);
1434 		emit_a32_mov_i(dst_hi, hi, dstk, ctx);
1435 
1436 		return 1;
1437 	}
1438 	/* LDX: dst = *(size *)(src + off) */
1439 	case BPF_LDX | BPF_MEM | BPF_W:
1440 	case BPF_LDX | BPF_MEM | BPF_H:
1441 	case BPF_LDX | BPF_MEM | BPF_B:
1442 	case BPF_LDX | BPF_MEM | BPF_DW:
1443 		rn = sstk ? tmp2[1] : src_lo;
1444 		if (sstk)
1445 			emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
1446 		emit_ldx_r(dst, rn, dstk, off, ctx, BPF_SIZE(code));
1447 		break;
1448 	/* ST: *(size *)(dst + off) = imm */
1449 	case BPF_ST | BPF_MEM | BPF_W:
1450 	case BPF_ST | BPF_MEM | BPF_H:
1451 	case BPF_ST | BPF_MEM | BPF_B:
1452 	case BPF_ST | BPF_MEM | BPF_DW:
1453 		switch (BPF_SIZE(code)) {
1454 		case BPF_DW:
1455 			/* Sign-extend immediate value into temp reg */
1456 			emit_a32_mov_i64(true, tmp2, imm, false, ctx);
1457 			emit_str_r(dst_lo, tmp2[1], dstk, off, ctx, BPF_W);
1458 			emit_str_r(dst_lo, tmp2[0], dstk, off+4, ctx, BPF_W);
1459 			break;
1460 		case BPF_W:
1461 		case BPF_H:
1462 		case BPF_B:
1463 			emit_a32_mov_i(tmp2[1], imm, false, ctx);
1464 			emit_str_r(dst_lo, tmp2[1], dstk, off, ctx,
1465 				   BPF_SIZE(code));
1466 			break;
1467 		}
1468 		break;
1469 	/* STX XADD: lock *(u32 *)(dst + off) += src */
1470 	case BPF_STX | BPF_XADD | BPF_W:
1471 	/* STX XADD: lock *(u64 *)(dst + off) += src */
1472 	case BPF_STX | BPF_XADD | BPF_DW:
1473 		goto notyet;
1474 	/* STX: *(size *)(dst + off) = src */
1475 	case BPF_STX | BPF_MEM | BPF_W:
1476 	case BPF_STX | BPF_MEM | BPF_H:
1477 	case BPF_STX | BPF_MEM | BPF_B:
1478 	case BPF_STX | BPF_MEM | BPF_DW:
1479 	{
1480 		u8 sz = BPF_SIZE(code);
1481 
1482 		rn = sstk ? tmp2[1] : src_lo;
1483 		rm = sstk ? tmp2[0] : src_hi;
1484 		if (sstk) {
1485 			emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
1486 			emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
1487 		}
1488 
1489 		/* Store the value */
1490 		if (BPF_SIZE(code) == BPF_DW) {
1491 			emit_str_r(dst_lo, rn, dstk, off, ctx, BPF_W);
1492 			emit_str_r(dst_lo, rm, dstk, off+4, ctx, BPF_W);
1493 		} else {
1494 			emit_str_r(dst_lo, rn, dstk, off, ctx, sz);
1495 		}
1496 		break;
1497 	}
1498 	/* PC += off if dst == src */
1499 	/* PC += off if dst > src */
1500 	/* PC += off if dst >= src */
1501 	/* PC += off if dst < src */
1502 	/* PC += off if dst <= src */
1503 	/* PC += off if dst != src */
1504 	/* PC += off if dst > src (signed) */
1505 	/* PC += off if dst >= src (signed) */
1506 	/* PC += off if dst < src (signed) */
1507 	/* PC += off if dst <= src (signed) */
1508 	/* PC += off if dst & src */
1509 	case BPF_JMP | BPF_JEQ | BPF_X:
1510 	case BPF_JMP | BPF_JGT | BPF_X:
1511 	case BPF_JMP | BPF_JGE | BPF_X:
1512 	case BPF_JMP | BPF_JNE | BPF_X:
1513 	case BPF_JMP | BPF_JSGT | BPF_X:
1514 	case BPF_JMP | BPF_JSGE | BPF_X:
1515 	case BPF_JMP | BPF_JSET | BPF_X:
1516 	case BPF_JMP | BPF_JLE | BPF_X:
1517 	case BPF_JMP | BPF_JLT | BPF_X:
1518 	case BPF_JMP | BPF_JSLT | BPF_X:
1519 	case BPF_JMP | BPF_JSLE | BPF_X:
1520 		/* Setup source registers */
1521 		rm = sstk ? tmp2[0] : src_hi;
1522 		rn = sstk ? tmp2[1] : src_lo;
1523 		if (sstk) {
1524 			emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
1525 			emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
1526 		}
1527 		goto go_jmp;
1528 	/* PC += off if dst == imm */
1529 	/* PC += off if dst > imm */
1530 	/* PC += off if dst >= imm */
1531 	/* PC += off if dst < imm */
1532 	/* PC += off if dst <= imm */
1533 	/* PC += off if dst != imm */
1534 	/* PC += off if dst > imm (signed) */
1535 	/* PC += off if dst >= imm (signed) */
1536 	/* PC += off if dst < imm (signed) */
1537 	/* PC += off if dst <= imm (signed) */
1538 	/* PC += off if dst & imm */
1539 	case BPF_JMP | BPF_JEQ | BPF_K:
1540 	case BPF_JMP | BPF_JGT | BPF_K:
1541 	case BPF_JMP | BPF_JGE | BPF_K:
1542 	case BPF_JMP | BPF_JNE | BPF_K:
1543 	case BPF_JMP | BPF_JSGT | BPF_K:
1544 	case BPF_JMP | BPF_JSGE | BPF_K:
1545 	case BPF_JMP | BPF_JSET | BPF_K:
1546 	case BPF_JMP | BPF_JLT | BPF_K:
1547 	case BPF_JMP | BPF_JLE | BPF_K:
1548 	case BPF_JMP | BPF_JSLT | BPF_K:
1549 	case BPF_JMP | BPF_JSLE | BPF_K:
1550 		if (off == 0)
1551 			break;
1552 		rm = tmp2[0];
1553 		rn = tmp2[1];
1554 		/* Sign-extend immediate value */
1555 		emit_a32_mov_i64(true, tmp2, imm, false, ctx);
1556 go_jmp:
1557 		/* Setup destination register */
1558 		rd = dstk ? tmp[0] : dst_hi;
1559 		rt = dstk ? tmp[1] : dst_lo;
1560 		if (dstk) {
1561 			emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
1562 			emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
1563 		}
1564 
1565 		/* Check for the condition */
1566 		emit_ar_r(rd, rt, rm, rn, ctx, BPF_OP(code));
1567 
1568 		/* Setup JUMP instruction */
1569 		jmp_offset = bpf2a32_offset(i+off, i, ctx);
1570 		switch (BPF_OP(code)) {
1571 		case BPF_JNE:
1572 		case BPF_JSET:
1573 			_emit(ARM_COND_NE, ARM_B(jmp_offset), ctx);
1574 			break;
1575 		case BPF_JEQ:
1576 			_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1577 			break;
1578 		case BPF_JGT:
1579 			_emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
1580 			break;
1581 		case BPF_JGE:
1582 			_emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1583 			break;
1584 		case BPF_JSGT:
1585 			_emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1586 			break;
1587 		case BPF_JSGE:
1588 			_emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
1589 			break;
1590 		case BPF_JLE:
1591 			_emit(ARM_COND_LS, ARM_B(jmp_offset), ctx);
1592 			break;
1593 		case BPF_JLT:
1594 			_emit(ARM_COND_CC, ARM_B(jmp_offset), ctx);
1595 			break;
1596 		case BPF_JSLT:
1597 			_emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1598 			break;
1599 		case BPF_JSLE:
1600 			_emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
1601 			break;
1602 		}
1603 		break;
1604 	/* JMP OFF */
1605 	case BPF_JMP | BPF_JA:
1606 	{
1607 		if (off == 0)
1608 			break;
1609 		jmp_offset = bpf2a32_offset(i+off, i, ctx);
1610 		check_imm24(jmp_offset);
1611 		emit(ARM_B(jmp_offset), ctx);
1612 		break;
1613 	}
1614 	/* tail call */
1615 	case BPF_JMP | BPF_TAIL_CALL:
1616 		if (emit_bpf_tail_call(ctx))
1617 			return -EFAULT;
1618 		break;
1619 	/* function call */
1620 	case BPF_JMP | BPF_CALL:
1621 	{
1622 		const u8 *r0 = bpf2a32[BPF_REG_0];
1623 		const u8 *r1 = bpf2a32[BPF_REG_1];
1624 		const u8 *r2 = bpf2a32[BPF_REG_2];
1625 		const u8 *r3 = bpf2a32[BPF_REG_3];
1626 		const u8 *r4 = bpf2a32[BPF_REG_4];
1627 		const u8 *r5 = bpf2a32[BPF_REG_5];
1628 		const u32 func = (u32)__bpf_call_base + (u32)imm;
1629 
1630 		emit_a32_mov_r64(true, r0, r1, false, false, ctx);
1631 		emit_a32_mov_r64(true, r1, r2, false, true, ctx);
1632 		emit_push_r64(r5, 0, ctx);
1633 		emit_push_r64(r4, 8, ctx);
1634 		emit_push_r64(r3, 16, ctx);
1635 
1636 		emit_a32_mov_i(tmp[1], func, false, ctx);
1637 		emit_blx_r(tmp[1], ctx);
1638 
1639 		emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean
1640 		break;
1641 	}
1642 	/* function return */
1643 	case BPF_JMP | BPF_EXIT:
1644 		/* Optimization: when last instruction is EXIT
1645 		 * simply fallthrough to epilogue.
1646 		 */
1647 		if (i == ctx->prog->len - 1)
1648 			break;
1649 		jmp_offset = epilogue_offset(ctx);
1650 		check_imm24(jmp_offset);
1651 		emit(ARM_B(jmp_offset), ctx);
1652 		break;
1653 notyet:
1654 		pr_info_once("*** NOT YET: opcode %02x ***\n", code);
1655 		return -EFAULT;
1656 	default:
1657 		pr_err_once("unknown opcode %02x\n", code);
1658 		return -EINVAL;
1659 	}
1660 
1661 	if (ctx->flags & FLAG_IMM_OVERFLOW)
1662 		/*
1663 		 * this instruction generated an overflow when
1664 		 * trying to access the literal pool, so
1665 		 * delegate this filter to the kernel interpreter.
1666 		 */
1667 		return -1;
1668 	return 0;
1669 }
1670 
1671 static int build_body(struct jit_ctx *ctx)
1672 {
1673 	const struct bpf_prog *prog = ctx->prog;
1674 	unsigned int i;
1675 
1676 	for (i = 0; i < prog->len; i++) {
1677 		const struct bpf_insn *insn = &(prog->insnsi[i]);
1678 		int ret;
1679 
1680 		ret = build_insn(insn, ctx);
1681 
1682 		/* It's used with loading the 64 bit immediate value. */
1683 		if (ret > 0) {
1684 			i++;
1685 			if (ctx->target == NULL)
1686 				ctx->offsets[i] = ctx->idx;
1687 			continue;
1688 		}
1689 
1690 		if (ctx->target == NULL)
1691 			ctx->offsets[i] = ctx->idx;
1692 
1693 		/* If unsuccesfull, return with error code */
1694 		if (ret)
1695 			return ret;
1696 	}
1697 	return 0;
1698 }
1699 
1700 static int validate_code(struct jit_ctx *ctx)
1701 {
1702 	int i;
1703 
1704 	for (i = 0; i < ctx->idx; i++) {
1705 		if (ctx->target[i] == __opcode_to_mem_arm(ARM_INST_UDF))
1706 			return -1;
1707 	}
1708 
1709 	return 0;
1710 }
1711 
1712 void bpf_jit_compile(struct bpf_prog *prog)
1713 {
1714 	/* Nothing to do here. We support Internal BPF. */
1715 }
1716 
1717 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1718 {
1719 	struct bpf_prog *tmp, *orig_prog = prog;
1720 	struct bpf_binary_header *header;
1721 	bool tmp_blinded = false;
1722 	struct jit_ctx ctx;
1723 	unsigned int tmp_idx;
1724 	unsigned int image_size;
1725 	u8 *image_ptr;
1726 
1727 	/* If BPF JIT was not enabled then we must fall back to
1728 	 * the interpreter.
1729 	 */
1730 	if (!prog->jit_requested)
1731 		return orig_prog;
1732 
1733 	/* If constant blinding was enabled and we failed during blinding
1734 	 * then we must fall back to the interpreter. Otherwise, we save
1735 	 * the new JITed code.
1736 	 */
1737 	tmp = bpf_jit_blind_constants(prog);
1738 
1739 	if (IS_ERR(tmp))
1740 		return orig_prog;
1741 	if (tmp != prog) {
1742 		tmp_blinded = true;
1743 		prog = tmp;
1744 	}
1745 
1746 	memset(&ctx, 0, sizeof(ctx));
1747 	ctx.prog = prog;
1748 
1749 	/* Not able to allocate memory for offsets[] , then
1750 	 * we must fall back to the interpreter
1751 	 */
1752 	ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
1753 	if (ctx.offsets == NULL) {
1754 		prog = orig_prog;
1755 		goto out;
1756 	}
1757 
1758 	/* 1) fake pass to find in the length of the JITed code,
1759 	 * to compute ctx->offsets and other context variables
1760 	 * needed to compute final JITed code.
1761 	 * Also, calculate random starting pointer/start of JITed code
1762 	 * which is prefixed by random number of fault instructions.
1763 	 *
1764 	 * If the first pass fails then there is no chance of it
1765 	 * being successful in the second pass, so just fall back
1766 	 * to the interpreter.
1767 	 */
1768 	if (build_body(&ctx)) {
1769 		prog = orig_prog;
1770 		goto out_off;
1771 	}
1772 
1773 	tmp_idx = ctx.idx;
1774 	build_prologue(&ctx);
1775 	ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
1776 
1777 	ctx.epilogue_offset = ctx.idx;
1778 
1779 #if __LINUX_ARM_ARCH__ < 7
1780 	tmp_idx = ctx.idx;
1781 	build_epilogue(&ctx);
1782 	ctx.epilogue_bytes = (ctx.idx - tmp_idx) * 4;
1783 
1784 	ctx.idx += ctx.imm_count;
1785 	if (ctx.imm_count) {
1786 		ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);
1787 		if (ctx.imms == NULL) {
1788 			prog = orig_prog;
1789 			goto out_off;
1790 		}
1791 	}
1792 #else
1793 	/* there's nothing about the epilogue on ARMv7 */
1794 	build_epilogue(&ctx);
1795 #endif
1796 	/* Now we can get the actual image size of the JITed arm code.
1797 	 * Currently, we are not considering the THUMB-2 instructions
1798 	 * for jit, although it can decrease the size of the image.
1799 	 *
1800 	 * As each arm instruction is of length 32bit, we are translating
1801 	 * number of JITed intructions into the size required to store these
1802 	 * JITed code.
1803 	 */
1804 	image_size = sizeof(u32) * ctx.idx;
1805 
1806 	/* Now we know the size of the structure to make */
1807 	header = bpf_jit_binary_alloc(image_size, &image_ptr,
1808 				      sizeof(u32), jit_fill_hole);
1809 	/* Not able to allocate memory for the structure then
1810 	 * we must fall back to the interpretation
1811 	 */
1812 	if (header == NULL) {
1813 		prog = orig_prog;
1814 		goto out_imms;
1815 	}
1816 
1817 	/* 2.) Actual pass to generate final JIT code */
1818 	ctx.target = (u32 *) image_ptr;
1819 	ctx.idx = 0;
1820 
1821 	build_prologue(&ctx);
1822 
1823 	/* If building the body of the JITed code fails somehow,
1824 	 * we fall back to the interpretation.
1825 	 */
1826 	if (build_body(&ctx) < 0) {
1827 		image_ptr = NULL;
1828 		bpf_jit_binary_free(header);
1829 		prog = orig_prog;
1830 		goto out_imms;
1831 	}
1832 	build_epilogue(&ctx);
1833 
1834 	/* 3.) Extra pass to validate JITed Code */
1835 	if (validate_code(&ctx)) {
1836 		image_ptr = NULL;
1837 		bpf_jit_binary_free(header);
1838 		prog = orig_prog;
1839 		goto out_imms;
1840 	}
1841 	flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
1842 
1843 	if (bpf_jit_enable > 1)
1844 		/* there are 2 passes here */
1845 		bpf_jit_dump(prog->len, image_size, 2, ctx.target);
1846 
1847 	bpf_jit_binary_lock_ro(header);
1848 	prog->bpf_func = (void *)ctx.target;
1849 	prog->jited = 1;
1850 	prog->jited_len = image_size;
1851 
1852 out_imms:
1853 #if __LINUX_ARM_ARCH__ < 7
1854 	if (ctx.imm_count)
1855 		kfree(ctx.imms);
1856 #endif
1857 out_off:
1858 	kfree(ctx.offsets);
1859 out:
1860 	if (tmp_blinded)
1861 		bpf_jit_prog_release_other(prog, prog == orig_prog ?
1862 					   tmp : orig_prog);
1863 	return prog;
1864 }
1865 
1866