1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e54bcde3SZi Shen Lim /* 3e54bcde3SZi Shen Lim * BPF JIT compiler for ARM64 4e54bcde3SZi Shen Lim * 542ff712bSZi Shen Lim * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com> 6e54bcde3SZi Shen Lim */ 7e54bcde3SZi Shen Lim 8e54bcde3SZi Shen Lim #define pr_fmt(fmt) "bpf_jit: " fmt 9e54bcde3SZi Shen Lim 1080083428SJean-Philippe Brucker #include <linux/bitfield.h> 11ddb55992SZi Shen Lim #include <linux/bpf.h> 12e54bcde3SZi Shen Lim #include <linux/filter.h> 13e54bcde3SZi Shen Lim #include <linux/printk.h> 14e54bcde3SZi Shen Lim #include <linux/slab.h> 15b569c1c6SDaniel Borkmann 16e54bcde3SZi Shen Lim #include <asm/byteorder.h> 17e54bcde3SZi Shen Lim #include <asm/cacheflush.h> 18b569c1c6SDaniel Borkmann #include <asm/debug-monitors.h> 19*3e00e39dSMark Rutland #include <asm/insn.h> 20d4bbc30bSLaura Abbott #include <asm/set_memory.h> 21e54bcde3SZi Shen Lim 22e54bcde3SZi Shen Lim #include "bpf_jit.h" 23e54bcde3SZi Shen Lim 2426eb042eSDaniel Borkmann #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) 2526eb042eSDaniel Borkmann #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 26ddb55992SZi Shen Lim #define TCALL_CNT (MAX_BPF_JIT_REG + 2) 277005cadeSDaniel Borkmann #define TMP_REG_3 (MAX_BPF_JIT_REG + 3) 28e54bcde3SZi Shen Lim 29e54bcde3SZi Shen Lim /* Map BPF registers to A64 registers */ 30e54bcde3SZi Shen Lim static const int bpf2a64[] = { 31e54bcde3SZi Shen Lim /* return value from in-kernel function, and exit value from eBPF */ 32e54bcde3SZi Shen Lim [BPF_REG_0] = A64_R(7), 33e54bcde3SZi Shen Lim /* arguments from eBPF program to in-kernel function */ 34e54bcde3SZi Shen Lim [BPF_REG_1] = A64_R(0), 35e54bcde3SZi Shen Lim [BPF_REG_2] = A64_R(1), 36e54bcde3SZi Shen Lim [BPF_REG_3] = A64_R(2), 37e54bcde3SZi Shen Lim [BPF_REG_4] = A64_R(3), 38e54bcde3SZi Shen Lim [BPF_REG_5] = A64_R(4), 39e54bcde3SZi Shen Lim /* callee saved registers that in-kernel function will preserve */ 40e54bcde3SZi Shen Lim [BPF_REG_6] = A64_R(19), 41e54bcde3SZi Shen Lim [BPF_REG_7] = A64_R(20), 42e54bcde3SZi Shen Lim [BPF_REG_8] = A64_R(21), 43e54bcde3SZi Shen Lim [BPF_REG_9] = A64_R(22), 44e54bcde3SZi Shen Lim /* read-only frame pointer to access stack */ 45ec0738dbSYang Shi [BPF_REG_FP] = A64_R(25), 464c1cd4fdSYang Shi /* temporary registers for internal BPF JIT */ 474c1cd4fdSYang Shi [TMP_REG_1] = A64_R(10), 484c1cd4fdSYang Shi [TMP_REG_2] = A64_R(11), 497005cadeSDaniel Borkmann [TMP_REG_3] = A64_R(12), 50ddb55992SZi Shen Lim /* tail_call_cnt */ 51ddb55992SZi Shen Lim [TCALL_CNT] = A64_R(26), 5226eb042eSDaniel Borkmann /* temporary register for blinding constants */ 5326eb042eSDaniel Borkmann [BPF_REG_AX] = A64_R(9), 54e54bcde3SZi Shen Lim }; 55e54bcde3SZi Shen Lim 56e54bcde3SZi Shen Lim struct jit_ctx { 57e54bcde3SZi Shen Lim const struct bpf_prog *prog; 58e54bcde3SZi Shen Lim int idx; 5951c9fbb1SZi Shen Lim int epilogue_offset; 60e54bcde3SZi Shen Lim int *offset; 6180083428SJean-Philippe Brucker int exentry_idx; 62425e1ed7SLuc Van Oostenryck __le32 *image; 63f1c9eed7SDaniel Borkmann u32 stack_size; 64e54bcde3SZi Shen Lim }; 65e54bcde3SZi Shen Lim 66e54bcde3SZi Shen Lim static inline void emit(const u32 insn, struct jit_ctx *ctx) 67e54bcde3SZi Shen Lim { 68e54bcde3SZi Shen Lim if (ctx->image != NULL) 69e54bcde3SZi Shen Lim ctx->image[ctx->idx] = cpu_to_le32(insn); 70e54bcde3SZi Shen Lim 71e54bcde3SZi Shen Lim ctx->idx++; 72e54bcde3SZi Shen Lim } 73e54bcde3SZi Shen Lim 74e54bcde3SZi Shen Lim static inline void emit_a64_mov_i(const int is64, const int reg, 75e54bcde3SZi Shen Lim const s32 val, struct jit_ctx *ctx) 76e54bcde3SZi Shen Lim { 77e54bcde3SZi Shen Lim u16 hi = val >> 16; 78e54bcde3SZi Shen Lim u16 lo = val & 0xffff; 79e54bcde3SZi Shen Lim 80e54bcde3SZi Shen Lim if (hi & 0x8000) { 81e54bcde3SZi Shen Lim if (hi == 0xffff) { 82e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx); 83e54bcde3SZi Shen Lim } else { 84e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx); 856d2eea6fSDaniel Borkmann if (lo != 0xffff) 86e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, lo, 0), ctx); 87e54bcde3SZi Shen Lim } 88e54bcde3SZi Shen Lim } else { 89e54bcde3SZi Shen Lim emit(A64_MOVZ(is64, reg, lo, 0), ctx); 90e54bcde3SZi Shen Lim if (hi) 91e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, hi, 16), ctx); 92e54bcde3SZi Shen Lim } 93e54bcde3SZi Shen Lim } 94e54bcde3SZi Shen Lim 956d2eea6fSDaniel Borkmann static int i64_i16_blocks(const u64 val, bool inverse) 966d2eea6fSDaniel Borkmann { 976d2eea6fSDaniel Borkmann return (((val >> 0) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 986d2eea6fSDaniel Borkmann (((val >> 16) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 996d2eea6fSDaniel Borkmann (((val >> 32) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 1006d2eea6fSDaniel Borkmann (((val >> 48) & 0xffff) != (inverse ? 0xffff : 0x0000)); 1016d2eea6fSDaniel Borkmann } 1026d2eea6fSDaniel Borkmann 1036d2eea6fSDaniel Borkmann static inline void emit_a64_mov_i64(const int reg, const u64 val, 1046d2eea6fSDaniel Borkmann struct jit_ctx *ctx) 1056d2eea6fSDaniel Borkmann { 1066d2eea6fSDaniel Borkmann u64 nrm_tmp = val, rev_tmp = ~val; 1076d2eea6fSDaniel Borkmann bool inverse; 1086d2eea6fSDaniel Borkmann int shift; 1096d2eea6fSDaniel Borkmann 1106d2eea6fSDaniel Borkmann if (!(nrm_tmp >> 32)) 1116d2eea6fSDaniel Borkmann return emit_a64_mov_i(0, reg, (u32)val, ctx); 1126d2eea6fSDaniel Borkmann 1136d2eea6fSDaniel Borkmann inverse = i64_i16_blocks(nrm_tmp, true) < i64_i16_blocks(nrm_tmp, false); 1146d2eea6fSDaniel Borkmann shift = max(round_down((inverse ? (fls64(rev_tmp) - 1) : 1156d2eea6fSDaniel Borkmann (fls64(nrm_tmp) - 1)), 16), 0); 1166d2eea6fSDaniel Borkmann if (inverse) 1176d2eea6fSDaniel Borkmann emit(A64_MOVN(1, reg, (rev_tmp >> shift) & 0xffff, shift), ctx); 1186d2eea6fSDaniel Borkmann else 1196d2eea6fSDaniel Borkmann emit(A64_MOVZ(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx); 1206d2eea6fSDaniel Borkmann shift -= 16; 1216d2eea6fSDaniel Borkmann while (shift >= 0) { 1226d2eea6fSDaniel Borkmann if (((nrm_tmp >> shift) & 0xffff) != (inverse ? 0xffff : 0x0000)) 1236d2eea6fSDaniel Borkmann emit(A64_MOVK(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx); 1246d2eea6fSDaniel Borkmann shift -= 16; 1256d2eea6fSDaniel Borkmann } 1266d2eea6fSDaniel Borkmann } 1276d2eea6fSDaniel Borkmann 1286d2eea6fSDaniel Borkmann /* 129cc2b8ed1SArd Biesheuvel * Kernel addresses in the vmalloc space use at most 48 bits, and the 130cc2b8ed1SArd Biesheuvel * remaining bits are guaranteed to be 0x1. So we can compose the address 131cc2b8ed1SArd Biesheuvel * with a fixed length movn/movk/movk sequence. 1326d2eea6fSDaniel Borkmann */ 1336d2eea6fSDaniel Borkmann static inline void emit_addr_mov_i64(const int reg, const u64 val, 1346d2eea6fSDaniel Borkmann struct jit_ctx *ctx) 1356d2eea6fSDaniel Borkmann { 1366d2eea6fSDaniel Borkmann u64 tmp = val; 1376d2eea6fSDaniel Borkmann int shift = 0; 1386d2eea6fSDaniel Borkmann 139cc2b8ed1SArd Biesheuvel emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx); 140cc2b8ed1SArd Biesheuvel while (shift < 32) { 1416d2eea6fSDaniel Borkmann tmp >>= 16; 1426d2eea6fSDaniel Borkmann shift += 16; 1436d2eea6fSDaniel Borkmann emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx); 1446d2eea6fSDaniel Borkmann } 1456d2eea6fSDaniel Borkmann } 1466d2eea6fSDaniel Borkmann 14732f6865cSIlias Apalodimas static inline int bpf2a64_offset(int bpf_insn, int off, 148e54bcde3SZi Shen Lim const struct jit_ctx *ctx) 149e54bcde3SZi Shen Lim { 15032f6865cSIlias Apalodimas /* BPF JMP offset is relative to the next instruction */ 15132f6865cSIlias Apalodimas bpf_insn++; 15232f6865cSIlias Apalodimas /* 15332f6865cSIlias Apalodimas * Whereas arm64 branch instructions encode the offset 15432f6865cSIlias Apalodimas * from the branch itself, so we must subtract 1 from the 15532f6865cSIlias Apalodimas * instruction offset. 15632f6865cSIlias Apalodimas */ 15732f6865cSIlias Apalodimas return ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1); 158e54bcde3SZi Shen Lim } 159e54bcde3SZi Shen Lim 160b569c1c6SDaniel Borkmann static void jit_fill_hole(void *area, unsigned int size) 161b569c1c6SDaniel Borkmann { 162425e1ed7SLuc Van Oostenryck __le32 *ptr; 163b569c1c6SDaniel Borkmann /* We are guaranteed to have aligned memory. */ 164b569c1c6SDaniel Borkmann for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 165b569c1c6SDaniel Borkmann *ptr++ = cpu_to_le32(AARCH64_BREAK_FAULT); 166b569c1c6SDaniel Borkmann } 167b569c1c6SDaniel Borkmann 168e54bcde3SZi Shen Lim static inline int epilogue_offset(const struct jit_ctx *ctx) 169e54bcde3SZi Shen Lim { 17051c9fbb1SZi Shen Lim int to = ctx->epilogue_offset; 17151c9fbb1SZi Shen Lim int from = ctx->idx; 172e54bcde3SZi Shen Lim 173e54bcde3SZi Shen Lim return to - from; 174e54bcde3SZi Shen Lim } 175e54bcde3SZi Shen Lim 176fd868f14SLuke Nelson static bool is_addsub_imm(u32 imm) 177fd868f14SLuke Nelson { 178fd868f14SLuke Nelson /* Either imm12 or shifted imm12. */ 179fd868f14SLuke Nelson return !(imm & ~0xfff) || !(imm & ~0xfff000); 180fd868f14SLuke Nelson } 181fd868f14SLuke Nelson 182e54bcde3SZi Shen Lim /* Stack must be multiples of 16B */ 183e54bcde3SZi Shen Lim #define STACK_ALIGN(sz) (((sz) + 15) & ~15) 184e54bcde3SZi Shen Lim 185a2284d91SDaniel Borkmann /* Tail call offset to jump into */ 186fa76cfe6SMark Brown #if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) 187fa76cfe6SMark Brown #define PROLOGUE_OFFSET 8 188fa76cfe6SMark Brown #else 189a2284d91SDaniel Borkmann #define PROLOGUE_OFFSET 7 190fa76cfe6SMark Brown #endif 191ddb55992SZi Shen Lim 19256ea6a8bSDaniel Borkmann static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) 193e54bcde3SZi Shen Lim { 194f1c9eed7SDaniel Borkmann const struct bpf_prog *prog = ctx->prog; 195e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 196e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 197e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 198e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 199e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 200ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 201ddb55992SZi Shen Lim const int idx0 = ctx->idx; 202ddb55992SZi Shen Lim int cur_offset; 203e54bcde3SZi Shen Lim 204ec0738dbSYang Shi /* 205ec0738dbSYang Shi * BPF prog stack layout 206ec0738dbSYang Shi * 207ec0738dbSYang Shi * high 208ec0738dbSYang Shi * original A64_SP => 0:+-----+ BPF prologue 209ec0738dbSYang Shi * |FP/LR| 210ec0738dbSYang Shi * current A64_FP => -16:+-----+ 211ec0738dbSYang Shi * | ... | callee saved registers 2124c1cd4fdSYang Shi * BPF fp register => -64:+-----+ <= (BPF_FP) 213ec0738dbSYang Shi * | | 214ec0738dbSYang Shi * | ... | BPF prog stack 215ec0738dbSYang Shi * | | 216f1c9eed7SDaniel Borkmann * +-----+ <= (BPF_FP - prog->aux->stack_depth) 21709ece3d0SDaniel Borkmann * |RSVD | padding 218f1c9eed7SDaniel Borkmann * current A64_SP => +-----+ <= (BPF_FP - ctx->stack_size) 219ec0738dbSYang Shi * | | 220ec0738dbSYang Shi * | ... | Function call stack 221ec0738dbSYang Shi * | | 222ec0738dbSYang Shi * +-----+ 223ec0738dbSYang Shi * low 224ec0738dbSYang Shi * 225ec0738dbSYang Shi */ 226ec0738dbSYang Shi 227fa76cfe6SMark Brown /* BTI landing pad */ 228fa76cfe6SMark Brown if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) 229fa76cfe6SMark Brown emit(A64_BTI_C, ctx); 230fa76cfe6SMark Brown 231ec0738dbSYang Shi /* Save FP and LR registers to stay align with ARM64 AAPCS */ 232ec0738dbSYang Shi emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); 233ec0738dbSYang Shi emit(A64_MOV(1, A64_FP, A64_SP), ctx); 234ec0738dbSYang Shi 235ddb55992SZi Shen Lim /* Save callee-saved registers */ 236e54bcde3SZi Shen Lim emit(A64_PUSH(r6, r7, A64_SP), ctx); 237e54bcde3SZi Shen Lim emit(A64_PUSH(r8, r9, A64_SP), ctx); 238ddb55992SZi Shen Lim emit(A64_PUSH(fp, tcc, A64_SP), ctx); 239e54bcde3SZi Shen Lim 240ddb55992SZi Shen Lim /* Set up BPF prog stack base register */ 241e54bcde3SZi Shen Lim emit(A64_MOV(1, fp, A64_SP), ctx); 242e54bcde3SZi Shen Lim 24356ea6a8bSDaniel Borkmann if (!ebpf_from_cbpf) { 244ddb55992SZi Shen Lim /* Initialize tail_call_cnt */ 245ddb55992SZi Shen Lim emit(A64_MOVZ(1, tcc, 0, 0), ctx); 246ddb55992SZi Shen Lim 247ddb55992SZi Shen Lim cur_offset = ctx->idx - idx0; 248ddb55992SZi Shen Lim if (cur_offset != PROLOGUE_OFFSET) { 249ddb55992SZi Shen Lim pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n", 250ddb55992SZi Shen Lim cur_offset, PROLOGUE_OFFSET); 251ddb55992SZi Shen Lim return -1; 252ddb55992SZi Shen Lim } 253fa76cfe6SMark Brown 254fa76cfe6SMark Brown /* BTI landing pad for the tail call, done with a BR */ 255fa76cfe6SMark Brown if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) 256fa76cfe6SMark Brown emit(A64_BTI_J, ctx); 25756ea6a8bSDaniel Borkmann } 258a2284d91SDaniel Borkmann 25909ece3d0SDaniel Borkmann ctx->stack_size = STACK_ALIGN(prog->aux->stack_depth); 260a2284d91SDaniel Borkmann 261a2284d91SDaniel Borkmann /* Set up function call stack */ 262a2284d91SDaniel Borkmann emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 263ddb55992SZi Shen Lim return 0; 264ddb55992SZi Shen Lim } 265ddb55992SZi Shen Lim 266ddb55992SZi Shen Lim static int out_offset = -1; /* initialized on the first pass of build_body() */ 267ddb55992SZi Shen Lim static int emit_bpf_tail_call(struct jit_ctx *ctx) 268ddb55992SZi Shen Lim { 269ddb55992SZi Shen Lim /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ 270ddb55992SZi Shen Lim const u8 r2 = bpf2a64[BPF_REG_2]; 271ddb55992SZi Shen Lim const u8 r3 = bpf2a64[BPF_REG_3]; 272ddb55992SZi Shen Lim 273ddb55992SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 274ddb55992SZi Shen Lim const u8 prg = bpf2a64[TMP_REG_2]; 275ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 276ddb55992SZi Shen Lim const int idx0 = ctx->idx; 277ddb55992SZi Shen Lim #define cur_offset (ctx->idx - idx0) 278ddb55992SZi Shen Lim #define jmp_offset (out_offset - (cur_offset)) 279ddb55992SZi Shen Lim size_t off; 280ddb55992SZi Shen Lim 281ddb55992SZi Shen Lim /* if (index >= array->map.max_entries) 282ddb55992SZi Shen Lim * goto out; 283ddb55992SZi Shen Lim */ 284ddb55992SZi Shen Lim off = offsetof(struct bpf_array, map.max_entries); 285ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 286ddb55992SZi Shen Lim emit(A64_LDR32(tmp, r2, tmp), ctx); 28716338a9bSDaniel Borkmann emit(A64_MOV(0, r3, r3), ctx); 288ddb55992SZi Shen Lim emit(A64_CMP(0, r3, tmp), ctx); 28916338a9bSDaniel Borkmann emit(A64_B_(A64_COND_CS, jmp_offset), ctx); 290ddb55992SZi Shen Lim 291ddb55992SZi Shen Lim /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) 292ddb55992SZi Shen Lim * goto out; 293ddb55992SZi Shen Lim * tail_call_cnt++; 294ddb55992SZi Shen Lim */ 295ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx); 296ddb55992SZi Shen Lim emit(A64_CMP(1, tcc, tmp), ctx); 29716338a9bSDaniel Borkmann emit(A64_B_(A64_COND_HI, jmp_offset), ctx); 298ddb55992SZi Shen Lim emit(A64_ADD_I(1, tcc, tcc, 1), ctx); 299ddb55992SZi Shen Lim 300ddb55992SZi Shen Lim /* prog = array->ptrs[index]; 301ddb55992SZi Shen Lim * if (prog == NULL) 302ddb55992SZi Shen Lim * goto out; 303ddb55992SZi Shen Lim */ 304ddb55992SZi Shen Lim off = offsetof(struct bpf_array, ptrs); 305ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 306d8b54110SDaniel Borkmann emit(A64_ADD(1, tmp, r2, tmp), ctx); 307d8b54110SDaniel Borkmann emit(A64_LSL(1, prg, r3, 3), ctx); 308d8b54110SDaniel Borkmann emit(A64_LDR64(prg, tmp, prg), ctx); 309ddb55992SZi Shen Lim emit(A64_CBZ(1, prg, jmp_offset), ctx); 310ddb55992SZi Shen Lim 311a2284d91SDaniel Borkmann /* goto *(prog->bpf_func + prologue_offset); */ 312ddb55992SZi Shen Lim off = offsetof(struct bpf_prog, bpf_func); 313ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 314ddb55992SZi Shen Lim emit(A64_LDR64(tmp, prg, tmp), ctx); 315ddb55992SZi Shen Lim emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx); 316a2284d91SDaniel Borkmann emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 317ddb55992SZi Shen Lim emit(A64_BR(tmp), ctx); 318ddb55992SZi Shen Lim 319ddb55992SZi Shen Lim /* out: */ 320ddb55992SZi Shen Lim if (out_offset == -1) 321ddb55992SZi Shen Lim out_offset = cur_offset; 322ddb55992SZi Shen Lim if (cur_offset != out_offset) { 323ddb55992SZi Shen Lim pr_err_once("tail_call out_offset = %d, expected %d!\n", 324ddb55992SZi Shen Lim cur_offset, out_offset); 325ddb55992SZi Shen Lim return -1; 326ddb55992SZi Shen Lim } 327ddb55992SZi Shen Lim return 0; 328ddb55992SZi Shen Lim #undef cur_offset 329ddb55992SZi Shen Lim #undef jmp_offset 330e54bcde3SZi Shen Lim } 331e54bcde3SZi Shen Lim 332e54bcde3SZi Shen Lim static void build_epilogue(struct jit_ctx *ctx) 333e54bcde3SZi Shen Lim { 334e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 335e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 336e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 337e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 338e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 339e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 340e54bcde3SZi Shen Lim 341e54bcde3SZi Shen Lim /* We're done with BPF stack */ 342f1c9eed7SDaniel Borkmann emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 343e54bcde3SZi Shen Lim 344ec0738dbSYang Shi /* Restore fs (x25) and x26 */ 345ec0738dbSYang Shi emit(A64_POP(fp, A64_R(26), A64_SP), ctx); 346ec0738dbSYang Shi 347e54bcde3SZi Shen Lim /* Restore callee-saved register */ 348e54bcde3SZi Shen Lim emit(A64_POP(r8, r9, A64_SP), ctx); 349e54bcde3SZi Shen Lim emit(A64_POP(r6, r7, A64_SP), ctx); 350e54bcde3SZi Shen Lim 351ec0738dbSYang Shi /* Restore FP/LR registers */ 352ec0738dbSYang Shi emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); 353e54bcde3SZi Shen Lim 354e54bcde3SZi Shen Lim /* Set return value */ 355e54bcde3SZi Shen Lim emit(A64_MOV(1, A64_R(0), r0), ctx); 356e54bcde3SZi Shen Lim 357e54bcde3SZi Shen Lim emit(A64_RET(A64_LR), ctx); 358e54bcde3SZi Shen Lim } 359e54bcde3SZi Shen Lim 36080083428SJean-Philippe Brucker #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 36180083428SJean-Philippe Brucker #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 36280083428SJean-Philippe Brucker 36380083428SJean-Philippe Brucker int arm64_bpf_fixup_exception(const struct exception_table_entry *ex, 36480083428SJean-Philippe Brucker struct pt_regs *regs) 36580083428SJean-Philippe Brucker { 36680083428SJean-Philippe Brucker off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 36780083428SJean-Philippe Brucker int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 36880083428SJean-Philippe Brucker 36980083428SJean-Philippe Brucker regs->regs[dst_reg] = 0; 37080083428SJean-Philippe Brucker regs->pc = (unsigned long)&ex->fixup - offset; 37180083428SJean-Philippe Brucker return 1; 37280083428SJean-Philippe Brucker } 37380083428SJean-Philippe Brucker 37480083428SJean-Philippe Brucker /* For accesses to BTF pointers, add an entry to the exception table */ 37580083428SJean-Philippe Brucker static int add_exception_handler(const struct bpf_insn *insn, 37680083428SJean-Philippe Brucker struct jit_ctx *ctx, 37780083428SJean-Philippe Brucker int dst_reg) 37880083428SJean-Philippe Brucker { 37980083428SJean-Philippe Brucker off_t offset; 38080083428SJean-Philippe Brucker unsigned long pc; 38180083428SJean-Philippe Brucker struct exception_table_entry *ex; 38280083428SJean-Philippe Brucker 38380083428SJean-Philippe Brucker if (!ctx->image) 38480083428SJean-Philippe Brucker /* First pass */ 38580083428SJean-Philippe Brucker return 0; 38680083428SJean-Philippe Brucker 38780083428SJean-Philippe Brucker if (BPF_MODE(insn->code) != BPF_PROBE_MEM) 38880083428SJean-Philippe Brucker return 0; 38980083428SJean-Philippe Brucker 39080083428SJean-Philippe Brucker if (!ctx->prog->aux->extable || 39180083428SJean-Philippe Brucker WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries)) 39280083428SJean-Philippe Brucker return -EINVAL; 39380083428SJean-Philippe Brucker 39480083428SJean-Philippe Brucker ex = &ctx->prog->aux->extable[ctx->exentry_idx]; 39580083428SJean-Philippe Brucker pc = (unsigned long)&ctx->image[ctx->idx - 1]; 39680083428SJean-Philippe Brucker 39780083428SJean-Philippe Brucker offset = pc - (long)&ex->insn; 39880083428SJean-Philippe Brucker if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 39980083428SJean-Philippe Brucker return -ERANGE; 40080083428SJean-Philippe Brucker ex->insn = offset; 40180083428SJean-Philippe Brucker 40280083428SJean-Philippe Brucker /* 40380083428SJean-Philippe Brucker * Since the extable follows the program, the fixup offset is always 40480083428SJean-Philippe Brucker * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 40580083428SJean-Philippe Brucker * to keep things simple, and put the destination register in the upper 40680083428SJean-Philippe Brucker * bits. We don't need to worry about buildtime or runtime sort 40780083428SJean-Philippe Brucker * modifying the upper bits because the table is already sorted, and 40880083428SJean-Philippe Brucker * isn't part of the main exception table. 40980083428SJean-Philippe Brucker */ 41080083428SJean-Philippe Brucker offset = (long)&ex->fixup - (pc + AARCH64_INSN_SIZE); 41180083428SJean-Philippe Brucker if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 41280083428SJean-Philippe Brucker return -ERANGE; 41380083428SJean-Philippe Brucker 41480083428SJean-Philippe Brucker ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | 41580083428SJean-Philippe Brucker FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 41680083428SJean-Philippe Brucker 41780083428SJean-Philippe Brucker ctx->exentry_idx++; 41880083428SJean-Philippe Brucker return 0; 41980083428SJean-Philippe Brucker } 42080083428SJean-Philippe Brucker 42130d3d94cSZi Shen Lim /* JITs an eBPF instruction. 42230d3d94cSZi Shen Lim * Returns: 42330d3d94cSZi Shen Lim * 0 - successfully JITed an 8-byte eBPF instruction. 42430d3d94cSZi Shen Lim * >0 - successfully JITed a 16-byte eBPF instruction. 42530d3d94cSZi Shen Lim * <0 - failed to JIT. 42630d3d94cSZi Shen Lim */ 4278c11ea5cSDaniel Borkmann static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 4288c11ea5cSDaniel Borkmann bool extra_pass) 429e54bcde3SZi Shen Lim { 430e54bcde3SZi Shen Lim const u8 code = insn->code; 431e54bcde3SZi Shen Lim const u8 dst = bpf2a64[insn->dst_reg]; 432e54bcde3SZi Shen Lim const u8 src = bpf2a64[insn->src_reg]; 433e54bcde3SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 434e54bcde3SZi Shen Lim const u8 tmp2 = bpf2a64[TMP_REG_2]; 4357005cadeSDaniel Borkmann const u8 tmp3 = bpf2a64[TMP_REG_3]; 436e54bcde3SZi Shen Lim const s16 off = insn->off; 437e54bcde3SZi Shen Lim const s32 imm = insn->imm; 438e54bcde3SZi Shen Lim const int i = insn - ctx->prog->insnsi; 439654b65a0SJiong Wang const bool is64 = BPF_CLASS(code) == BPF_ALU64 || 440654b65a0SJiong Wang BPF_CLASS(code) == BPF_JMP; 44185f68fe8SDaniel Borkmann const bool isdw = BPF_SIZE(code) == BPF_DW; 44234b8ab09SDaniel Borkmann u8 jmp_cond, reg; 443e54bcde3SZi Shen Lim s32 jmp_offset; 444fd49591cSLuke Nelson u32 a64_insn; 44580083428SJean-Philippe Brucker int ret; 446e54bcde3SZi Shen Lim 447251599e1SZi Shen Lim #define check_imm(bits, imm) do { \ 448251599e1SZi Shen Lim if ((((imm) > 0) && ((imm) >> (bits))) || \ 449251599e1SZi Shen Lim (((imm) < 0) && (~(imm) >> (bits)))) { \ 450251599e1SZi Shen Lim pr_info("[%2d] imm=%d(0x%x) out of range\n", \ 451251599e1SZi Shen Lim i, imm, imm); \ 452251599e1SZi Shen Lim return -EINVAL; \ 453251599e1SZi Shen Lim } \ 454251599e1SZi Shen Lim } while (0) 455251599e1SZi Shen Lim #define check_imm19(imm) check_imm(19, imm) 456251599e1SZi Shen Lim #define check_imm26(imm) check_imm(26, imm) 457251599e1SZi Shen Lim 458e54bcde3SZi Shen Lim switch (code) { 459e54bcde3SZi Shen Lim /* dst = src */ 460e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_X: 461e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_X: 462e54bcde3SZi Shen Lim emit(A64_MOV(is64, dst, src), ctx); 463e54bcde3SZi Shen Lim break; 464e54bcde3SZi Shen Lim /* dst = dst OP src */ 465e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_X: 466e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_X: 467e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, src), ctx); 468e54bcde3SZi Shen Lim break; 469e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_X: 470e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_X: 471e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, src), ctx); 472e54bcde3SZi Shen Lim break; 473e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_X: 474e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_X: 475e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, src), ctx); 476e54bcde3SZi Shen Lim break; 477e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_X: 478e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_X: 479e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, src), ctx); 480e54bcde3SZi Shen Lim break; 481e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_X: 482e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_X: 483e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, src), ctx); 484e54bcde3SZi Shen Lim break; 485e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_X: 486e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_X: 487e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, src), ctx); 488e54bcde3SZi Shen Lim break; 489e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_X: 490e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_X: 491e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOD | BPF_X: 492e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOD | BPF_X: 49314e589ffSZi Shen Lim switch (BPF_OP(code)) { 49414e589ffSZi Shen Lim case BPF_DIV: 495e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, src), ctx); 496e54bcde3SZi Shen Lim break; 49714e589ffSZi Shen Lim case BPF_MOD: 498e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, src), ctx); 499504792e0SJerin Jacob emit(A64_MSUB(is64, dst, dst, tmp, src), ctx); 500e54bcde3SZi Shen Lim break; 50114e589ffSZi Shen Lim } 50214e589ffSZi Shen Lim break; 503d65a634aSZi Shen Lim case BPF_ALU | BPF_LSH | BPF_X: 504d65a634aSZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_X: 505d65a634aSZi Shen Lim emit(A64_LSLV(is64, dst, dst, src), ctx); 506d65a634aSZi Shen Lim break; 507d65a634aSZi Shen Lim case BPF_ALU | BPF_RSH | BPF_X: 508d65a634aSZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_X: 509d65a634aSZi Shen Lim emit(A64_LSRV(is64, dst, dst, src), ctx); 510d65a634aSZi Shen Lim break; 511d65a634aSZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_X: 512d65a634aSZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_X: 513d65a634aSZi Shen Lim emit(A64_ASRV(is64, dst, dst, src), ctx); 514d65a634aSZi Shen Lim break; 515e54bcde3SZi Shen Lim /* dst = -dst */ 516e54bcde3SZi Shen Lim case BPF_ALU | BPF_NEG: 517e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_NEG: 518e54bcde3SZi Shen Lim emit(A64_NEG(is64, dst, dst), ctx); 519e54bcde3SZi Shen Lim break; 520e54bcde3SZi Shen Lim /* dst = BSWAP##imm(dst) */ 521e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_LE: 522e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_BE: 523e54bcde3SZi Shen Lim #ifdef CONFIG_CPU_BIG_ENDIAN 524e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_BE) 525d63903bbSXi Wang goto emit_bswap_uxt; 526e54bcde3SZi Shen Lim #else /* !CONFIG_CPU_BIG_ENDIAN */ 527e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_LE) 528d63903bbSXi Wang goto emit_bswap_uxt; 529e54bcde3SZi Shen Lim #endif 530e54bcde3SZi Shen Lim switch (imm) { 531e54bcde3SZi Shen Lim case 16: 532e54bcde3SZi Shen Lim emit(A64_REV16(is64, dst, dst), ctx); 533d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 534d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 535e54bcde3SZi Shen Lim break; 536e54bcde3SZi Shen Lim case 32: 537e54bcde3SZi Shen Lim emit(A64_REV32(is64, dst, dst), ctx); 538d63903bbSXi Wang /* upper 32 bits already cleared */ 539e54bcde3SZi Shen Lim break; 540e54bcde3SZi Shen Lim case 64: 541e54bcde3SZi Shen Lim emit(A64_REV64(dst, dst), ctx); 542e54bcde3SZi Shen Lim break; 543e54bcde3SZi Shen Lim } 544e54bcde3SZi Shen Lim break; 545d63903bbSXi Wang emit_bswap_uxt: 546d63903bbSXi Wang switch (imm) { 547d63903bbSXi Wang case 16: 548d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 549d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 550d63903bbSXi Wang break; 551d63903bbSXi Wang case 32: 552d63903bbSXi Wang /* zero-extend 32 bits into 64 bits */ 553d63903bbSXi Wang emit(A64_UXTW(is64, dst, dst), ctx); 554d63903bbSXi Wang break; 555d63903bbSXi Wang case 64: 556d63903bbSXi Wang /* nop */ 557d63903bbSXi Wang break; 558d63903bbSXi Wang } 559d63903bbSXi Wang break; 560e54bcde3SZi Shen Lim /* dst = imm */ 561e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_K: 562e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_K: 563e54bcde3SZi Shen Lim emit_a64_mov_i(is64, dst, imm, ctx); 564e54bcde3SZi Shen Lim break; 565e54bcde3SZi Shen Lim /* dst = dst OP imm */ 566e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_K: 567e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_K: 568fd868f14SLuke Nelson if (is_addsub_imm(imm)) { 569fd868f14SLuke Nelson emit(A64_ADD_I(is64, dst, dst, imm), ctx); 570fd868f14SLuke Nelson } else if (is_addsub_imm(-imm)) { 571fd868f14SLuke Nelson emit(A64_SUB_I(is64, dst, dst, -imm), ctx); 572fd868f14SLuke Nelson } else { 573e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 574e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, tmp), ctx); 575fd868f14SLuke Nelson } 576e54bcde3SZi Shen Lim break; 577e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_K: 578e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_K: 579fd868f14SLuke Nelson if (is_addsub_imm(imm)) { 580fd868f14SLuke Nelson emit(A64_SUB_I(is64, dst, dst, imm), ctx); 581fd868f14SLuke Nelson } else if (is_addsub_imm(-imm)) { 582fd868f14SLuke Nelson emit(A64_ADD_I(is64, dst, dst, -imm), ctx); 583fd868f14SLuke Nelson } else { 584e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 585e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 586fd868f14SLuke Nelson } 587e54bcde3SZi Shen Lim break; 588e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_K: 589e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_K: 590fd49591cSLuke Nelson a64_insn = A64_AND_I(is64, dst, dst, imm); 591fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 592fd49591cSLuke Nelson emit(a64_insn, ctx); 593fd49591cSLuke Nelson } else { 594e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 595e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, tmp), ctx); 596fd49591cSLuke Nelson } 597e54bcde3SZi Shen Lim break; 598e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_K: 599e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_K: 600fd49591cSLuke Nelson a64_insn = A64_ORR_I(is64, dst, dst, imm); 601fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 602fd49591cSLuke Nelson emit(a64_insn, ctx); 603fd49591cSLuke Nelson } else { 604e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 605e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, tmp), ctx); 606fd49591cSLuke Nelson } 607e54bcde3SZi Shen Lim break; 608e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_K: 609e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_K: 610fd49591cSLuke Nelson a64_insn = A64_EOR_I(is64, dst, dst, imm); 611fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 612fd49591cSLuke Nelson emit(a64_insn, ctx); 613fd49591cSLuke Nelson } else { 614e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 615e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, tmp), ctx); 616fd49591cSLuke Nelson } 617e54bcde3SZi Shen Lim break; 618e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_K: 619e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_K: 620e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 621e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, tmp), ctx); 622e54bcde3SZi Shen Lim break; 623e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_K: 624e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_K: 625e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 626e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, tmp), ctx); 627e54bcde3SZi Shen Lim break; 628e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOD | BPF_K: 629e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOD | BPF_K: 630e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp2, imm, ctx); 631e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, tmp2), ctx); 632504792e0SJerin Jacob emit(A64_MSUB(is64, dst, dst, tmp, tmp2), ctx); 633e54bcde3SZi Shen Lim break; 634e54bcde3SZi Shen Lim case BPF_ALU | BPF_LSH | BPF_K: 635e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_K: 636e54bcde3SZi Shen Lim emit(A64_LSL(is64, dst, dst, imm), ctx); 637e54bcde3SZi Shen Lim break; 638e54bcde3SZi Shen Lim case BPF_ALU | BPF_RSH | BPF_K: 639e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_K: 640e54bcde3SZi Shen Lim emit(A64_LSR(is64, dst, dst, imm), ctx); 641e54bcde3SZi Shen Lim break; 642e54bcde3SZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_K: 643e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_K: 644e54bcde3SZi Shen Lim emit(A64_ASR(is64, dst, dst, imm), ctx); 645e54bcde3SZi Shen Lim break; 646e54bcde3SZi Shen Lim 647e54bcde3SZi Shen Lim /* JUMP off */ 648e54bcde3SZi Shen Lim case BPF_JMP | BPF_JA: 64932f6865cSIlias Apalodimas jmp_offset = bpf2a64_offset(i, off, ctx); 650e54bcde3SZi Shen Lim check_imm26(jmp_offset); 651e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 652e54bcde3SZi Shen Lim break; 653e54bcde3SZi Shen Lim /* IF (dst COND src) JUMP off */ 654e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_X: 655e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_X: 656c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_X: 657e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_X: 658c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_X: 659e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_X: 660e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_X: 661c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_X: 662e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_X: 663c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_X: 664654b65a0SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_X: 665654b65a0SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_X: 666654b65a0SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_X: 667654b65a0SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_X: 668654b65a0SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_X: 669654b65a0SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_X: 670654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_X: 671654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_X: 672654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_X: 673654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_X: 674654b65a0SJiong Wang emit(A64_CMP(is64, dst, src), ctx); 675e54bcde3SZi Shen Lim emit_cond_jmp: 67632f6865cSIlias Apalodimas jmp_offset = bpf2a64_offset(i, off, ctx); 677e54bcde3SZi Shen Lim check_imm19(jmp_offset); 678e54bcde3SZi Shen Lim switch (BPF_OP(code)) { 679e54bcde3SZi Shen Lim case BPF_JEQ: 680e54bcde3SZi Shen Lim jmp_cond = A64_COND_EQ; 681e54bcde3SZi Shen Lim break; 682e54bcde3SZi Shen Lim case BPF_JGT: 683e54bcde3SZi Shen Lim jmp_cond = A64_COND_HI; 684e54bcde3SZi Shen Lim break; 685c362b2f3SDaniel Borkmann case BPF_JLT: 686c362b2f3SDaniel Borkmann jmp_cond = A64_COND_CC; 687c362b2f3SDaniel Borkmann break; 688e54bcde3SZi Shen Lim case BPF_JGE: 689e54bcde3SZi Shen Lim jmp_cond = A64_COND_CS; 690e54bcde3SZi Shen Lim break; 691c362b2f3SDaniel Borkmann case BPF_JLE: 692c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LS; 693c362b2f3SDaniel Borkmann break; 69498397fc5SZi Shen Lim case BPF_JSET: 695e54bcde3SZi Shen Lim case BPF_JNE: 696e54bcde3SZi Shen Lim jmp_cond = A64_COND_NE; 697e54bcde3SZi Shen Lim break; 698e54bcde3SZi Shen Lim case BPF_JSGT: 699e54bcde3SZi Shen Lim jmp_cond = A64_COND_GT; 700e54bcde3SZi Shen Lim break; 701c362b2f3SDaniel Borkmann case BPF_JSLT: 702c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LT; 703c362b2f3SDaniel Borkmann break; 704e54bcde3SZi Shen Lim case BPF_JSGE: 705e54bcde3SZi Shen Lim jmp_cond = A64_COND_GE; 706e54bcde3SZi Shen Lim break; 707c362b2f3SDaniel Borkmann case BPF_JSLE: 708c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LE; 709c362b2f3SDaniel Borkmann break; 710e54bcde3SZi Shen Lim default: 711e54bcde3SZi Shen Lim return -EFAULT; 712e54bcde3SZi Shen Lim } 713e54bcde3SZi Shen Lim emit(A64_B_(jmp_cond, jmp_offset), ctx); 714e54bcde3SZi Shen Lim break; 715e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_X: 716654b65a0SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_X: 717654b65a0SJiong Wang emit(A64_TST(is64, dst, src), ctx); 718e54bcde3SZi Shen Lim goto emit_cond_jmp; 719e54bcde3SZi Shen Lim /* IF (dst COND imm) JUMP off */ 720e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_K: 721e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_K: 722c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_K: 723e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_K: 724c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_K: 725e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_K: 726e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_K: 727c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_K: 728e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_K: 729c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_K: 730654b65a0SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_K: 731654b65a0SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_K: 732654b65a0SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_K: 733654b65a0SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_K: 734654b65a0SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_K: 735654b65a0SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_K: 736654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_K: 737654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_K: 738654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_K: 739654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_K: 740fd868f14SLuke Nelson if (is_addsub_imm(imm)) { 741fd868f14SLuke Nelson emit(A64_CMP_I(is64, dst, imm), ctx); 742fd868f14SLuke Nelson } else if (is_addsub_imm(-imm)) { 743fd868f14SLuke Nelson emit(A64_CMN_I(is64, dst, -imm), ctx); 744fd868f14SLuke Nelson } else { 745654b65a0SJiong Wang emit_a64_mov_i(is64, tmp, imm, ctx); 746654b65a0SJiong Wang emit(A64_CMP(is64, dst, tmp), ctx); 747fd868f14SLuke Nelson } 748e54bcde3SZi Shen Lim goto emit_cond_jmp; 749e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_K: 750654b65a0SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_K: 751fd49591cSLuke Nelson a64_insn = A64_TST_I(is64, dst, imm); 752fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 753fd49591cSLuke Nelson emit(a64_insn, ctx); 754fd49591cSLuke Nelson } else { 755654b65a0SJiong Wang emit_a64_mov_i(is64, tmp, imm, ctx); 756654b65a0SJiong Wang emit(A64_TST(is64, dst, tmp), ctx); 757fd49591cSLuke Nelson } 758e54bcde3SZi Shen Lim goto emit_cond_jmp; 759e54bcde3SZi Shen Lim /* function call */ 760e54bcde3SZi Shen Lim case BPF_JMP | BPF_CALL: 761e54bcde3SZi Shen Lim { 762e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 7638c11ea5cSDaniel Borkmann bool func_addr_fixed; 7648c11ea5cSDaniel Borkmann u64 func_addr; 765e54bcde3SZi Shen Lim 7668c11ea5cSDaniel Borkmann ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 7678c11ea5cSDaniel Borkmann &func_addr, &func_addr_fixed); 7688c11ea5cSDaniel Borkmann if (ret < 0) 7698c11ea5cSDaniel Borkmann return ret; 7708c11ea5cSDaniel Borkmann emit_addr_mov_i64(tmp, func_addr, ctx); 771e54bcde3SZi Shen Lim emit(A64_BLR(tmp), ctx); 772e54bcde3SZi Shen Lim emit(A64_MOV(1, r0, A64_R(0)), ctx); 773e54bcde3SZi Shen Lim break; 774e54bcde3SZi Shen Lim } 775ddb55992SZi Shen Lim /* tail call */ 77671189fa9SAlexei Starovoitov case BPF_JMP | BPF_TAIL_CALL: 777ddb55992SZi Shen Lim if (emit_bpf_tail_call(ctx)) 778ddb55992SZi Shen Lim return -EFAULT; 779ddb55992SZi Shen Lim break; 780e54bcde3SZi Shen Lim /* function return */ 781e54bcde3SZi Shen Lim case BPF_JMP | BPF_EXIT: 78251c9fbb1SZi Shen Lim /* Optimization: when last instruction is EXIT, 78351c9fbb1SZi Shen Lim simply fallthrough to epilogue. */ 784e54bcde3SZi Shen Lim if (i == ctx->prog->len - 1) 785e54bcde3SZi Shen Lim break; 786e54bcde3SZi Shen Lim jmp_offset = epilogue_offset(ctx); 787e54bcde3SZi Shen Lim check_imm26(jmp_offset); 788e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 789e54bcde3SZi Shen Lim break; 790e54bcde3SZi Shen Lim 79130d3d94cSZi Shen Lim /* dst = imm64 */ 79230d3d94cSZi Shen Lim case BPF_LD | BPF_IMM | BPF_DW: 79330d3d94cSZi Shen Lim { 79430d3d94cSZi Shen Lim const struct bpf_insn insn1 = insn[1]; 79530d3d94cSZi Shen Lim u64 imm64; 79630d3d94cSZi Shen Lim 7971e4df6b7SXi Wang imm64 = (u64)insn1.imm << 32 | (u32)imm; 79830d3d94cSZi Shen Lim emit_a64_mov_i64(dst, imm64, ctx); 79930d3d94cSZi Shen Lim 80030d3d94cSZi Shen Lim return 1; 80130d3d94cSZi Shen Lim } 80230d3d94cSZi Shen Lim 803e54bcde3SZi Shen Lim /* LDX: dst = *(size *)(src + off) */ 804e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_W: 805e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_H: 806e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_B: 807e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_DW: 80880083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 80980083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_W: 81080083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_H: 81180083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_B: 812e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 813e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 814e54bcde3SZi Shen Lim case BPF_W: 815e54bcde3SZi Shen Lim emit(A64_LDR32(dst, src, tmp), ctx); 816e54bcde3SZi Shen Lim break; 817e54bcde3SZi Shen Lim case BPF_H: 818e54bcde3SZi Shen Lim emit(A64_LDRH(dst, src, tmp), ctx); 819e54bcde3SZi Shen Lim break; 820e54bcde3SZi Shen Lim case BPF_B: 821e54bcde3SZi Shen Lim emit(A64_LDRB(dst, src, tmp), ctx); 822e54bcde3SZi Shen Lim break; 823e54bcde3SZi Shen Lim case BPF_DW: 824e54bcde3SZi Shen Lim emit(A64_LDR64(dst, src, tmp), ctx); 825e54bcde3SZi Shen Lim break; 826e54bcde3SZi Shen Lim } 82780083428SJean-Philippe Brucker 82880083428SJean-Philippe Brucker ret = add_exception_handler(insn, ctx, dst); 82980083428SJean-Philippe Brucker if (ret) 83080083428SJean-Philippe Brucker return ret; 831e54bcde3SZi Shen Lim break; 832e54bcde3SZi Shen Lim 833e54bcde3SZi Shen Lim /* ST: *(size *)(dst + off) = imm */ 834e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_W: 835e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_H: 836e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_B: 837e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_DW: 838df849ba3SYang Shi /* Load imm to a register then store it */ 839df849ba3SYang Shi emit_a64_mov_i(1, tmp2, off, ctx); 840df849ba3SYang Shi emit_a64_mov_i(1, tmp, imm, ctx); 841df849ba3SYang Shi switch (BPF_SIZE(code)) { 842df849ba3SYang Shi case BPF_W: 843df849ba3SYang Shi emit(A64_STR32(tmp, dst, tmp2), ctx); 844df849ba3SYang Shi break; 845df849ba3SYang Shi case BPF_H: 846df849ba3SYang Shi emit(A64_STRH(tmp, dst, tmp2), ctx); 847df849ba3SYang Shi break; 848df849ba3SYang Shi case BPF_B: 849df849ba3SYang Shi emit(A64_STRB(tmp, dst, tmp2), ctx); 850df849ba3SYang Shi break; 851df849ba3SYang Shi case BPF_DW: 852df849ba3SYang Shi emit(A64_STR64(tmp, dst, tmp2), ctx); 853df849ba3SYang Shi break; 854df849ba3SYang Shi } 855df849ba3SYang Shi break; 856e54bcde3SZi Shen Lim 857e54bcde3SZi Shen Lim /* STX: *(size *)(dst + off) = src */ 858e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_W: 859e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_H: 860e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_B: 861e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_DW: 862e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 863e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 864e54bcde3SZi Shen Lim case BPF_W: 865e54bcde3SZi Shen Lim emit(A64_STR32(src, dst, tmp), ctx); 866e54bcde3SZi Shen Lim break; 867e54bcde3SZi Shen Lim case BPF_H: 868e54bcde3SZi Shen Lim emit(A64_STRH(src, dst, tmp), ctx); 869e54bcde3SZi Shen Lim break; 870e54bcde3SZi Shen Lim case BPF_B: 871e54bcde3SZi Shen Lim emit(A64_STRB(src, dst, tmp), ctx); 872e54bcde3SZi Shen Lim break; 873e54bcde3SZi Shen Lim case BPF_DW: 874e54bcde3SZi Shen Lim emit(A64_STR64(src, dst, tmp), ctx); 875e54bcde3SZi Shen Lim break; 876e54bcde3SZi Shen Lim } 877e54bcde3SZi Shen Lim break; 87834b8ab09SDaniel Borkmann 87991c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_W: 88091c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_DW: 88191c960b0SBrendan Jackman if (insn->imm != BPF_ADD) { 88291c960b0SBrendan Jackman pr_err_once("unknown atomic op code %02x\n", insn->imm); 88391c960b0SBrendan Jackman return -EINVAL; 88491c960b0SBrendan Jackman } 88591c960b0SBrendan Jackman 88691c960b0SBrendan Jackman /* STX XADD: lock *(u32 *)(dst + off) += src 88791c960b0SBrendan Jackman * and 88891c960b0SBrendan Jackman * STX XADD: lock *(u64 *)(dst + off) += src 88991c960b0SBrendan Jackman */ 89091c960b0SBrendan Jackman 89134b8ab09SDaniel Borkmann if (!off) { 89234b8ab09SDaniel Borkmann reg = dst; 89334b8ab09SDaniel Borkmann } else { 89485f68fe8SDaniel Borkmann emit_a64_mov_i(1, tmp, off, ctx); 89585f68fe8SDaniel Borkmann emit(A64_ADD(1, tmp, tmp, dst), ctx); 89634b8ab09SDaniel Borkmann reg = tmp; 89734b8ab09SDaniel Borkmann } 89834b8ab09SDaniel Borkmann if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS)) { 89934b8ab09SDaniel Borkmann emit(A64_STADD(isdw, reg, src), ctx); 90034b8ab09SDaniel Borkmann } else { 90134b8ab09SDaniel Borkmann emit(A64_LDXR(isdw, tmp2, reg), ctx); 90285f68fe8SDaniel Borkmann emit(A64_ADD(isdw, tmp2, tmp2, src), ctx); 90334b8ab09SDaniel Borkmann emit(A64_STXR(isdw, tmp2, reg, tmp3), ctx); 90485f68fe8SDaniel Borkmann jmp_offset = -3; 90585f68fe8SDaniel Borkmann check_imm19(jmp_offset); 9067005cadeSDaniel Borkmann emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); 90734b8ab09SDaniel Borkmann } 90885f68fe8SDaniel Borkmann break; 909e54bcde3SZi Shen Lim 910e54bcde3SZi Shen Lim default: 911e54bcde3SZi Shen Lim pr_err_once("unknown opcode %02x\n", code); 912e54bcde3SZi Shen Lim return -EINVAL; 913e54bcde3SZi Shen Lim } 914e54bcde3SZi Shen Lim 915e54bcde3SZi Shen Lim return 0; 916e54bcde3SZi Shen Lim } 917e54bcde3SZi Shen Lim 9188c11ea5cSDaniel Borkmann static int build_body(struct jit_ctx *ctx, bool extra_pass) 919e54bcde3SZi Shen Lim { 920e54bcde3SZi Shen Lim const struct bpf_prog *prog = ctx->prog; 921e54bcde3SZi Shen Lim int i; 922e54bcde3SZi Shen Lim 92332f6865cSIlias Apalodimas /* 92432f6865cSIlias Apalodimas * - offset[0] offset of the end of prologue, 92532f6865cSIlias Apalodimas * start of the 1st instruction. 92632f6865cSIlias Apalodimas * - offset[1] - offset of the end of 1st instruction, 92732f6865cSIlias Apalodimas * start of the 2nd instruction 92832f6865cSIlias Apalodimas * [....] 92932f6865cSIlias Apalodimas * - offset[3] - offset of the end of 3rd instruction, 93032f6865cSIlias Apalodimas * start of 4th instruction 93132f6865cSIlias Apalodimas */ 932e54bcde3SZi Shen Lim for (i = 0; i < prog->len; i++) { 933e54bcde3SZi Shen Lim const struct bpf_insn *insn = &prog->insnsi[i]; 934e54bcde3SZi Shen Lim int ret; 935e54bcde3SZi Shen Lim 93632f6865cSIlias Apalodimas if (ctx->image == NULL) 93732f6865cSIlias Apalodimas ctx->offset[i] = ctx->idx; 9388c11ea5cSDaniel Borkmann ret = build_insn(insn, ctx, extra_pass); 93930d3d94cSZi Shen Lim if (ret > 0) { 94030d3d94cSZi Shen Lim i++; 941ddc665a4SDaniel Borkmann if (ctx->image == NULL) 942ddc665a4SDaniel Borkmann ctx->offset[i] = ctx->idx; 94330d3d94cSZi Shen Lim continue; 94430d3d94cSZi Shen Lim } 945e54bcde3SZi Shen Lim if (ret) 946e54bcde3SZi Shen Lim return ret; 947e54bcde3SZi Shen Lim } 94832f6865cSIlias Apalodimas /* 94932f6865cSIlias Apalodimas * offset is allocated with prog->len + 1 so fill in 95032f6865cSIlias Apalodimas * the last element with the offset after the last 95132f6865cSIlias Apalodimas * instruction (end of program) 95232f6865cSIlias Apalodimas */ 95332f6865cSIlias Apalodimas if (ctx->image == NULL) 95432f6865cSIlias Apalodimas ctx->offset[i] = ctx->idx; 955e54bcde3SZi Shen Lim 956e54bcde3SZi Shen Lim return 0; 957e54bcde3SZi Shen Lim } 958e54bcde3SZi Shen Lim 95942ff712bSZi Shen Lim static int validate_code(struct jit_ctx *ctx) 96042ff712bSZi Shen Lim { 96142ff712bSZi Shen Lim int i; 96242ff712bSZi Shen Lim 96342ff712bSZi Shen Lim for (i = 0; i < ctx->idx; i++) { 96442ff712bSZi Shen Lim u32 a64_insn = le32_to_cpu(ctx->image[i]); 96542ff712bSZi Shen Lim 96642ff712bSZi Shen Lim if (a64_insn == AARCH64_BREAK_FAULT) 96742ff712bSZi Shen Lim return -1; 96842ff712bSZi Shen Lim } 96942ff712bSZi Shen Lim 97080083428SJean-Philippe Brucker if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries)) 97180083428SJean-Philippe Brucker return -1; 97280083428SJean-Philippe Brucker 97342ff712bSZi Shen Lim return 0; 97442ff712bSZi Shen Lim } 97542ff712bSZi Shen Lim 976e54bcde3SZi Shen Lim static inline void bpf_flush_icache(void *start, void *end) 977e54bcde3SZi Shen Lim { 978e54bcde3SZi Shen Lim flush_icache_range((unsigned long)start, (unsigned long)end); 979e54bcde3SZi Shen Lim } 980e54bcde3SZi Shen Lim 981db496944SAlexei Starovoitov struct arm64_jit_data { 982db496944SAlexei Starovoitov struct bpf_binary_header *header; 983db496944SAlexei Starovoitov u8 *image; 984db496944SAlexei Starovoitov struct jit_ctx ctx; 985db496944SAlexei Starovoitov }; 986db496944SAlexei Starovoitov 987d1c55ab5SDaniel Borkmann struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 988e54bcde3SZi Shen Lim { 98980083428SJean-Philippe Brucker int image_size, prog_size, extable_size; 99026eb042eSDaniel Borkmann struct bpf_prog *tmp, *orig_prog = prog; 991b569c1c6SDaniel Borkmann struct bpf_binary_header *header; 992db496944SAlexei Starovoitov struct arm64_jit_data *jit_data; 99356ea6a8bSDaniel Borkmann bool was_classic = bpf_prog_was_classic(prog); 99426eb042eSDaniel Borkmann bool tmp_blinded = false; 995db496944SAlexei Starovoitov bool extra_pass = false; 996e54bcde3SZi Shen Lim struct jit_ctx ctx; 997b569c1c6SDaniel Borkmann u8 *image_ptr; 998e54bcde3SZi Shen Lim 99960b58afcSAlexei Starovoitov if (!prog->jit_requested) 100026eb042eSDaniel Borkmann return orig_prog; 100126eb042eSDaniel Borkmann 100226eb042eSDaniel Borkmann tmp = bpf_jit_blind_constants(prog); 100326eb042eSDaniel Borkmann /* If blinding was requested and we failed during blinding, 100426eb042eSDaniel Borkmann * we must fall back to the interpreter. 100526eb042eSDaniel Borkmann */ 100626eb042eSDaniel Borkmann if (IS_ERR(tmp)) 100726eb042eSDaniel Borkmann return orig_prog; 100826eb042eSDaniel Borkmann if (tmp != prog) { 100926eb042eSDaniel Borkmann tmp_blinded = true; 101026eb042eSDaniel Borkmann prog = tmp; 101126eb042eSDaniel Borkmann } 1012e54bcde3SZi Shen Lim 1013db496944SAlexei Starovoitov jit_data = prog->aux->jit_data; 1014db496944SAlexei Starovoitov if (!jit_data) { 1015db496944SAlexei Starovoitov jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1016db496944SAlexei Starovoitov if (!jit_data) { 1017db496944SAlexei Starovoitov prog = orig_prog; 1018db496944SAlexei Starovoitov goto out; 1019db496944SAlexei Starovoitov } 1020db496944SAlexei Starovoitov prog->aux->jit_data = jit_data; 1021db496944SAlexei Starovoitov } 1022db496944SAlexei Starovoitov if (jit_data->ctx.offset) { 1023db496944SAlexei Starovoitov ctx = jit_data->ctx; 1024db496944SAlexei Starovoitov image_ptr = jit_data->image; 1025db496944SAlexei Starovoitov header = jit_data->header; 1026db496944SAlexei Starovoitov extra_pass = true; 102780083428SJean-Philippe Brucker prog_size = sizeof(u32) * ctx.idx; 1028db496944SAlexei Starovoitov goto skip_init_ctx; 1029db496944SAlexei Starovoitov } 1030e54bcde3SZi Shen Lim memset(&ctx, 0, sizeof(ctx)); 1031e54bcde3SZi Shen Lim ctx.prog = prog; 1032e54bcde3SZi Shen Lim 103332f6865cSIlias Apalodimas ctx.offset = kcalloc(prog->len + 1, sizeof(int), GFP_KERNEL); 103426eb042eSDaniel Borkmann if (ctx.offset == NULL) { 103526eb042eSDaniel Borkmann prog = orig_prog; 1036db496944SAlexei Starovoitov goto out_off; 103726eb042eSDaniel Borkmann } 1038e54bcde3SZi Shen Lim 1039e54bcde3SZi Shen Lim /* 1. Initial fake pass to compute ctx->idx. */ 1040e54bcde3SZi Shen Lim 10414c1cd4fdSYang Shi /* Fake pass to fill in ctx->offset. */ 10428c11ea5cSDaniel Borkmann if (build_body(&ctx, extra_pass)) { 104326eb042eSDaniel Borkmann prog = orig_prog; 104426eb042eSDaniel Borkmann goto out_off; 104526eb042eSDaniel Borkmann } 1046e54bcde3SZi Shen Lim 104756ea6a8bSDaniel Borkmann if (build_prologue(&ctx, was_classic)) { 1048ddb55992SZi Shen Lim prog = orig_prog; 1049ddb55992SZi Shen Lim goto out_off; 1050ddb55992SZi Shen Lim } 105151c9fbb1SZi Shen Lim 105251c9fbb1SZi Shen Lim ctx.epilogue_offset = ctx.idx; 1053e54bcde3SZi Shen Lim build_epilogue(&ctx); 1054e54bcde3SZi Shen Lim 105580083428SJean-Philippe Brucker extable_size = prog->aux->num_exentries * 105680083428SJean-Philippe Brucker sizeof(struct exception_table_entry); 105780083428SJean-Philippe Brucker 1058e54bcde3SZi Shen Lim /* Now we know the actual image size. */ 105980083428SJean-Philippe Brucker prog_size = sizeof(u32) * ctx.idx; 106080083428SJean-Philippe Brucker image_size = prog_size + extable_size; 1061b569c1c6SDaniel Borkmann header = bpf_jit_binary_alloc(image_size, &image_ptr, 1062b569c1c6SDaniel Borkmann sizeof(u32), jit_fill_hole); 106326eb042eSDaniel Borkmann if (header == NULL) { 106426eb042eSDaniel Borkmann prog = orig_prog; 106526eb042eSDaniel Borkmann goto out_off; 106626eb042eSDaniel Borkmann } 1067e54bcde3SZi Shen Lim 1068e54bcde3SZi Shen Lim /* 2. Now, the actual pass. */ 1069e54bcde3SZi Shen Lim 1070425e1ed7SLuc Van Oostenryck ctx.image = (__le32 *)image_ptr; 107180083428SJean-Philippe Brucker if (extable_size) 107280083428SJean-Philippe Brucker prog->aux->extable = (void *)image_ptr + prog_size; 1073db496944SAlexei Starovoitov skip_init_ctx: 1074e54bcde3SZi Shen Lim ctx.idx = 0; 107580083428SJean-Philippe Brucker ctx.exentry_idx = 0; 1076b569c1c6SDaniel Borkmann 107756ea6a8bSDaniel Borkmann build_prologue(&ctx, was_classic); 1078e54bcde3SZi Shen Lim 10798c11ea5cSDaniel Borkmann if (build_body(&ctx, extra_pass)) { 1080b569c1c6SDaniel Borkmann bpf_jit_binary_free(header); 108126eb042eSDaniel Borkmann prog = orig_prog; 108226eb042eSDaniel Borkmann goto out_off; 108360ef0494SDaniel Borkmann } 1084e54bcde3SZi Shen Lim 1085e54bcde3SZi Shen Lim build_epilogue(&ctx); 1086e54bcde3SZi Shen Lim 108742ff712bSZi Shen Lim /* 3. Extra pass to validate JITed code. */ 108842ff712bSZi Shen Lim if (validate_code(&ctx)) { 108942ff712bSZi Shen Lim bpf_jit_binary_free(header); 109026eb042eSDaniel Borkmann prog = orig_prog; 109126eb042eSDaniel Borkmann goto out_off; 109242ff712bSZi Shen Lim } 109342ff712bSZi Shen Lim 1094e54bcde3SZi Shen Lim /* And we're done. */ 1095e54bcde3SZi Shen Lim if (bpf_jit_enable > 1) 109680083428SJean-Philippe Brucker bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 1097e54bcde3SZi Shen Lim 1098c3d4c682SDaniel Borkmann bpf_flush_icache(header, ctx.image + ctx.idx); 1099b569c1c6SDaniel Borkmann 1100db496944SAlexei Starovoitov if (!prog->is_func || extra_pass) { 1101db496944SAlexei Starovoitov if (extra_pass && ctx.idx != jit_data->ctx.idx) { 1102db496944SAlexei Starovoitov pr_err_once("multi-func JIT bug %d != %d\n", 1103db496944SAlexei Starovoitov ctx.idx, jit_data->ctx.idx); 1104db496944SAlexei Starovoitov bpf_jit_binary_free(header); 1105db496944SAlexei Starovoitov prog->bpf_func = NULL; 1106db496944SAlexei Starovoitov prog->jited = 0; 1107db496944SAlexei Starovoitov goto out_off; 1108db496944SAlexei Starovoitov } 11099d876e79SDaniel Borkmann bpf_jit_binary_lock_ro(header); 1110db496944SAlexei Starovoitov } else { 1111db496944SAlexei Starovoitov jit_data->ctx = ctx; 1112db496944SAlexei Starovoitov jit_data->image = image_ptr; 1113db496944SAlexei Starovoitov jit_data->header = header; 1114db496944SAlexei Starovoitov } 1115e54bcde3SZi Shen Lim prog->bpf_func = (void *)ctx.image; 1116a91263d5SDaniel Borkmann prog->jited = 1; 111780083428SJean-Philippe Brucker prog->jited_len = prog_size; 111826eb042eSDaniel Borkmann 1119db496944SAlexei Starovoitov if (!prog->is_func || extra_pass) { 112032f6865cSIlias Apalodimas bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); 112126eb042eSDaniel Borkmann out_off: 1122e54bcde3SZi Shen Lim kfree(ctx.offset); 1123db496944SAlexei Starovoitov kfree(jit_data); 1124db496944SAlexei Starovoitov prog->aux->jit_data = NULL; 1125db496944SAlexei Starovoitov } 112626eb042eSDaniel Borkmann out: 112726eb042eSDaniel Borkmann if (tmp_blinded) 112826eb042eSDaniel Borkmann bpf_jit_prog_release_other(prog, prog == orig_prog ? 112926eb042eSDaniel Borkmann tmp : orig_prog); 1130d1c55ab5SDaniel Borkmann return prog; 1131e54bcde3SZi Shen Lim } 113291fc957cSArd Biesheuvel 113391fc957cSArd Biesheuvel void *bpf_jit_alloc_exec(unsigned long size) 113491fc957cSArd Biesheuvel { 113591fc957cSArd Biesheuvel return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, 113691fc957cSArd Biesheuvel BPF_JIT_REGION_END, GFP_KERNEL, 11373f750706SArd Biesheuvel PAGE_KERNEL, 0, NUMA_NO_NODE, 113891fc957cSArd Biesheuvel __builtin_return_address(0)); 113991fc957cSArd Biesheuvel } 114091fc957cSArd Biesheuvel 114191fc957cSArd Biesheuvel void bpf_jit_free_exec(void *addr) 114291fc957cSArd Biesheuvel { 114391fc957cSArd Biesheuvel return vfree(addr); 114491fc957cSArd Biesheuvel } 1145