1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e54bcde3SZi Shen Lim /* 3e54bcde3SZi Shen Lim * BPF JIT compiler for ARM64 4e54bcde3SZi Shen Lim * 542ff712bSZi Shen Lim * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com> 6e54bcde3SZi Shen Lim */ 7e54bcde3SZi Shen Lim 8e54bcde3SZi Shen Lim #define pr_fmt(fmt) "bpf_jit: " fmt 9e54bcde3SZi Shen Lim 1080083428SJean-Philippe Brucker #include <linux/bitfield.h> 11ddb55992SZi Shen Lim #include <linux/bpf.h> 12e54bcde3SZi Shen Lim #include <linux/filter.h> 13e54bcde3SZi Shen Lim #include <linux/printk.h> 14e54bcde3SZi Shen Lim #include <linux/slab.h> 15b569c1c6SDaniel Borkmann 16e54bcde3SZi Shen Lim #include <asm/byteorder.h> 17e54bcde3SZi Shen Lim #include <asm/cacheflush.h> 18b569c1c6SDaniel Borkmann #include <asm/debug-monitors.h> 193e00e39dSMark Rutland #include <asm/insn.h> 20d4bbc30bSLaura Abbott #include <asm/set_memory.h> 21e54bcde3SZi Shen Lim 22e54bcde3SZi Shen Lim #include "bpf_jit.h" 23e54bcde3SZi Shen Lim 2426eb042eSDaniel Borkmann #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) 2526eb042eSDaniel Borkmann #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 26ddb55992SZi Shen Lim #define TCALL_CNT (MAX_BPF_JIT_REG + 2) 277005cadeSDaniel Borkmann #define TMP_REG_3 (MAX_BPF_JIT_REG + 3) 28e54bcde3SZi Shen Lim 29e54bcde3SZi Shen Lim /* Map BPF registers to A64 registers */ 30e54bcde3SZi Shen Lim static const int bpf2a64[] = { 31e54bcde3SZi Shen Lim /* return value from in-kernel function, and exit value from eBPF */ 32e54bcde3SZi Shen Lim [BPF_REG_0] = A64_R(7), 33e54bcde3SZi Shen Lim /* arguments from eBPF program to in-kernel function */ 34e54bcde3SZi Shen Lim [BPF_REG_1] = A64_R(0), 35e54bcde3SZi Shen Lim [BPF_REG_2] = A64_R(1), 36e54bcde3SZi Shen Lim [BPF_REG_3] = A64_R(2), 37e54bcde3SZi Shen Lim [BPF_REG_4] = A64_R(3), 38e54bcde3SZi Shen Lim [BPF_REG_5] = A64_R(4), 39e54bcde3SZi Shen Lim /* callee saved registers that in-kernel function will preserve */ 40e54bcde3SZi Shen Lim [BPF_REG_6] = A64_R(19), 41e54bcde3SZi Shen Lim [BPF_REG_7] = A64_R(20), 42e54bcde3SZi Shen Lim [BPF_REG_8] = A64_R(21), 43e54bcde3SZi Shen Lim [BPF_REG_9] = A64_R(22), 44e54bcde3SZi Shen Lim /* read-only frame pointer to access stack */ 45ec0738dbSYang Shi [BPF_REG_FP] = A64_R(25), 464c1cd4fdSYang Shi /* temporary registers for internal BPF JIT */ 474c1cd4fdSYang Shi [TMP_REG_1] = A64_R(10), 484c1cd4fdSYang Shi [TMP_REG_2] = A64_R(11), 497005cadeSDaniel Borkmann [TMP_REG_3] = A64_R(12), 50ddb55992SZi Shen Lim /* tail_call_cnt */ 51ddb55992SZi Shen Lim [TCALL_CNT] = A64_R(26), 5226eb042eSDaniel Borkmann /* temporary register for blinding constants */ 5326eb042eSDaniel Borkmann [BPF_REG_AX] = A64_R(9), 54e54bcde3SZi Shen Lim }; 55e54bcde3SZi Shen Lim 56e54bcde3SZi Shen Lim struct jit_ctx { 57e54bcde3SZi Shen Lim const struct bpf_prog *prog; 58e54bcde3SZi Shen Lim int idx; 5951c9fbb1SZi Shen Lim int epilogue_offset; 60e54bcde3SZi Shen Lim int *offset; 6180083428SJean-Philippe Brucker int exentry_idx; 62425e1ed7SLuc Van Oostenryck __le32 *image; 63f1c9eed7SDaniel Borkmann u32 stack_size; 64e54bcde3SZi Shen Lim }; 65e54bcde3SZi Shen Lim 66e54bcde3SZi Shen Lim static inline void emit(const u32 insn, struct jit_ctx *ctx) 67e54bcde3SZi Shen Lim { 68e54bcde3SZi Shen Lim if (ctx->image != NULL) 69e54bcde3SZi Shen Lim ctx->image[ctx->idx] = cpu_to_le32(insn); 70e54bcde3SZi Shen Lim 71e54bcde3SZi Shen Lim ctx->idx++; 72e54bcde3SZi Shen Lim } 73e54bcde3SZi Shen Lim 74e54bcde3SZi Shen Lim static inline void emit_a64_mov_i(const int is64, const int reg, 75e54bcde3SZi Shen Lim const s32 val, struct jit_ctx *ctx) 76e54bcde3SZi Shen Lim { 77e54bcde3SZi Shen Lim u16 hi = val >> 16; 78e54bcde3SZi Shen Lim u16 lo = val & 0xffff; 79e54bcde3SZi Shen Lim 80e54bcde3SZi Shen Lim if (hi & 0x8000) { 81e54bcde3SZi Shen Lim if (hi == 0xffff) { 82e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx); 83e54bcde3SZi Shen Lim } else { 84e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx); 856d2eea6fSDaniel Borkmann if (lo != 0xffff) 86e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, lo, 0), ctx); 87e54bcde3SZi Shen Lim } 88e54bcde3SZi Shen Lim } else { 89e54bcde3SZi Shen Lim emit(A64_MOVZ(is64, reg, lo, 0), ctx); 90e54bcde3SZi Shen Lim if (hi) 91e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, hi, 16), ctx); 92e54bcde3SZi Shen Lim } 93e54bcde3SZi Shen Lim } 94e54bcde3SZi Shen Lim 956d2eea6fSDaniel Borkmann static int i64_i16_blocks(const u64 val, bool inverse) 966d2eea6fSDaniel Borkmann { 976d2eea6fSDaniel Borkmann return (((val >> 0) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 986d2eea6fSDaniel Borkmann (((val >> 16) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 996d2eea6fSDaniel Borkmann (((val >> 32) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 1006d2eea6fSDaniel Borkmann (((val >> 48) & 0xffff) != (inverse ? 0xffff : 0x0000)); 1016d2eea6fSDaniel Borkmann } 1026d2eea6fSDaniel Borkmann 1036d2eea6fSDaniel Borkmann static inline void emit_a64_mov_i64(const int reg, const u64 val, 1046d2eea6fSDaniel Borkmann struct jit_ctx *ctx) 1056d2eea6fSDaniel Borkmann { 1066d2eea6fSDaniel Borkmann u64 nrm_tmp = val, rev_tmp = ~val; 1076d2eea6fSDaniel Borkmann bool inverse; 1086d2eea6fSDaniel Borkmann int shift; 1096d2eea6fSDaniel Borkmann 1106d2eea6fSDaniel Borkmann if (!(nrm_tmp >> 32)) 1116d2eea6fSDaniel Borkmann return emit_a64_mov_i(0, reg, (u32)val, ctx); 1126d2eea6fSDaniel Borkmann 1136d2eea6fSDaniel Borkmann inverse = i64_i16_blocks(nrm_tmp, true) < i64_i16_blocks(nrm_tmp, false); 1146d2eea6fSDaniel Borkmann shift = max(round_down((inverse ? (fls64(rev_tmp) - 1) : 1156d2eea6fSDaniel Borkmann (fls64(nrm_tmp) - 1)), 16), 0); 1166d2eea6fSDaniel Borkmann if (inverse) 1176d2eea6fSDaniel Borkmann emit(A64_MOVN(1, reg, (rev_tmp >> shift) & 0xffff, shift), ctx); 1186d2eea6fSDaniel Borkmann else 1196d2eea6fSDaniel Borkmann emit(A64_MOVZ(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx); 1206d2eea6fSDaniel Borkmann shift -= 16; 1216d2eea6fSDaniel Borkmann while (shift >= 0) { 1226d2eea6fSDaniel Borkmann if (((nrm_tmp >> shift) & 0xffff) != (inverse ? 0xffff : 0x0000)) 1236d2eea6fSDaniel Borkmann emit(A64_MOVK(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx); 1246d2eea6fSDaniel Borkmann shift -= 16; 1256d2eea6fSDaniel Borkmann } 1266d2eea6fSDaniel Borkmann } 1276d2eea6fSDaniel Borkmann 1286d2eea6fSDaniel Borkmann /* 129cc2b8ed1SArd Biesheuvel * Kernel addresses in the vmalloc space use at most 48 bits, and the 130cc2b8ed1SArd Biesheuvel * remaining bits are guaranteed to be 0x1. So we can compose the address 131cc2b8ed1SArd Biesheuvel * with a fixed length movn/movk/movk sequence. 1326d2eea6fSDaniel Borkmann */ 1336d2eea6fSDaniel Borkmann static inline void emit_addr_mov_i64(const int reg, const u64 val, 1346d2eea6fSDaniel Borkmann struct jit_ctx *ctx) 1356d2eea6fSDaniel Borkmann { 1366d2eea6fSDaniel Borkmann u64 tmp = val; 1376d2eea6fSDaniel Borkmann int shift = 0; 1386d2eea6fSDaniel Borkmann 139cc2b8ed1SArd Biesheuvel emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx); 140cc2b8ed1SArd Biesheuvel while (shift < 32) { 1416d2eea6fSDaniel Borkmann tmp >>= 16; 1426d2eea6fSDaniel Borkmann shift += 16; 1436d2eea6fSDaniel Borkmann emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx); 1446d2eea6fSDaniel Borkmann } 1456d2eea6fSDaniel Borkmann } 1466d2eea6fSDaniel Borkmann 14732f6865cSIlias Apalodimas static inline int bpf2a64_offset(int bpf_insn, int off, 148e54bcde3SZi Shen Lim const struct jit_ctx *ctx) 149e54bcde3SZi Shen Lim { 15032f6865cSIlias Apalodimas /* BPF JMP offset is relative to the next instruction */ 15132f6865cSIlias Apalodimas bpf_insn++; 15232f6865cSIlias Apalodimas /* 15332f6865cSIlias Apalodimas * Whereas arm64 branch instructions encode the offset 15432f6865cSIlias Apalodimas * from the branch itself, so we must subtract 1 from the 15532f6865cSIlias Apalodimas * instruction offset. 15632f6865cSIlias Apalodimas */ 15732f6865cSIlias Apalodimas return ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1); 158e54bcde3SZi Shen Lim } 159e54bcde3SZi Shen Lim 160b569c1c6SDaniel Borkmann static void jit_fill_hole(void *area, unsigned int size) 161b569c1c6SDaniel Borkmann { 162425e1ed7SLuc Van Oostenryck __le32 *ptr; 163b569c1c6SDaniel Borkmann /* We are guaranteed to have aligned memory. */ 164b569c1c6SDaniel Borkmann for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 165b569c1c6SDaniel Borkmann *ptr++ = cpu_to_le32(AARCH64_BREAK_FAULT); 166b569c1c6SDaniel Borkmann } 167b569c1c6SDaniel Borkmann 168e54bcde3SZi Shen Lim static inline int epilogue_offset(const struct jit_ctx *ctx) 169e54bcde3SZi Shen Lim { 17051c9fbb1SZi Shen Lim int to = ctx->epilogue_offset; 17151c9fbb1SZi Shen Lim int from = ctx->idx; 172e54bcde3SZi Shen Lim 173e54bcde3SZi Shen Lim return to - from; 174e54bcde3SZi Shen Lim } 175e54bcde3SZi Shen Lim 176fd868f14SLuke Nelson static bool is_addsub_imm(u32 imm) 177fd868f14SLuke Nelson { 178fd868f14SLuke Nelson /* Either imm12 or shifted imm12. */ 179fd868f14SLuke Nelson return !(imm & ~0xfff) || !(imm & ~0xfff000); 180fd868f14SLuke Nelson } 181fd868f14SLuke Nelson 182a2284d91SDaniel Borkmann /* Tail call offset to jump into */ 183fa76cfe6SMark Brown #if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) 184fa76cfe6SMark Brown #define PROLOGUE_OFFSET 8 185fa76cfe6SMark Brown #else 186a2284d91SDaniel Borkmann #define PROLOGUE_OFFSET 7 187fa76cfe6SMark Brown #endif 188ddb55992SZi Shen Lim 18956ea6a8bSDaniel Borkmann static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) 190e54bcde3SZi Shen Lim { 191f1c9eed7SDaniel Borkmann const struct bpf_prog *prog = ctx->prog; 192e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 193e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 194e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 195e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 196e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 197ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 198ddb55992SZi Shen Lim const int idx0 = ctx->idx; 199ddb55992SZi Shen Lim int cur_offset; 200e54bcde3SZi Shen Lim 201ec0738dbSYang Shi /* 202ec0738dbSYang Shi * BPF prog stack layout 203ec0738dbSYang Shi * 204ec0738dbSYang Shi * high 205ec0738dbSYang Shi * original A64_SP => 0:+-----+ BPF prologue 206ec0738dbSYang Shi * |FP/LR| 207ec0738dbSYang Shi * current A64_FP => -16:+-----+ 208ec0738dbSYang Shi * | ... | callee saved registers 2094c1cd4fdSYang Shi * BPF fp register => -64:+-----+ <= (BPF_FP) 210ec0738dbSYang Shi * | | 211ec0738dbSYang Shi * | ... | BPF prog stack 212ec0738dbSYang Shi * | | 213f1c9eed7SDaniel Borkmann * +-----+ <= (BPF_FP - prog->aux->stack_depth) 21409ece3d0SDaniel Borkmann * |RSVD | padding 215f1c9eed7SDaniel Borkmann * current A64_SP => +-----+ <= (BPF_FP - ctx->stack_size) 216ec0738dbSYang Shi * | | 217ec0738dbSYang Shi * | ... | Function call stack 218ec0738dbSYang Shi * | | 219ec0738dbSYang Shi * +-----+ 220ec0738dbSYang Shi * low 221ec0738dbSYang Shi * 222ec0738dbSYang Shi */ 223ec0738dbSYang Shi 224fa76cfe6SMark Brown /* BTI landing pad */ 225fa76cfe6SMark Brown if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) 226fa76cfe6SMark Brown emit(A64_BTI_C, ctx); 227fa76cfe6SMark Brown 228ec0738dbSYang Shi /* Save FP and LR registers to stay align with ARM64 AAPCS */ 229ec0738dbSYang Shi emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); 230ec0738dbSYang Shi emit(A64_MOV(1, A64_FP, A64_SP), ctx); 231ec0738dbSYang Shi 232ddb55992SZi Shen Lim /* Save callee-saved registers */ 233e54bcde3SZi Shen Lim emit(A64_PUSH(r6, r7, A64_SP), ctx); 234e54bcde3SZi Shen Lim emit(A64_PUSH(r8, r9, A64_SP), ctx); 235ddb55992SZi Shen Lim emit(A64_PUSH(fp, tcc, A64_SP), ctx); 236e54bcde3SZi Shen Lim 237ddb55992SZi Shen Lim /* Set up BPF prog stack base register */ 238e54bcde3SZi Shen Lim emit(A64_MOV(1, fp, A64_SP), ctx); 239e54bcde3SZi Shen Lim 24056ea6a8bSDaniel Borkmann if (!ebpf_from_cbpf) { 241ddb55992SZi Shen Lim /* Initialize tail_call_cnt */ 242ddb55992SZi Shen Lim emit(A64_MOVZ(1, tcc, 0, 0), ctx); 243ddb55992SZi Shen Lim 244ddb55992SZi Shen Lim cur_offset = ctx->idx - idx0; 245ddb55992SZi Shen Lim if (cur_offset != PROLOGUE_OFFSET) { 246ddb55992SZi Shen Lim pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n", 247ddb55992SZi Shen Lim cur_offset, PROLOGUE_OFFSET); 248ddb55992SZi Shen Lim return -1; 249ddb55992SZi Shen Lim } 250fa76cfe6SMark Brown 251fa76cfe6SMark Brown /* BTI landing pad for the tail call, done with a BR */ 252fa76cfe6SMark Brown if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) 253fa76cfe6SMark Brown emit(A64_BTI_J, ctx); 25456ea6a8bSDaniel Borkmann } 255a2284d91SDaniel Borkmann 2563f287098STiezhu Yang /* Stack must be multiples of 16B */ 2573f287098STiezhu Yang ctx->stack_size = round_up(prog->aux->stack_depth, 16); 258a2284d91SDaniel Borkmann 259a2284d91SDaniel Borkmann /* Set up function call stack */ 260a2284d91SDaniel Borkmann emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 261ddb55992SZi Shen Lim return 0; 262ddb55992SZi Shen Lim } 263ddb55992SZi Shen Lim 264ddb55992SZi Shen Lim static int out_offset = -1; /* initialized on the first pass of build_body() */ 265ddb55992SZi Shen Lim static int emit_bpf_tail_call(struct jit_ctx *ctx) 266ddb55992SZi Shen Lim { 267ddb55992SZi Shen Lim /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ 268ddb55992SZi Shen Lim const u8 r2 = bpf2a64[BPF_REG_2]; 269ddb55992SZi Shen Lim const u8 r3 = bpf2a64[BPF_REG_3]; 270ddb55992SZi Shen Lim 271ddb55992SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 272ddb55992SZi Shen Lim const u8 prg = bpf2a64[TMP_REG_2]; 273ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 274ddb55992SZi Shen Lim const int idx0 = ctx->idx; 275ddb55992SZi Shen Lim #define cur_offset (ctx->idx - idx0) 276ddb55992SZi Shen Lim #define jmp_offset (out_offset - (cur_offset)) 277ddb55992SZi Shen Lim size_t off; 278ddb55992SZi Shen Lim 279ddb55992SZi Shen Lim /* if (index >= array->map.max_entries) 280ddb55992SZi Shen Lim * goto out; 281ddb55992SZi Shen Lim */ 282ddb55992SZi Shen Lim off = offsetof(struct bpf_array, map.max_entries); 283ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 284ddb55992SZi Shen Lim emit(A64_LDR32(tmp, r2, tmp), ctx); 28516338a9bSDaniel Borkmann emit(A64_MOV(0, r3, r3), ctx); 286ddb55992SZi Shen Lim emit(A64_CMP(0, r3, tmp), ctx); 28716338a9bSDaniel Borkmann emit(A64_B_(A64_COND_CS, jmp_offset), ctx); 288ddb55992SZi Shen Lim 289ddb55992SZi Shen Lim /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) 290ddb55992SZi Shen Lim * goto out; 291ddb55992SZi Shen Lim * tail_call_cnt++; 292ddb55992SZi Shen Lim */ 293ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx); 294ddb55992SZi Shen Lim emit(A64_CMP(1, tcc, tmp), ctx); 29516338a9bSDaniel Borkmann emit(A64_B_(A64_COND_HI, jmp_offset), ctx); 296ddb55992SZi Shen Lim emit(A64_ADD_I(1, tcc, tcc, 1), ctx); 297ddb55992SZi Shen Lim 298ddb55992SZi Shen Lim /* prog = array->ptrs[index]; 299ddb55992SZi Shen Lim * if (prog == NULL) 300ddb55992SZi Shen Lim * goto out; 301ddb55992SZi Shen Lim */ 302ddb55992SZi Shen Lim off = offsetof(struct bpf_array, ptrs); 303ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 304d8b54110SDaniel Borkmann emit(A64_ADD(1, tmp, r2, tmp), ctx); 305d8b54110SDaniel Borkmann emit(A64_LSL(1, prg, r3, 3), ctx); 306d8b54110SDaniel Borkmann emit(A64_LDR64(prg, tmp, prg), ctx); 307ddb55992SZi Shen Lim emit(A64_CBZ(1, prg, jmp_offset), ctx); 308ddb55992SZi Shen Lim 309a2284d91SDaniel Borkmann /* goto *(prog->bpf_func + prologue_offset); */ 310ddb55992SZi Shen Lim off = offsetof(struct bpf_prog, bpf_func); 311ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 312ddb55992SZi Shen Lim emit(A64_LDR64(tmp, prg, tmp), ctx); 313ddb55992SZi Shen Lim emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx); 314a2284d91SDaniel Borkmann emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 315ddb55992SZi Shen Lim emit(A64_BR(tmp), ctx); 316ddb55992SZi Shen Lim 317ddb55992SZi Shen Lim /* out: */ 318ddb55992SZi Shen Lim if (out_offset == -1) 319ddb55992SZi Shen Lim out_offset = cur_offset; 320ddb55992SZi Shen Lim if (cur_offset != out_offset) { 321ddb55992SZi Shen Lim pr_err_once("tail_call out_offset = %d, expected %d!\n", 322ddb55992SZi Shen Lim cur_offset, out_offset); 323ddb55992SZi Shen Lim return -1; 324ddb55992SZi Shen Lim } 325ddb55992SZi Shen Lim return 0; 326ddb55992SZi Shen Lim #undef cur_offset 327ddb55992SZi Shen Lim #undef jmp_offset 328e54bcde3SZi Shen Lim } 329e54bcde3SZi Shen Lim 330e54bcde3SZi Shen Lim static void build_epilogue(struct jit_ctx *ctx) 331e54bcde3SZi Shen Lim { 332e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 333e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 334e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 335e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 336e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 337e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 338e54bcde3SZi Shen Lim 339e54bcde3SZi Shen Lim /* We're done with BPF stack */ 340f1c9eed7SDaniel Borkmann emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 341e54bcde3SZi Shen Lim 342ec0738dbSYang Shi /* Restore fs (x25) and x26 */ 343ec0738dbSYang Shi emit(A64_POP(fp, A64_R(26), A64_SP), ctx); 344ec0738dbSYang Shi 345e54bcde3SZi Shen Lim /* Restore callee-saved register */ 346e54bcde3SZi Shen Lim emit(A64_POP(r8, r9, A64_SP), ctx); 347e54bcde3SZi Shen Lim emit(A64_POP(r6, r7, A64_SP), ctx); 348e54bcde3SZi Shen Lim 349ec0738dbSYang Shi /* Restore FP/LR registers */ 350ec0738dbSYang Shi emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); 351e54bcde3SZi Shen Lim 352e54bcde3SZi Shen Lim /* Set return value */ 353e54bcde3SZi Shen Lim emit(A64_MOV(1, A64_R(0), r0), ctx); 354e54bcde3SZi Shen Lim 355e54bcde3SZi Shen Lim emit(A64_RET(A64_LR), ctx); 356e54bcde3SZi Shen Lim } 357e54bcde3SZi Shen Lim 35880083428SJean-Philippe Brucker #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 35980083428SJean-Philippe Brucker #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 36080083428SJean-Philippe Brucker 36180083428SJean-Philippe Brucker int arm64_bpf_fixup_exception(const struct exception_table_entry *ex, 36280083428SJean-Philippe Brucker struct pt_regs *regs) 36380083428SJean-Philippe Brucker { 36480083428SJean-Philippe Brucker off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 36580083428SJean-Philippe Brucker int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 36680083428SJean-Philippe Brucker 36780083428SJean-Philippe Brucker regs->regs[dst_reg] = 0; 36880083428SJean-Philippe Brucker regs->pc = (unsigned long)&ex->fixup - offset; 36980083428SJean-Philippe Brucker return 1; 37080083428SJean-Philippe Brucker } 37180083428SJean-Philippe Brucker 37280083428SJean-Philippe Brucker /* For accesses to BTF pointers, add an entry to the exception table */ 37380083428SJean-Philippe Brucker static int add_exception_handler(const struct bpf_insn *insn, 37480083428SJean-Philippe Brucker struct jit_ctx *ctx, 37580083428SJean-Philippe Brucker int dst_reg) 37680083428SJean-Philippe Brucker { 37780083428SJean-Philippe Brucker off_t offset; 37880083428SJean-Philippe Brucker unsigned long pc; 37980083428SJean-Philippe Brucker struct exception_table_entry *ex; 38080083428SJean-Philippe Brucker 38180083428SJean-Philippe Brucker if (!ctx->image) 38280083428SJean-Philippe Brucker /* First pass */ 38380083428SJean-Philippe Brucker return 0; 38480083428SJean-Philippe Brucker 38580083428SJean-Philippe Brucker if (BPF_MODE(insn->code) != BPF_PROBE_MEM) 38680083428SJean-Philippe Brucker return 0; 38780083428SJean-Philippe Brucker 38880083428SJean-Philippe Brucker if (!ctx->prog->aux->extable || 38980083428SJean-Philippe Brucker WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries)) 39080083428SJean-Philippe Brucker return -EINVAL; 39180083428SJean-Philippe Brucker 39280083428SJean-Philippe Brucker ex = &ctx->prog->aux->extable[ctx->exentry_idx]; 39380083428SJean-Philippe Brucker pc = (unsigned long)&ctx->image[ctx->idx - 1]; 39480083428SJean-Philippe Brucker 39580083428SJean-Philippe Brucker offset = pc - (long)&ex->insn; 39680083428SJean-Philippe Brucker if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 39780083428SJean-Philippe Brucker return -ERANGE; 39880083428SJean-Philippe Brucker ex->insn = offset; 39980083428SJean-Philippe Brucker 40080083428SJean-Philippe Brucker /* 40180083428SJean-Philippe Brucker * Since the extable follows the program, the fixup offset is always 40280083428SJean-Philippe Brucker * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 40380083428SJean-Philippe Brucker * to keep things simple, and put the destination register in the upper 40480083428SJean-Philippe Brucker * bits. We don't need to worry about buildtime or runtime sort 40580083428SJean-Philippe Brucker * modifying the upper bits because the table is already sorted, and 40680083428SJean-Philippe Brucker * isn't part of the main exception table. 40780083428SJean-Philippe Brucker */ 40880083428SJean-Philippe Brucker offset = (long)&ex->fixup - (pc + AARCH64_INSN_SIZE); 40980083428SJean-Philippe Brucker if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 41080083428SJean-Philippe Brucker return -ERANGE; 41180083428SJean-Philippe Brucker 41280083428SJean-Philippe Brucker ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | 41380083428SJean-Philippe Brucker FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 41480083428SJean-Philippe Brucker 41580083428SJean-Philippe Brucker ctx->exentry_idx++; 41680083428SJean-Philippe Brucker return 0; 41780083428SJean-Philippe Brucker } 41880083428SJean-Philippe Brucker 41930d3d94cSZi Shen Lim /* JITs an eBPF instruction. 42030d3d94cSZi Shen Lim * Returns: 42130d3d94cSZi Shen Lim * 0 - successfully JITed an 8-byte eBPF instruction. 42230d3d94cSZi Shen Lim * >0 - successfully JITed a 16-byte eBPF instruction. 42330d3d94cSZi Shen Lim * <0 - failed to JIT. 42430d3d94cSZi Shen Lim */ 4258c11ea5cSDaniel Borkmann static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 4268c11ea5cSDaniel Borkmann bool extra_pass) 427e54bcde3SZi Shen Lim { 428e54bcde3SZi Shen Lim const u8 code = insn->code; 429e54bcde3SZi Shen Lim const u8 dst = bpf2a64[insn->dst_reg]; 430e54bcde3SZi Shen Lim const u8 src = bpf2a64[insn->src_reg]; 431e54bcde3SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 432e54bcde3SZi Shen Lim const u8 tmp2 = bpf2a64[TMP_REG_2]; 4337005cadeSDaniel Borkmann const u8 tmp3 = bpf2a64[TMP_REG_3]; 434e54bcde3SZi Shen Lim const s16 off = insn->off; 435e54bcde3SZi Shen Lim const s32 imm = insn->imm; 436e54bcde3SZi Shen Lim const int i = insn - ctx->prog->insnsi; 437654b65a0SJiong Wang const bool is64 = BPF_CLASS(code) == BPF_ALU64 || 438654b65a0SJiong Wang BPF_CLASS(code) == BPF_JMP; 43985f68fe8SDaniel Borkmann const bool isdw = BPF_SIZE(code) == BPF_DW; 44034b8ab09SDaniel Borkmann u8 jmp_cond, reg; 441e54bcde3SZi Shen Lim s32 jmp_offset; 442fd49591cSLuke Nelson u32 a64_insn; 44380083428SJean-Philippe Brucker int ret; 444e54bcde3SZi Shen Lim 445251599e1SZi Shen Lim #define check_imm(bits, imm) do { \ 446251599e1SZi Shen Lim if ((((imm) > 0) && ((imm) >> (bits))) || \ 447251599e1SZi Shen Lim (((imm) < 0) && (~(imm) >> (bits)))) { \ 448251599e1SZi Shen Lim pr_info("[%2d] imm=%d(0x%x) out of range\n", \ 449251599e1SZi Shen Lim i, imm, imm); \ 450251599e1SZi Shen Lim return -EINVAL; \ 451251599e1SZi Shen Lim } \ 452251599e1SZi Shen Lim } while (0) 453251599e1SZi Shen Lim #define check_imm19(imm) check_imm(19, imm) 454251599e1SZi Shen Lim #define check_imm26(imm) check_imm(26, imm) 455251599e1SZi Shen Lim 456e54bcde3SZi Shen Lim switch (code) { 457e54bcde3SZi Shen Lim /* dst = src */ 458e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_X: 459e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_X: 460e54bcde3SZi Shen Lim emit(A64_MOV(is64, dst, src), ctx); 461e54bcde3SZi Shen Lim break; 462e54bcde3SZi Shen Lim /* dst = dst OP src */ 463e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_X: 464e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_X: 465e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, src), ctx); 466e54bcde3SZi Shen Lim break; 467e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_X: 468e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_X: 469e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, src), ctx); 470e54bcde3SZi Shen Lim break; 471e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_X: 472e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_X: 473e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, src), ctx); 474e54bcde3SZi Shen Lim break; 475e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_X: 476e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_X: 477e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, src), ctx); 478e54bcde3SZi Shen Lim break; 479e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_X: 480e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_X: 481e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, src), ctx); 482e54bcde3SZi Shen Lim break; 483e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_X: 484e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_X: 485e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, src), ctx); 486e54bcde3SZi Shen Lim break; 487e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_X: 488e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_X: 489e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, src), ctx); 490e54bcde3SZi Shen Lim break; 491119220d8STiezhu Yang case BPF_ALU | BPF_MOD | BPF_X: 492119220d8STiezhu Yang case BPF_ALU64 | BPF_MOD | BPF_X: 493e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, src), ctx); 494504792e0SJerin Jacob emit(A64_MSUB(is64, dst, dst, tmp, src), ctx); 495e54bcde3SZi Shen Lim break; 496d65a634aSZi Shen Lim case BPF_ALU | BPF_LSH | BPF_X: 497d65a634aSZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_X: 498d65a634aSZi Shen Lim emit(A64_LSLV(is64, dst, dst, src), ctx); 499d65a634aSZi Shen Lim break; 500d65a634aSZi Shen Lim case BPF_ALU | BPF_RSH | BPF_X: 501d65a634aSZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_X: 502d65a634aSZi Shen Lim emit(A64_LSRV(is64, dst, dst, src), ctx); 503d65a634aSZi Shen Lim break; 504d65a634aSZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_X: 505d65a634aSZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_X: 506d65a634aSZi Shen Lim emit(A64_ASRV(is64, dst, dst, src), ctx); 507d65a634aSZi Shen Lim break; 508e54bcde3SZi Shen Lim /* dst = -dst */ 509e54bcde3SZi Shen Lim case BPF_ALU | BPF_NEG: 510e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_NEG: 511e54bcde3SZi Shen Lim emit(A64_NEG(is64, dst, dst), ctx); 512e54bcde3SZi Shen Lim break; 513e54bcde3SZi Shen Lim /* dst = BSWAP##imm(dst) */ 514e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_LE: 515e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_BE: 516e54bcde3SZi Shen Lim #ifdef CONFIG_CPU_BIG_ENDIAN 517e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_BE) 518d63903bbSXi Wang goto emit_bswap_uxt; 519e54bcde3SZi Shen Lim #else /* !CONFIG_CPU_BIG_ENDIAN */ 520e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_LE) 521d63903bbSXi Wang goto emit_bswap_uxt; 522e54bcde3SZi Shen Lim #endif 523e54bcde3SZi Shen Lim switch (imm) { 524e54bcde3SZi Shen Lim case 16: 525e54bcde3SZi Shen Lim emit(A64_REV16(is64, dst, dst), ctx); 526d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 527d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 528e54bcde3SZi Shen Lim break; 529e54bcde3SZi Shen Lim case 32: 530e54bcde3SZi Shen Lim emit(A64_REV32(is64, dst, dst), ctx); 531d63903bbSXi Wang /* upper 32 bits already cleared */ 532e54bcde3SZi Shen Lim break; 533e54bcde3SZi Shen Lim case 64: 534e54bcde3SZi Shen Lim emit(A64_REV64(dst, dst), ctx); 535e54bcde3SZi Shen Lim break; 536e54bcde3SZi Shen Lim } 537e54bcde3SZi Shen Lim break; 538d63903bbSXi Wang emit_bswap_uxt: 539d63903bbSXi Wang switch (imm) { 540d63903bbSXi Wang case 16: 541d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 542d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 543d63903bbSXi Wang break; 544d63903bbSXi Wang case 32: 545d63903bbSXi Wang /* zero-extend 32 bits into 64 bits */ 546d63903bbSXi Wang emit(A64_UXTW(is64, dst, dst), ctx); 547d63903bbSXi Wang break; 548d63903bbSXi Wang case 64: 549d63903bbSXi Wang /* nop */ 550d63903bbSXi Wang break; 551d63903bbSXi Wang } 552d63903bbSXi Wang break; 553e54bcde3SZi Shen Lim /* dst = imm */ 554e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_K: 555e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_K: 556e54bcde3SZi Shen Lim emit_a64_mov_i(is64, dst, imm, ctx); 557e54bcde3SZi Shen Lim break; 558e54bcde3SZi Shen Lim /* dst = dst OP imm */ 559e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_K: 560e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_K: 561fd868f14SLuke Nelson if (is_addsub_imm(imm)) { 562fd868f14SLuke Nelson emit(A64_ADD_I(is64, dst, dst, imm), ctx); 563fd868f14SLuke Nelson } else if (is_addsub_imm(-imm)) { 564fd868f14SLuke Nelson emit(A64_SUB_I(is64, dst, dst, -imm), ctx); 565fd868f14SLuke Nelson } else { 566e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 567e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, tmp), ctx); 568fd868f14SLuke Nelson } 569e54bcde3SZi Shen Lim break; 570e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_K: 571e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_K: 572fd868f14SLuke Nelson if (is_addsub_imm(imm)) { 573fd868f14SLuke Nelson emit(A64_SUB_I(is64, dst, dst, imm), ctx); 574fd868f14SLuke Nelson } else if (is_addsub_imm(-imm)) { 575fd868f14SLuke Nelson emit(A64_ADD_I(is64, dst, dst, -imm), ctx); 576fd868f14SLuke Nelson } else { 577e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 578e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 579fd868f14SLuke Nelson } 580e54bcde3SZi Shen Lim break; 581e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_K: 582e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_K: 583fd49591cSLuke Nelson a64_insn = A64_AND_I(is64, dst, dst, imm); 584fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 585fd49591cSLuke Nelson emit(a64_insn, ctx); 586fd49591cSLuke Nelson } else { 587e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 588e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, tmp), ctx); 589fd49591cSLuke Nelson } 590e54bcde3SZi Shen Lim break; 591e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_K: 592e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_K: 593fd49591cSLuke Nelson a64_insn = A64_ORR_I(is64, dst, dst, imm); 594fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 595fd49591cSLuke Nelson emit(a64_insn, ctx); 596fd49591cSLuke Nelson } else { 597e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 598e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, tmp), ctx); 599fd49591cSLuke Nelson } 600e54bcde3SZi Shen Lim break; 601e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_K: 602e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_K: 603fd49591cSLuke Nelson a64_insn = A64_EOR_I(is64, dst, dst, imm); 604fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 605fd49591cSLuke Nelson emit(a64_insn, ctx); 606fd49591cSLuke Nelson } else { 607e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 608e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, tmp), ctx); 609fd49591cSLuke Nelson } 610e54bcde3SZi Shen Lim break; 611e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_K: 612e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_K: 613e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 614e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, tmp), ctx); 615e54bcde3SZi Shen Lim break; 616e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_K: 617e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_K: 618e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 619e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, tmp), ctx); 620e54bcde3SZi Shen Lim break; 621e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOD | BPF_K: 622e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOD | BPF_K: 623e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp2, imm, ctx); 624e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, tmp2), ctx); 625504792e0SJerin Jacob emit(A64_MSUB(is64, dst, dst, tmp, tmp2), ctx); 626e54bcde3SZi Shen Lim break; 627e54bcde3SZi Shen Lim case BPF_ALU | BPF_LSH | BPF_K: 628e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_K: 629e54bcde3SZi Shen Lim emit(A64_LSL(is64, dst, dst, imm), ctx); 630e54bcde3SZi Shen Lim break; 631e54bcde3SZi Shen Lim case BPF_ALU | BPF_RSH | BPF_K: 632e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_K: 633e54bcde3SZi Shen Lim emit(A64_LSR(is64, dst, dst, imm), ctx); 634e54bcde3SZi Shen Lim break; 635e54bcde3SZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_K: 636e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_K: 637e54bcde3SZi Shen Lim emit(A64_ASR(is64, dst, dst, imm), ctx); 638e54bcde3SZi Shen Lim break; 639e54bcde3SZi Shen Lim 640e54bcde3SZi Shen Lim /* JUMP off */ 641e54bcde3SZi Shen Lim case BPF_JMP | BPF_JA: 64232f6865cSIlias Apalodimas jmp_offset = bpf2a64_offset(i, off, ctx); 643e54bcde3SZi Shen Lim check_imm26(jmp_offset); 644e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 645e54bcde3SZi Shen Lim break; 646e54bcde3SZi Shen Lim /* IF (dst COND src) JUMP off */ 647e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_X: 648e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_X: 649c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_X: 650e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_X: 651c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_X: 652e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_X: 653e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_X: 654c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_X: 655e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_X: 656c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_X: 657654b65a0SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_X: 658654b65a0SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_X: 659654b65a0SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_X: 660654b65a0SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_X: 661654b65a0SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_X: 662654b65a0SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_X: 663654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_X: 664654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_X: 665654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_X: 666654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_X: 667654b65a0SJiong Wang emit(A64_CMP(is64, dst, src), ctx); 668e54bcde3SZi Shen Lim emit_cond_jmp: 66932f6865cSIlias Apalodimas jmp_offset = bpf2a64_offset(i, off, ctx); 670e54bcde3SZi Shen Lim check_imm19(jmp_offset); 671e54bcde3SZi Shen Lim switch (BPF_OP(code)) { 672e54bcde3SZi Shen Lim case BPF_JEQ: 673e54bcde3SZi Shen Lim jmp_cond = A64_COND_EQ; 674e54bcde3SZi Shen Lim break; 675e54bcde3SZi Shen Lim case BPF_JGT: 676e54bcde3SZi Shen Lim jmp_cond = A64_COND_HI; 677e54bcde3SZi Shen Lim break; 678c362b2f3SDaniel Borkmann case BPF_JLT: 679c362b2f3SDaniel Borkmann jmp_cond = A64_COND_CC; 680c362b2f3SDaniel Borkmann break; 681e54bcde3SZi Shen Lim case BPF_JGE: 682e54bcde3SZi Shen Lim jmp_cond = A64_COND_CS; 683e54bcde3SZi Shen Lim break; 684c362b2f3SDaniel Borkmann case BPF_JLE: 685c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LS; 686c362b2f3SDaniel Borkmann break; 68798397fc5SZi Shen Lim case BPF_JSET: 688e54bcde3SZi Shen Lim case BPF_JNE: 689e54bcde3SZi Shen Lim jmp_cond = A64_COND_NE; 690e54bcde3SZi Shen Lim break; 691e54bcde3SZi Shen Lim case BPF_JSGT: 692e54bcde3SZi Shen Lim jmp_cond = A64_COND_GT; 693e54bcde3SZi Shen Lim break; 694c362b2f3SDaniel Borkmann case BPF_JSLT: 695c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LT; 696c362b2f3SDaniel Borkmann break; 697e54bcde3SZi Shen Lim case BPF_JSGE: 698e54bcde3SZi Shen Lim jmp_cond = A64_COND_GE; 699e54bcde3SZi Shen Lim break; 700c362b2f3SDaniel Borkmann case BPF_JSLE: 701c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LE; 702c362b2f3SDaniel Borkmann break; 703e54bcde3SZi Shen Lim default: 704e54bcde3SZi Shen Lim return -EFAULT; 705e54bcde3SZi Shen Lim } 706e54bcde3SZi Shen Lim emit(A64_B_(jmp_cond, jmp_offset), ctx); 707e54bcde3SZi Shen Lim break; 708e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_X: 709654b65a0SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_X: 710654b65a0SJiong Wang emit(A64_TST(is64, dst, src), ctx); 711e54bcde3SZi Shen Lim goto emit_cond_jmp; 712e54bcde3SZi Shen Lim /* IF (dst COND imm) JUMP off */ 713e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_K: 714e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_K: 715c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_K: 716e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_K: 717c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_K: 718e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_K: 719e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_K: 720c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_K: 721e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_K: 722c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_K: 723654b65a0SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_K: 724654b65a0SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_K: 725654b65a0SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_K: 726654b65a0SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_K: 727654b65a0SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_K: 728654b65a0SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_K: 729654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_K: 730654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_K: 731654b65a0SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_K: 732654b65a0SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_K: 733fd868f14SLuke Nelson if (is_addsub_imm(imm)) { 734fd868f14SLuke Nelson emit(A64_CMP_I(is64, dst, imm), ctx); 735fd868f14SLuke Nelson } else if (is_addsub_imm(-imm)) { 736fd868f14SLuke Nelson emit(A64_CMN_I(is64, dst, -imm), ctx); 737fd868f14SLuke Nelson } else { 738654b65a0SJiong Wang emit_a64_mov_i(is64, tmp, imm, ctx); 739654b65a0SJiong Wang emit(A64_CMP(is64, dst, tmp), ctx); 740fd868f14SLuke Nelson } 741e54bcde3SZi Shen Lim goto emit_cond_jmp; 742e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_K: 743654b65a0SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_K: 744fd49591cSLuke Nelson a64_insn = A64_TST_I(is64, dst, imm); 745fd49591cSLuke Nelson if (a64_insn != AARCH64_BREAK_FAULT) { 746fd49591cSLuke Nelson emit(a64_insn, ctx); 747fd49591cSLuke Nelson } else { 748654b65a0SJiong Wang emit_a64_mov_i(is64, tmp, imm, ctx); 749654b65a0SJiong Wang emit(A64_TST(is64, dst, tmp), ctx); 750fd49591cSLuke Nelson } 751e54bcde3SZi Shen Lim goto emit_cond_jmp; 752e54bcde3SZi Shen Lim /* function call */ 753e54bcde3SZi Shen Lim case BPF_JMP | BPF_CALL: 754e54bcde3SZi Shen Lim { 755e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 7568c11ea5cSDaniel Borkmann bool func_addr_fixed; 7578c11ea5cSDaniel Borkmann u64 func_addr; 758e54bcde3SZi Shen Lim 7598c11ea5cSDaniel Borkmann ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 7608c11ea5cSDaniel Borkmann &func_addr, &func_addr_fixed); 7618c11ea5cSDaniel Borkmann if (ret < 0) 7628c11ea5cSDaniel Borkmann return ret; 7638c11ea5cSDaniel Borkmann emit_addr_mov_i64(tmp, func_addr, ctx); 764e54bcde3SZi Shen Lim emit(A64_BLR(tmp), ctx); 765e54bcde3SZi Shen Lim emit(A64_MOV(1, r0, A64_R(0)), ctx); 766e54bcde3SZi Shen Lim break; 767e54bcde3SZi Shen Lim } 768ddb55992SZi Shen Lim /* tail call */ 76971189fa9SAlexei Starovoitov case BPF_JMP | BPF_TAIL_CALL: 770ddb55992SZi Shen Lim if (emit_bpf_tail_call(ctx)) 771ddb55992SZi Shen Lim return -EFAULT; 772ddb55992SZi Shen Lim break; 773e54bcde3SZi Shen Lim /* function return */ 774e54bcde3SZi Shen Lim case BPF_JMP | BPF_EXIT: 77551c9fbb1SZi Shen Lim /* Optimization: when last instruction is EXIT, 77651c9fbb1SZi Shen Lim simply fallthrough to epilogue. */ 777e54bcde3SZi Shen Lim if (i == ctx->prog->len - 1) 778e54bcde3SZi Shen Lim break; 779e54bcde3SZi Shen Lim jmp_offset = epilogue_offset(ctx); 780e54bcde3SZi Shen Lim check_imm26(jmp_offset); 781e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 782e54bcde3SZi Shen Lim break; 783e54bcde3SZi Shen Lim 78430d3d94cSZi Shen Lim /* dst = imm64 */ 78530d3d94cSZi Shen Lim case BPF_LD | BPF_IMM | BPF_DW: 78630d3d94cSZi Shen Lim { 78730d3d94cSZi Shen Lim const struct bpf_insn insn1 = insn[1]; 78830d3d94cSZi Shen Lim u64 imm64; 78930d3d94cSZi Shen Lim 7901e4df6b7SXi Wang imm64 = (u64)insn1.imm << 32 | (u32)imm; 79130d3d94cSZi Shen Lim emit_a64_mov_i64(dst, imm64, ctx); 79230d3d94cSZi Shen Lim 79330d3d94cSZi Shen Lim return 1; 79430d3d94cSZi Shen Lim } 79530d3d94cSZi Shen Lim 796e54bcde3SZi Shen Lim /* LDX: dst = *(size *)(src + off) */ 797e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_W: 798e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_H: 799e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_B: 800e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_DW: 80180083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 80280083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_W: 80380083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_H: 80480083428SJean-Philippe Brucker case BPF_LDX | BPF_PROBE_MEM | BPF_B: 805e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 806e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 807e54bcde3SZi Shen Lim case BPF_W: 808e54bcde3SZi Shen Lim emit(A64_LDR32(dst, src, tmp), ctx); 809e54bcde3SZi Shen Lim break; 810e54bcde3SZi Shen Lim case BPF_H: 811e54bcde3SZi Shen Lim emit(A64_LDRH(dst, src, tmp), ctx); 812e54bcde3SZi Shen Lim break; 813e54bcde3SZi Shen Lim case BPF_B: 814e54bcde3SZi Shen Lim emit(A64_LDRB(dst, src, tmp), ctx); 815e54bcde3SZi Shen Lim break; 816e54bcde3SZi Shen Lim case BPF_DW: 817e54bcde3SZi Shen Lim emit(A64_LDR64(dst, src, tmp), ctx); 818e54bcde3SZi Shen Lim break; 819e54bcde3SZi Shen Lim } 82080083428SJean-Philippe Brucker 82180083428SJean-Philippe Brucker ret = add_exception_handler(insn, ctx, dst); 82280083428SJean-Philippe Brucker if (ret) 82380083428SJean-Philippe Brucker return ret; 824e54bcde3SZi Shen Lim break; 825e54bcde3SZi Shen Lim 826f5e81d11SDaniel Borkmann /* speculation barrier */ 827f5e81d11SDaniel Borkmann case BPF_ST | BPF_NOSPEC: 828f5e81d11SDaniel Borkmann /* 829f5e81d11SDaniel Borkmann * Nothing required here. 830f5e81d11SDaniel Borkmann * 831f5e81d11SDaniel Borkmann * In case of arm64, we rely on the firmware mitigation of 832f5e81d11SDaniel Borkmann * Speculative Store Bypass as controlled via the ssbd kernel 833f5e81d11SDaniel Borkmann * parameter. Whenever the mitigation is enabled, it works 834f5e81d11SDaniel Borkmann * for all of the kernel code with no need to provide any 835f5e81d11SDaniel Borkmann * additional instructions. 836f5e81d11SDaniel Borkmann */ 837f5e81d11SDaniel Borkmann break; 838f5e81d11SDaniel Borkmann 839e54bcde3SZi Shen Lim /* ST: *(size *)(dst + off) = imm */ 840e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_W: 841e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_H: 842e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_B: 843e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_DW: 844df849ba3SYang Shi /* Load imm to a register then store it */ 845df849ba3SYang Shi emit_a64_mov_i(1, tmp2, off, ctx); 846df849ba3SYang Shi emit_a64_mov_i(1, tmp, imm, ctx); 847df849ba3SYang Shi switch (BPF_SIZE(code)) { 848df849ba3SYang Shi case BPF_W: 849df849ba3SYang Shi emit(A64_STR32(tmp, dst, tmp2), ctx); 850df849ba3SYang Shi break; 851df849ba3SYang Shi case BPF_H: 852df849ba3SYang Shi emit(A64_STRH(tmp, dst, tmp2), ctx); 853df849ba3SYang Shi break; 854df849ba3SYang Shi case BPF_B: 855df849ba3SYang Shi emit(A64_STRB(tmp, dst, tmp2), ctx); 856df849ba3SYang Shi break; 857df849ba3SYang Shi case BPF_DW: 858df849ba3SYang Shi emit(A64_STR64(tmp, dst, tmp2), ctx); 859df849ba3SYang Shi break; 860df849ba3SYang Shi } 861df849ba3SYang Shi break; 862e54bcde3SZi Shen Lim 863e54bcde3SZi Shen Lim /* STX: *(size *)(dst + off) = src */ 864e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_W: 865e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_H: 866e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_B: 867e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_DW: 868e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 869e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 870e54bcde3SZi Shen Lim case BPF_W: 871e54bcde3SZi Shen Lim emit(A64_STR32(src, dst, tmp), ctx); 872e54bcde3SZi Shen Lim break; 873e54bcde3SZi Shen Lim case BPF_H: 874e54bcde3SZi Shen Lim emit(A64_STRH(src, dst, tmp), ctx); 875e54bcde3SZi Shen Lim break; 876e54bcde3SZi Shen Lim case BPF_B: 877e54bcde3SZi Shen Lim emit(A64_STRB(src, dst, tmp), ctx); 878e54bcde3SZi Shen Lim break; 879e54bcde3SZi Shen Lim case BPF_DW: 880e54bcde3SZi Shen Lim emit(A64_STR64(src, dst, tmp), ctx); 881e54bcde3SZi Shen Lim break; 882e54bcde3SZi Shen Lim } 883e54bcde3SZi Shen Lim break; 88434b8ab09SDaniel Borkmann 88591c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_W: 88691c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_DW: 88791c960b0SBrendan Jackman if (insn->imm != BPF_ADD) { 88891c960b0SBrendan Jackman pr_err_once("unknown atomic op code %02x\n", insn->imm); 88991c960b0SBrendan Jackman return -EINVAL; 89091c960b0SBrendan Jackman } 89191c960b0SBrendan Jackman 89291c960b0SBrendan Jackman /* STX XADD: lock *(u32 *)(dst + off) += src 89391c960b0SBrendan Jackman * and 89491c960b0SBrendan Jackman * STX XADD: lock *(u64 *)(dst + off) += src 89591c960b0SBrendan Jackman */ 89691c960b0SBrendan Jackman 89734b8ab09SDaniel Borkmann if (!off) { 89834b8ab09SDaniel Borkmann reg = dst; 89934b8ab09SDaniel Borkmann } else { 90085f68fe8SDaniel Borkmann emit_a64_mov_i(1, tmp, off, ctx); 90185f68fe8SDaniel Borkmann emit(A64_ADD(1, tmp, tmp, dst), ctx); 90234b8ab09SDaniel Borkmann reg = tmp; 90334b8ab09SDaniel Borkmann } 90434b8ab09SDaniel Borkmann if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS)) { 90534b8ab09SDaniel Borkmann emit(A64_STADD(isdw, reg, src), ctx); 90634b8ab09SDaniel Borkmann } else { 90734b8ab09SDaniel Borkmann emit(A64_LDXR(isdw, tmp2, reg), ctx); 90885f68fe8SDaniel Borkmann emit(A64_ADD(isdw, tmp2, tmp2, src), ctx); 90934b8ab09SDaniel Borkmann emit(A64_STXR(isdw, tmp2, reg, tmp3), ctx); 91085f68fe8SDaniel Borkmann jmp_offset = -3; 91185f68fe8SDaniel Borkmann check_imm19(jmp_offset); 9127005cadeSDaniel Borkmann emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); 91334b8ab09SDaniel Borkmann } 91485f68fe8SDaniel Borkmann break; 915e54bcde3SZi Shen Lim 916e54bcde3SZi Shen Lim default: 917e54bcde3SZi Shen Lim pr_err_once("unknown opcode %02x\n", code); 918e54bcde3SZi Shen Lim return -EINVAL; 919e54bcde3SZi Shen Lim } 920e54bcde3SZi Shen Lim 921e54bcde3SZi Shen Lim return 0; 922e54bcde3SZi Shen Lim } 923e54bcde3SZi Shen Lim 9248c11ea5cSDaniel Borkmann static int build_body(struct jit_ctx *ctx, bool extra_pass) 925e54bcde3SZi Shen Lim { 926e54bcde3SZi Shen Lim const struct bpf_prog *prog = ctx->prog; 927e54bcde3SZi Shen Lim int i; 928e54bcde3SZi Shen Lim 92932f6865cSIlias Apalodimas /* 93032f6865cSIlias Apalodimas * - offset[0] offset of the end of prologue, 93132f6865cSIlias Apalodimas * start of the 1st instruction. 93232f6865cSIlias Apalodimas * - offset[1] - offset of the end of 1st instruction, 93332f6865cSIlias Apalodimas * start of the 2nd instruction 93432f6865cSIlias Apalodimas * [....] 93532f6865cSIlias Apalodimas * - offset[3] - offset of the end of 3rd instruction, 93632f6865cSIlias Apalodimas * start of 4th instruction 93732f6865cSIlias Apalodimas */ 938e54bcde3SZi Shen Lim for (i = 0; i < prog->len; i++) { 939e54bcde3SZi Shen Lim const struct bpf_insn *insn = &prog->insnsi[i]; 940e54bcde3SZi Shen Lim int ret; 941e54bcde3SZi Shen Lim 94232f6865cSIlias Apalodimas if (ctx->image == NULL) 94332f6865cSIlias Apalodimas ctx->offset[i] = ctx->idx; 9448c11ea5cSDaniel Borkmann ret = build_insn(insn, ctx, extra_pass); 94530d3d94cSZi Shen Lim if (ret > 0) { 94630d3d94cSZi Shen Lim i++; 947ddc665a4SDaniel Borkmann if (ctx->image == NULL) 948ddc665a4SDaniel Borkmann ctx->offset[i] = ctx->idx; 94930d3d94cSZi Shen Lim continue; 95030d3d94cSZi Shen Lim } 951e54bcde3SZi Shen Lim if (ret) 952e54bcde3SZi Shen Lim return ret; 953e54bcde3SZi Shen Lim } 95432f6865cSIlias Apalodimas /* 95532f6865cSIlias Apalodimas * offset is allocated with prog->len + 1 so fill in 95632f6865cSIlias Apalodimas * the last element with the offset after the last 95732f6865cSIlias Apalodimas * instruction (end of program) 95832f6865cSIlias Apalodimas */ 95932f6865cSIlias Apalodimas if (ctx->image == NULL) 96032f6865cSIlias Apalodimas ctx->offset[i] = ctx->idx; 961e54bcde3SZi Shen Lim 962e54bcde3SZi Shen Lim return 0; 963e54bcde3SZi Shen Lim } 964e54bcde3SZi Shen Lim 96542ff712bSZi Shen Lim static int validate_code(struct jit_ctx *ctx) 96642ff712bSZi Shen Lim { 96742ff712bSZi Shen Lim int i; 96842ff712bSZi Shen Lim 96942ff712bSZi Shen Lim for (i = 0; i < ctx->idx; i++) { 97042ff712bSZi Shen Lim u32 a64_insn = le32_to_cpu(ctx->image[i]); 97142ff712bSZi Shen Lim 97242ff712bSZi Shen Lim if (a64_insn == AARCH64_BREAK_FAULT) 97342ff712bSZi Shen Lim return -1; 97442ff712bSZi Shen Lim } 97542ff712bSZi Shen Lim 97680083428SJean-Philippe Brucker if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries)) 97780083428SJean-Philippe Brucker return -1; 97880083428SJean-Philippe Brucker 97942ff712bSZi Shen Lim return 0; 98042ff712bSZi Shen Lim } 98142ff712bSZi Shen Lim 982e54bcde3SZi Shen Lim static inline void bpf_flush_icache(void *start, void *end) 983e54bcde3SZi Shen Lim { 984e54bcde3SZi Shen Lim flush_icache_range((unsigned long)start, (unsigned long)end); 985e54bcde3SZi Shen Lim } 986e54bcde3SZi Shen Lim 987db496944SAlexei Starovoitov struct arm64_jit_data { 988db496944SAlexei Starovoitov struct bpf_binary_header *header; 989db496944SAlexei Starovoitov u8 *image; 990db496944SAlexei Starovoitov struct jit_ctx ctx; 991db496944SAlexei Starovoitov }; 992db496944SAlexei Starovoitov 993d1c55ab5SDaniel Borkmann struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 994e54bcde3SZi Shen Lim { 99580083428SJean-Philippe Brucker int image_size, prog_size, extable_size; 99626eb042eSDaniel Borkmann struct bpf_prog *tmp, *orig_prog = prog; 997b569c1c6SDaniel Borkmann struct bpf_binary_header *header; 998db496944SAlexei Starovoitov struct arm64_jit_data *jit_data; 99956ea6a8bSDaniel Borkmann bool was_classic = bpf_prog_was_classic(prog); 100026eb042eSDaniel Borkmann bool tmp_blinded = false; 1001db496944SAlexei Starovoitov bool extra_pass = false; 1002e54bcde3SZi Shen Lim struct jit_ctx ctx; 1003b569c1c6SDaniel Borkmann u8 *image_ptr; 1004e54bcde3SZi Shen Lim 100560b58afcSAlexei Starovoitov if (!prog->jit_requested) 100626eb042eSDaniel Borkmann return orig_prog; 100726eb042eSDaniel Borkmann 100826eb042eSDaniel Borkmann tmp = bpf_jit_blind_constants(prog); 100926eb042eSDaniel Borkmann /* If blinding was requested and we failed during blinding, 101026eb042eSDaniel Borkmann * we must fall back to the interpreter. 101126eb042eSDaniel Borkmann */ 101226eb042eSDaniel Borkmann if (IS_ERR(tmp)) 101326eb042eSDaniel Borkmann return orig_prog; 101426eb042eSDaniel Borkmann if (tmp != prog) { 101526eb042eSDaniel Borkmann tmp_blinded = true; 101626eb042eSDaniel Borkmann prog = tmp; 101726eb042eSDaniel Borkmann } 1018e54bcde3SZi Shen Lim 1019db496944SAlexei Starovoitov jit_data = prog->aux->jit_data; 1020db496944SAlexei Starovoitov if (!jit_data) { 1021db496944SAlexei Starovoitov jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1022db496944SAlexei Starovoitov if (!jit_data) { 1023db496944SAlexei Starovoitov prog = orig_prog; 1024db496944SAlexei Starovoitov goto out; 1025db496944SAlexei Starovoitov } 1026db496944SAlexei Starovoitov prog->aux->jit_data = jit_data; 1027db496944SAlexei Starovoitov } 1028db496944SAlexei Starovoitov if (jit_data->ctx.offset) { 1029db496944SAlexei Starovoitov ctx = jit_data->ctx; 1030db496944SAlexei Starovoitov image_ptr = jit_data->image; 1031db496944SAlexei Starovoitov header = jit_data->header; 1032db496944SAlexei Starovoitov extra_pass = true; 103380083428SJean-Philippe Brucker prog_size = sizeof(u32) * ctx.idx; 1034db496944SAlexei Starovoitov goto skip_init_ctx; 1035db496944SAlexei Starovoitov } 1036e54bcde3SZi Shen Lim memset(&ctx, 0, sizeof(ctx)); 1037e54bcde3SZi Shen Lim ctx.prog = prog; 1038e54bcde3SZi Shen Lim 103932f6865cSIlias Apalodimas ctx.offset = kcalloc(prog->len + 1, sizeof(int), GFP_KERNEL); 104026eb042eSDaniel Borkmann if (ctx.offset == NULL) { 104126eb042eSDaniel Borkmann prog = orig_prog; 1042db496944SAlexei Starovoitov goto out_off; 104326eb042eSDaniel Borkmann } 1044e54bcde3SZi Shen Lim 1045e54bcde3SZi Shen Lim /* 1. Initial fake pass to compute ctx->idx. */ 1046e54bcde3SZi Shen Lim 10474c1cd4fdSYang Shi /* Fake pass to fill in ctx->offset. */ 10488c11ea5cSDaniel Borkmann if (build_body(&ctx, extra_pass)) { 104926eb042eSDaniel Borkmann prog = orig_prog; 105026eb042eSDaniel Borkmann goto out_off; 105126eb042eSDaniel Borkmann } 1052e54bcde3SZi Shen Lim 105356ea6a8bSDaniel Borkmann if (build_prologue(&ctx, was_classic)) { 1054ddb55992SZi Shen Lim prog = orig_prog; 1055ddb55992SZi Shen Lim goto out_off; 1056ddb55992SZi Shen Lim } 105751c9fbb1SZi Shen Lim 105851c9fbb1SZi Shen Lim ctx.epilogue_offset = ctx.idx; 1059e54bcde3SZi Shen Lim build_epilogue(&ctx); 1060e54bcde3SZi Shen Lim 106180083428SJean-Philippe Brucker extable_size = prog->aux->num_exentries * 106280083428SJean-Philippe Brucker sizeof(struct exception_table_entry); 106380083428SJean-Philippe Brucker 1064e54bcde3SZi Shen Lim /* Now we know the actual image size. */ 106580083428SJean-Philippe Brucker prog_size = sizeof(u32) * ctx.idx; 106680083428SJean-Philippe Brucker image_size = prog_size + extable_size; 1067b569c1c6SDaniel Borkmann header = bpf_jit_binary_alloc(image_size, &image_ptr, 1068b569c1c6SDaniel Borkmann sizeof(u32), jit_fill_hole); 106926eb042eSDaniel Borkmann if (header == NULL) { 107026eb042eSDaniel Borkmann prog = orig_prog; 107126eb042eSDaniel Borkmann goto out_off; 107226eb042eSDaniel Borkmann } 1073e54bcde3SZi Shen Lim 1074e54bcde3SZi Shen Lim /* 2. Now, the actual pass. */ 1075e54bcde3SZi Shen Lim 1076425e1ed7SLuc Van Oostenryck ctx.image = (__le32 *)image_ptr; 107780083428SJean-Philippe Brucker if (extable_size) 107880083428SJean-Philippe Brucker prog->aux->extable = (void *)image_ptr + prog_size; 1079db496944SAlexei Starovoitov skip_init_ctx: 1080e54bcde3SZi Shen Lim ctx.idx = 0; 108180083428SJean-Philippe Brucker ctx.exentry_idx = 0; 1082b569c1c6SDaniel Borkmann 108356ea6a8bSDaniel Borkmann build_prologue(&ctx, was_classic); 1084e54bcde3SZi Shen Lim 10858c11ea5cSDaniel Borkmann if (build_body(&ctx, extra_pass)) { 1086b569c1c6SDaniel Borkmann bpf_jit_binary_free(header); 108726eb042eSDaniel Borkmann prog = orig_prog; 108826eb042eSDaniel Borkmann goto out_off; 108960ef0494SDaniel Borkmann } 1090e54bcde3SZi Shen Lim 1091e54bcde3SZi Shen Lim build_epilogue(&ctx); 1092e54bcde3SZi Shen Lim 109342ff712bSZi Shen Lim /* 3. Extra pass to validate JITed code. */ 109442ff712bSZi Shen Lim if (validate_code(&ctx)) { 109542ff712bSZi Shen Lim bpf_jit_binary_free(header); 109626eb042eSDaniel Borkmann prog = orig_prog; 109726eb042eSDaniel Borkmann goto out_off; 109842ff712bSZi Shen Lim } 109942ff712bSZi Shen Lim 1100e54bcde3SZi Shen Lim /* And we're done. */ 1101e54bcde3SZi Shen Lim if (bpf_jit_enable > 1) 110280083428SJean-Philippe Brucker bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 1103e54bcde3SZi Shen Lim 1104c3d4c682SDaniel Borkmann bpf_flush_icache(header, ctx.image + ctx.idx); 1105b569c1c6SDaniel Borkmann 1106db496944SAlexei Starovoitov if (!prog->is_func || extra_pass) { 1107db496944SAlexei Starovoitov if (extra_pass && ctx.idx != jit_data->ctx.idx) { 1108db496944SAlexei Starovoitov pr_err_once("multi-func JIT bug %d != %d\n", 1109db496944SAlexei Starovoitov ctx.idx, jit_data->ctx.idx); 1110db496944SAlexei Starovoitov bpf_jit_binary_free(header); 1111db496944SAlexei Starovoitov prog->bpf_func = NULL; 1112db496944SAlexei Starovoitov prog->jited = 0; 1113db496944SAlexei Starovoitov goto out_off; 1114db496944SAlexei Starovoitov } 11159d876e79SDaniel Borkmann bpf_jit_binary_lock_ro(header); 1116db496944SAlexei Starovoitov } else { 1117db496944SAlexei Starovoitov jit_data->ctx = ctx; 1118db496944SAlexei Starovoitov jit_data->image = image_ptr; 1119db496944SAlexei Starovoitov jit_data->header = header; 1120db496944SAlexei Starovoitov } 1121e54bcde3SZi Shen Lim prog->bpf_func = (void *)ctx.image; 1122a91263d5SDaniel Borkmann prog->jited = 1; 112380083428SJean-Philippe Brucker prog->jited_len = prog_size; 112426eb042eSDaniel Borkmann 1125db496944SAlexei Starovoitov if (!prog->is_func || extra_pass) { 112632f6865cSIlias Apalodimas bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); 112726eb042eSDaniel Borkmann out_off: 1128e54bcde3SZi Shen Lim kfree(ctx.offset); 1129db496944SAlexei Starovoitov kfree(jit_data); 1130db496944SAlexei Starovoitov prog->aux->jit_data = NULL; 1131db496944SAlexei Starovoitov } 113226eb042eSDaniel Borkmann out: 113326eb042eSDaniel Borkmann if (tmp_blinded) 113426eb042eSDaniel Borkmann bpf_jit_prog_release_other(prog, prog == orig_prog ? 113526eb042eSDaniel Borkmann tmp : orig_prog); 1136d1c55ab5SDaniel Borkmann return prog; 1137e54bcde3SZi Shen Lim } 113891fc957cSArd Biesheuvel 1139*5d63ae90SLorenz Bauer u64 bpf_jit_alloc_exec_limit(void) 1140*5d63ae90SLorenz Bauer { 1141*5d63ae90SLorenz Bauer return BPF_JIT_REGION_SIZE; 1142*5d63ae90SLorenz Bauer } 1143*5d63ae90SLorenz Bauer 114491fc957cSArd Biesheuvel void *bpf_jit_alloc_exec(unsigned long size) 114591fc957cSArd Biesheuvel { 114691fc957cSArd Biesheuvel return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, 114791fc957cSArd Biesheuvel BPF_JIT_REGION_END, GFP_KERNEL, 11483f750706SArd Biesheuvel PAGE_KERNEL, 0, NUMA_NO_NODE, 114991fc957cSArd Biesheuvel __builtin_return_address(0)); 115091fc957cSArd Biesheuvel } 115191fc957cSArd Biesheuvel 115291fc957cSArd Biesheuvel void bpf_jit_free_exec(void *addr) 115391fc957cSArd Biesheuvel { 115491fc957cSArd Biesheuvel return vfree(addr); 115591fc957cSArd Biesheuvel } 1156