1e54bcde3SZi Shen Lim /* 2e54bcde3SZi Shen Lim * BPF JIT compiler for ARM64 3e54bcde3SZi Shen Lim * 442ff712bSZi Shen Lim * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com> 5e54bcde3SZi Shen Lim * 6e54bcde3SZi Shen Lim * This program is free software; you can redistribute it and/or modify 7e54bcde3SZi Shen Lim * it under the terms of the GNU General Public License version 2 as 8e54bcde3SZi Shen Lim * published by the Free Software Foundation. 9e54bcde3SZi Shen Lim * 10e54bcde3SZi Shen Lim * This program is distributed in the hope that it will be useful, 11e54bcde3SZi Shen Lim * but WITHOUT ANY WARRANTY; without even the implied warranty of 12e54bcde3SZi Shen Lim * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13e54bcde3SZi Shen Lim * GNU General Public License for more details. 14e54bcde3SZi Shen Lim * 15e54bcde3SZi Shen Lim * You should have received a copy of the GNU General Public License 16e54bcde3SZi Shen Lim * along with this program. If not, see <http://www.gnu.org/licenses/>. 17e54bcde3SZi Shen Lim */ 18e54bcde3SZi Shen Lim 19e54bcde3SZi Shen Lim #define pr_fmt(fmt) "bpf_jit: " fmt 20e54bcde3SZi Shen Lim 21ddb55992SZi Shen Lim #include <linux/bpf.h> 22e54bcde3SZi Shen Lim #include <linux/filter.h> 23e54bcde3SZi Shen Lim #include <linux/printk.h> 24e54bcde3SZi Shen Lim #include <linux/slab.h> 25b569c1c6SDaniel Borkmann 26e54bcde3SZi Shen Lim #include <asm/byteorder.h> 27e54bcde3SZi Shen Lim #include <asm/cacheflush.h> 28b569c1c6SDaniel Borkmann #include <asm/debug-monitors.h> 29d4bbc30bSLaura Abbott #include <asm/set_memory.h> 30e54bcde3SZi Shen Lim 31e54bcde3SZi Shen Lim #include "bpf_jit.h" 32e54bcde3SZi Shen Lim 3326eb042eSDaniel Borkmann #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) 3426eb042eSDaniel Borkmann #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 35ddb55992SZi Shen Lim #define TCALL_CNT (MAX_BPF_JIT_REG + 2) 367005cadeSDaniel Borkmann #define TMP_REG_3 (MAX_BPF_JIT_REG + 3) 37e54bcde3SZi Shen Lim 38e54bcde3SZi Shen Lim /* Map BPF registers to A64 registers */ 39e54bcde3SZi Shen Lim static const int bpf2a64[] = { 40e54bcde3SZi Shen Lim /* return value from in-kernel function, and exit value from eBPF */ 41e54bcde3SZi Shen Lim [BPF_REG_0] = A64_R(7), 42e54bcde3SZi Shen Lim /* arguments from eBPF program to in-kernel function */ 43e54bcde3SZi Shen Lim [BPF_REG_1] = A64_R(0), 44e54bcde3SZi Shen Lim [BPF_REG_2] = A64_R(1), 45e54bcde3SZi Shen Lim [BPF_REG_3] = A64_R(2), 46e54bcde3SZi Shen Lim [BPF_REG_4] = A64_R(3), 47e54bcde3SZi Shen Lim [BPF_REG_5] = A64_R(4), 48e54bcde3SZi Shen Lim /* callee saved registers that in-kernel function will preserve */ 49e54bcde3SZi Shen Lim [BPF_REG_6] = A64_R(19), 50e54bcde3SZi Shen Lim [BPF_REG_7] = A64_R(20), 51e54bcde3SZi Shen Lim [BPF_REG_8] = A64_R(21), 52e54bcde3SZi Shen Lim [BPF_REG_9] = A64_R(22), 53e54bcde3SZi Shen Lim /* read-only frame pointer to access stack */ 54ec0738dbSYang Shi [BPF_REG_FP] = A64_R(25), 554c1cd4fdSYang Shi /* temporary registers for internal BPF JIT */ 564c1cd4fdSYang Shi [TMP_REG_1] = A64_R(10), 574c1cd4fdSYang Shi [TMP_REG_2] = A64_R(11), 587005cadeSDaniel Borkmann [TMP_REG_3] = A64_R(12), 59ddb55992SZi Shen Lim /* tail_call_cnt */ 60ddb55992SZi Shen Lim [TCALL_CNT] = A64_R(26), 6126eb042eSDaniel Borkmann /* temporary register for blinding constants */ 6226eb042eSDaniel Borkmann [BPF_REG_AX] = A64_R(9), 63e54bcde3SZi Shen Lim }; 64e54bcde3SZi Shen Lim 65e54bcde3SZi Shen Lim struct jit_ctx { 66e54bcde3SZi Shen Lim const struct bpf_prog *prog; 67e54bcde3SZi Shen Lim int idx; 6851c9fbb1SZi Shen Lim int epilogue_offset; 69e54bcde3SZi Shen Lim int *offset; 70425e1ed7SLuc Van Oostenryck __le32 *image; 71f1c9eed7SDaniel Borkmann u32 stack_size; 72e54bcde3SZi Shen Lim }; 73e54bcde3SZi Shen Lim 74e54bcde3SZi Shen Lim static inline void emit(const u32 insn, struct jit_ctx *ctx) 75e54bcde3SZi Shen Lim { 76e54bcde3SZi Shen Lim if (ctx->image != NULL) 77e54bcde3SZi Shen Lim ctx->image[ctx->idx] = cpu_to_le32(insn); 78e54bcde3SZi Shen Lim 79e54bcde3SZi Shen Lim ctx->idx++; 80e54bcde3SZi Shen Lim } 81e54bcde3SZi Shen Lim 82e54bcde3SZi Shen Lim static inline void emit_a64_mov_i(const int is64, const int reg, 83e54bcde3SZi Shen Lim const s32 val, struct jit_ctx *ctx) 84e54bcde3SZi Shen Lim { 85e54bcde3SZi Shen Lim u16 hi = val >> 16; 86e54bcde3SZi Shen Lim u16 lo = val & 0xffff; 87e54bcde3SZi Shen Lim 88e54bcde3SZi Shen Lim if (hi & 0x8000) { 89e54bcde3SZi Shen Lim if (hi == 0xffff) { 90e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx); 91e54bcde3SZi Shen Lim } else { 92e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx); 936d2eea6fSDaniel Borkmann if (lo != 0xffff) 94e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, lo, 0), ctx); 95e54bcde3SZi Shen Lim } 96e54bcde3SZi Shen Lim } else { 97e54bcde3SZi Shen Lim emit(A64_MOVZ(is64, reg, lo, 0), ctx); 98e54bcde3SZi Shen Lim if (hi) 99e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, hi, 16), ctx); 100e54bcde3SZi Shen Lim } 101e54bcde3SZi Shen Lim } 102e54bcde3SZi Shen Lim 1036d2eea6fSDaniel Borkmann static int i64_i16_blocks(const u64 val, bool inverse) 1046d2eea6fSDaniel Borkmann { 1056d2eea6fSDaniel Borkmann return (((val >> 0) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 1066d2eea6fSDaniel Borkmann (((val >> 16) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 1076d2eea6fSDaniel Borkmann (((val >> 32) & 0xffff) != (inverse ? 0xffff : 0x0000)) + 1086d2eea6fSDaniel Borkmann (((val >> 48) & 0xffff) != (inverse ? 0xffff : 0x0000)); 1096d2eea6fSDaniel Borkmann } 1106d2eea6fSDaniel Borkmann 1116d2eea6fSDaniel Borkmann static inline void emit_a64_mov_i64(const int reg, const u64 val, 1126d2eea6fSDaniel Borkmann struct jit_ctx *ctx) 1136d2eea6fSDaniel Borkmann { 1146d2eea6fSDaniel Borkmann u64 nrm_tmp = val, rev_tmp = ~val; 1156d2eea6fSDaniel Borkmann bool inverse; 1166d2eea6fSDaniel Borkmann int shift; 1176d2eea6fSDaniel Borkmann 1186d2eea6fSDaniel Borkmann if (!(nrm_tmp >> 32)) 1196d2eea6fSDaniel Borkmann return emit_a64_mov_i(0, reg, (u32)val, ctx); 1206d2eea6fSDaniel Borkmann 1216d2eea6fSDaniel Borkmann inverse = i64_i16_blocks(nrm_tmp, true) < i64_i16_blocks(nrm_tmp, false); 1226d2eea6fSDaniel Borkmann shift = max(round_down((inverse ? (fls64(rev_tmp) - 1) : 1236d2eea6fSDaniel Borkmann (fls64(nrm_tmp) - 1)), 16), 0); 1246d2eea6fSDaniel Borkmann if (inverse) 1256d2eea6fSDaniel Borkmann emit(A64_MOVN(1, reg, (rev_tmp >> shift) & 0xffff, shift), ctx); 1266d2eea6fSDaniel Borkmann else 1276d2eea6fSDaniel Borkmann emit(A64_MOVZ(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx); 1286d2eea6fSDaniel Borkmann shift -= 16; 1296d2eea6fSDaniel Borkmann while (shift >= 0) { 1306d2eea6fSDaniel Borkmann if (((nrm_tmp >> shift) & 0xffff) != (inverse ? 0xffff : 0x0000)) 1316d2eea6fSDaniel Borkmann emit(A64_MOVK(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx); 1326d2eea6fSDaniel Borkmann shift -= 16; 1336d2eea6fSDaniel Borkmann } 1346d2eea6fSDaniel Borkmann } 1356d2eea6fSDaniel Borkmann 1366d2eea6fSDaniel Borkmann /* 1376d2eea6fSDaniel Borkmann * This is an unoptimized 64 immediate emission used for BPF to BPF call 1386d2eea6fSDaniel Borkmann * addresses. It will always do a full 64 bit decomposition as otherwise 1396d2eea6fSDaniel Borkmann * more complexity in the last extra pass is required since we previously 1406d2eea6fSDaniel Borkmann * reserved 4 instructions for the address. 1416d2eea6fSDaniel Borkmann */ 1426d2eea6fSDaniel Borkmann static inline void emit_addr_mov_i64(const int reg, const u64 val, 1436d2eea6fSDaniel Borkmann struct jit_ctx *ctx) 1446d2eea6fSDaniel Borkmann { 1456d2eea6fSDaniel Borkmann u64 tmp = val; 1466d2eea6fSDaniel Borkmann int shift = 0; 1476d2eea6fSDaniel Borkmann 1486d2eea6fSDaniel Borkmann emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx); 1496d2eea6fSDaniel Borkmann for (;shift < 48;) { 1506d2eea6fSDaniel Borkmann tmp >>= 16; 1516d2eea6fSDaniel Borkmann shift += 16; 1526d2eea6fSDaniel Borkmann emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx); 1536d2eea6fSDaniel Borkmann } 1546d2eea6fSDaniel Borkmann } 1556d2eea6fSDaniel Borkmann 156e54bcde3SZi Shen Lim static inline int bpf2a64_offset(int bpf_to, int bpf_from, 157e54bcde3SZi Shen Lim const struct jit_ctx *ctx) 158e54bcde3SZi Shen Lim { 1598eee539dSXi Wang int to = ctx->offset[bpf_to]; 160e54bcde3SZi Shen Lim /* -1 to account for the Branch instruction */ 1618eee539dSXi Wang int from = ctx->offset[bpf_from] - 1; 162e54bcde3SZi Shen Lim 163e54bcde3SZi Shen Lim return to - from; 164e54bcde3SZi Shen Lim } 165e54bcde3SZi Shen Lim 166b569c1c6SDaniel Borkmann static void jit_fill_hole(void *area, unsigned int size) 167b569c1c6SDaniel Borkmann { 168425e1ed7SLuc Van Oostenryck __le32 *ptr; 169b569c1c6SDaniel Borkmann /* We are guaranteed to have aligned memory. */ 170b569c1c6SDaniel Borkmann for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 171b569c1c6SDaniel Borkmann *ptr++ = cpu_to_le32(AARCH64_BREAK_FAULT); 172b569c1c6SDaniel Borkmann } 173b569c1c6SDaniel Borkmann 174e54bcde3SZi Shen Lim static inline int epilogue_offset(const struct jit_ctx *ctx) 175e54bcde3SZi Shen Lim { 17651c9fbb1SZi Shen Lim int to = ctx->epilogue_offset; 17751c9fbb1SZi Shen Lim int from = ctx->idx; 178e54bcde3SZi Shen Lim 179e54bcde3SZi Shen Lim return to - from; 180e54bcde3SZi Shen Lim } 181e54bcde3SZi Shen Lim 182e54bcde3SZi Shen Lim /* Stack must be multiples of 16B */ 183e54bcde3SZi Shen Lim #define STACK_ALIGN(sz) (((sz) + 15) & ~15) 184e54bcde3SZi Shen Lim 185a2284d91SDaniel Borkmann /* Tail call offset to jump into */ 186a2284d91SDaniel Borkmann #define PROLOGUE_OFFSET 7 187ddb55992SZi Shen Lim 18856ea6a8bSDaniel Borkmann static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) 189e54bcde3SZi Shen Lim { 190f1c9eed7SDaniel Borkmann const struct bpf_prog *prog = ctx->prog; 191e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 192e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 193e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 194e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 195e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 196ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 197ddb55992SZi Shen Lim const int idx0 = ctx->idx; 198ddb55992SZi Shen Lim int cur_offset; 199e54bcde3SZi Shen Lim 200ec0738dbSYang Shi /* 201ec0738dbSYang Shi * BPF prog stack layout 202ec0738dbSYang Shi * 203ec0738dbSYang Shi * high 204ec0738dbSYang Shi * original A64_SP => 0:+-----+ BPF prologue 205ec0738dbSYang Shi * |FP/LR| 206ec0738dbSYang Shi * current A64_FP => -16:+-----+ 207ec0738dbSYang Shi * | ... | callee saved registers 2084c1cd4fdSYang Shi * BPF fp register => -64:+-----+ <= (BPF_FP) 209ec0738dbSYang Shi * | | 210ec0738dbSYang Shi * | ... | BPF prog stack 211ec0738dbSYang Shi * | | 212f1c9eed7SDaniel Borkmann * +-----+ <= (BPF_FP - prog->aux->stack_depth) 21309ece3d0SDaniel Borkmann * |RSVD | padding 214f1c9eed7SDaniel Borkmann * current A64_SP => +-----+ <= (BPF_FP - ctx->stack_size) 215ec0738dbSYang Shi * | | 216ec0738dbSYang Shi * | ... | Function call stack 217ec0738dbSYang Shi * | | 218ec0738dbSYang Shi * +-----+ 219ec0738dbSYang Shi * low 220ec0738dbSYang Shi * 221ec0738dbSYang Shi */ 222ec0738dbSYang Shi 223ec0738dbSYang Shi /* Save FP and LR registers to stay align with ARM64 AAPCS */ 224ec0738dbSYang Shi emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); 225ec0738dbSYang Shi emit(A64_MOV(1, A64_FP, A64_SP), ctx); 226ec0738dbSYang Shi 227ddb55992SZi Shen Lim /* Save callee-saved registers */ 228e54bcde3SZi Shen Lim emit(A64_PUSH(r6, r7, A64_SP), ctx); 229e54bcde3SZi Shen Lim emit(A64_PUSH(r8, r9, A64_SP), ctx); 230ddb55992SZi Shen Lim emit(A64_PUSH(fp, tcc, A64_SP), ctx); 231e54bcde3SZi Shen Lim 232ddb55992SZi Shen Lim /* Set up BPF prog stack base register */ 233e54bcde3SZi Shen Lim emit(A64_MOV(1, fp, A64_SP), ctx); 234e54bcde3SZi Shen Lim 23556ea6a8bSDaniel Borkmann if (!ebpf_from_cbpf) { 236ddb55992SZi Shen Lim /* Initialize tail_call_cnt */ 237ddb55992SZi Shen Lim emit(A64_MOVZ(1, tcc, 0, 0), ctx); 238ddb55992SZi Shen Lim 239ddb55992SZi Shen Lim cur_offset = ctx->idx - idx0; 240ddb55992SZi Shen Lim if (cur_offset != PROLOGUE_OFFSET) { 241ddb55992SZi Shen Lim pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n", 242ddb55992SZi Shen Lim cur_offset, PROLOGUE_OFFSET); 243ddb55992SZi Shen Lim return -1; 244ddb55992SZi Shen Lim } 24556ea6a8bSDaniel Borkmann } 246a2284d91SDaniel Borkmann 24709ece3d0SDaniel Borkmann ctx->stack_size = STACK_ALIGN(prog->aux->stack_depth); 248a2284d91SDaniel Borkmann 249a2284d91SDaniel Borkmann /* Set up function call stack */ 250a2284d91SDaniel Borkmann emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 251ddb55992SZi Shen Lim return 0; 252ddb55992SZi Shen Lim } 253ddb55992SZi Shen Lim 254ddb55992SZi Shen Lim static int out_offset = -1; /* initialized on the first pass of build_body() */ 255ddb55992SZi Shen Lim static int emit_bpf_tail_call(struct jit_ctx *ctx) 256ddb55992SZi Shen Lim { 257ddb55992SZi Shen Lim /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ 258ddb55992SZi Shen Lim const u8 r2 = bpf2a64[BPF_REG_2]; 259ddb55992SZi Shen Lim const u8 r3 = bpf2a64[BPF_REG_3]; 260ddb55992SZi Shen Lim 261ddb55992SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 262ddb55992SZi Shen Lim const u8 prg = bpf2a64[TMP_REG_2]; 263ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 264ddb55992SZi Shen Lim const int idx0 = ctx->idx; 265ddb55992SZi Shen Lim #define cur_offset (ctx->idx - idx0) 266ddb55992SZi Shen Lim #define jmp_offset (out_offset - (cur_offset)) 267ddb55992SZi Shen Lim size_t off; 268ddb55992SZi Shen Lim 269ddb55992SZi Shen Lim /* if (index >= array->map.max_entries) 270ddb55992SZi Shen Lim * goto out; 271ddb55992SZi Shen Lim */ 272ddb55992SZi Shen Lim off = offsetof(struct bpf_array, map.max_entries); 273ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 274ddb55992SZi Shen Lim emit(A64_LDR32(tmp, r2, tmp), ctx); 27516338a9bSDaniel Borkmann emit(A64_MOV(0, r3, r3), ctx); 276ddb55992SZi Shen Lim emit(A64_CMP(0, r3, tmp), ctx); 27716338a9bSDaniel Borkmann emit(A64_B_(A64_COND_CS, jmp_offset), ctx); 278ddb55992SZi Shen Lim 279ddb55992SZi Shen Lim /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) 280ddb55992SZi Shen Lim * goto out; 281ddb55992SZi Shen Lim * tail_call_cnt++; 282ddb55992SZi Shen Lim */ 283ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx); 284ddb55992SZi Shen Lim emit(A64_CMP(1, tcc, tmp), ctx); 28516338a9bSDaniel Borkmann emit(A64_B_(A64_COND_HI, jmp_offset), ctx); 286ddb55992SZi Shen Lim emit(A64_ADD_I(1, tcc, tcc, 1), ctx); 287ddb55992SZi Shen Lim 288ddb55992SZi Shen Lim /* prog = array->ptrs[index]; 289ddb55992SZi Shen Lim * if (prog == NULL) 290ddb55992SZi Shen Lim * goto out; 291ddb55992SZi Shen Lim */ 292ddb55992SZi Shen Lim off = offsetof(struct bpf_array, ptrs); 293ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 294d8b54110SDaniel Borkmann emit(A64_ADD(1, tmp, r2, tmp), ctx); 295d8b54110SDaniel Borkmann emit(A64_LSL(1, prg, r3, 3), ctx); 296d8b54110SDaniel Borkmann emit(A64_LDR64(prg, tmp, prg), ctx); 297ddb55992SZi Shen Lim emit(A64_CBZ(1, prg, jmp_offset), ctx); 298ddb55992SZi Shen Lim 299a2284d91SDaniel Borkmann /* goto *(prog->bpf_func + prologue_offset); */ 300ddb55992SZi Shen Lim off = offsetof(struct bpf_prog, bpf_func); 301ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 302ddb55992SZi Shen Lim emit(A64_LDR64(tmp, prg, tmp), ctx); 303ddb55992SZi Shen Lim emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx); 304a2284d91SDaniel Borkmann emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 305ddb55992SZi Shen Lim emit(A64_BR(tmp), ctx); 306ddb55992SZi Shen Lim 307ddb55992SZi Shen Lim /* out: */ 308ddb55992SZi Shen Lim if (out_offset == -1) 309ddb55992SZi Shen Lim out_offset = cur_offset; 310ddb55992SZi Shen Lim if (cur_offset != out_offset) { 311ddb55992SZi Shen Lim pr_err_once("tail_call out_offset = %d, expected %d!\n", 312ddb55992SZi Shen Lim cur_offset, out_offset); 313ddb55992SZi Shen Lim return -1; 314ddb55992SZi Shen Lim } 315ddb55992SZi Shen Lim return 0; 316ddb55992SZi Shen Lim #undef cur_offset 317ddb55992SZi Shen Lim #undef jmp_offset 318e54bcde3SZi Shen Lim } 319e54bcde3SZi Shen Lim 320e54bcde3SZi Shen Lim static void build_epilogue(struct jit_ctx *ctx) 321e54bcde3SZi Shen Lim { 322e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 323e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 324e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 325e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 326e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 327e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 328e54bcde3SZi Shen Lim 329e54bcde3SZi Shen Lim /* We're done with BPF stack */ 330f1c9eed7SDaniel Borkmann emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); 331e54bcde3SZi Shen Lim 332ec0738dbSYang Shi /* Restore fs (x25) and x26 */ 333ec0738dbSYang Shi emit(A64_POP(fp, A64_R(26), A64_SP), ctx); 334ec0738dbSYang Shi 335e54bcde3SZi Shen Lim /* Restore callee-saved register */ 336e54bcde3SZi Shen Lim emit(A64_POP(r8, r9, A64_SP), ctx); 337e54bcde3SZi Shen Lim emit(A64_POP(r6, r7, A64_SP), ctx); 338e54bcde3SZi Shen Lim 339ec0738dbSYang Shi /* Restore FP/LR registers */ 340ec0738dbSYang Shi emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); 341e54bcde3SZi Shen Lim 342e54bcde3SZi Shen Lim /* Set return value */ 343e54bcde3SZi Shen Lim emit(A64_MOV(1, A64_R(0), r0), ctx); 344e54bcde3SZi Shen Lim 345e54bcde3SZi Shen Lim emit(A64_RET(A64_LR), ctx); 346e54bcde3SZi Shen Lim } 347e54bcde3SZi Shen Lim 34830d3d94cSZi Shen Lim /* JITs an eBPF instruction. 34930d3d94cSZi Shen Lim * Returns: 35030d3d94cSZi Shen Lim * 0 - successfully JITed an 8-byte eBPF instruction. 35130d3d94cSZi Shen Lim * >0 - successfully JITed a 16-byte eBPF instruction. 35230d3d94cSZi Shen Lim * <0 - failed to JIT. 35330d3d94cSZi Shen Lim */ 354*8c11ea5cSDaniel Borkmann static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 355*8c11ea5cSDaniel Borkmann bool extra_pass) 356e54bcde3SZi Shen Lim { 357e54bcde3SZi Shen Lim const u8 code = insn->code; 358e54bcde3SZi Shen Lim const u8 dst = bpf2a64[insn->dst_reg]; 359e54bcde3SZi Shen Lim const u8 src = bpf2a64[insn->src_reg]; 360e54bcde3SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 361e54bcde3SZi Shen Lim const u8 tmp2 = bpf2a64[TMP_REG_2]; 3627005cadeSDaniel Borkmann const u8 tmp3 = bpf2a64[TMP_REG_3]; 363e54bcde3SZi Shen Lim const s16 off = insn->off; 364e54bcde3SZi Shen Lim const s32 imm = insn->imm; 365e54bcde3SZi Shen Lim const int i = insn - ctx->prog->insnsi; 366e54bcde3SZi Shen Lim const bool is64 = BPF_CLASS(code) == BPF_ALU64; 36785f68fe8SDaniel Borkmann const bool isdw = BPF_SIZE(code) == BPF_DW; 368e54bcde3SZi Shen Lim u8 jmp_cond; 369e54bcde3SZi Shen Lim s32 jmp_offset; 370e54bcde3SZi Shen Lim 371251599e1SZi Shen Lim #define check_imm(bits, imm) do { \ 372251599e1SZi Shen Lim if ((((imm) > 0) && ((imm) >> (bits))) || \ 373251599e1SZi Shen Lim (((imm) < 0) && (~(imm) >> (bits)))) { \ 374251599e1SZi Shen Lim pr_info("[%2d] imm=%d(0x%x) out of range\n", \ 375251599e1SZi Shen Lim i, imm, imm); \ 376251599e1SZi Shen Lim return -EINVAL; \ 377251599e1SZi Shen Lim } \ 378251599e1SZi Shen Lim } while (0) 379251599e1SZi Shen Lim #define check_imm19(imm) check_imm(19, imm) 380251599e1SZi Shen Lim #define check_imm26(imm) check_imm(26, imm) 381251599e1SZi Shen Lim 382e54bcde3SZi Shen Lim switch (code) { 383e54bcde3SZi Shen Lim /* dst = src */ 384e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_X: 385e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_X: 386e54bcde3SZi Shen Lim emit(A64_MOV(is64, dst, src), ctx); 387e54bcde3SZi Shen Lim break; 388e54bcde3SZi Shen Lim /* dst = dst OP src */ 389e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_X: 390e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_X: 391e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, src), ctx); 392e54bcde3SZi Shen Lim break; 393e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_X: 394e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_X: 395e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, src), ctx); 396e54bcde3SZi Shen Lim break; 397e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_X: 398e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_X: 399e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, src), ctx); 400e54bcde3SZi Shen Lim break; 401e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_X: 402e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_X: 403e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, src), ctx); 404e54bcde3SZi Shen Lim break; 405e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_X: 406e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_X: 407e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, src), ctx); 408e54bcde3SZi Shen Lim break; 409e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_X: 410e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_X: 411e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, src), ctx); 412e54bcde3SZi Shen Lim break; 413e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_X: 414e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_X: 415e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOD | BPF_X: 416e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOD | BPF_X: 41714e589ffSZi Shen Lim switch (BPF_OP(code)) { 41814e589ffSZi Shen Lim case BPF_DIV: 419e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, src), ctx); 420e54bcde3SZi Shen Lim break; 42114e589ffSZi Shen Lim case BPF_MOD: 422e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, src), ctx); 423e54bcde3SZi Shen Lim emit(A64_MUL(is64, tmp, tmp, src), ctx); 424e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 425e54bcde3SZi Shen Lim break; 42614e589ffSZi Shen Lim } 42714e589ffSZi Shen Lim break; 428d65a634aSZi Shen Lim case BPF_ALU | BPF_LSH | BPF_X: 429d65a634aSZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_X: 430d65a634aSZi Shen Lim emit(A64_LSLV(is64, dst, dst, src), ctx); 431d65a634aSZi Shen Lim break; 432d65a634aSZi Shen Lim case BPF_ALU | BPF_RSH | BPF_X: 433d65a634aSZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_X: 434d65a634aSZi Shen Lim emit(A64_LSRV(is64, dst, dst, src), ctx); 435d65a634aSZi Shen Lim break; 436d65a634aSZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_X: 437d65a634aSZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_X: 438d65a634aSZi Shen Lim emit(A64_ASRV(is64, dst, dst, src), ctx); 439d65a634aSZi Shen Lim break; 440e54bcde3SZi Shen Lim /* dst = -dst */ 441e54bcde3SZi Shen Lim case BPF_ALU | BPF_NEG: 442e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_NEG: 443e54bcde3SZi Shen Lim emit(A64_NEG(is64, dst, dst), ctx); 444e54bcde3SZi Shen Lim break; 445e54bcde3SZi Shen Lim /* dst = BSWAP##imm(dst) */ 446e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_LE: 447e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_BE: 448e54bcde3SZi Shen Lim #ifdef CONFIG_CPU_BIG_ENDIAN 449e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_BE) 450d63903bbSXi Wang goto emit_bswap_uxt; 451e54bcde3SZi Shen Lim #else /* !CONFIG_CPU_BIG_ENDIAN */ 452e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_LE) 453d63903bbSXi Wang goto emit_bswap_uxt; 454e54bcde3SZi Shen Lim #endif 455e54bcde3SZi Shen Lim switch (imm) { 456e54bcde3SZi Shen Lim case 16: 457e54bcde3SZi Shen Lim emit(A64_REV16(is64, dst, dst), ctx); 458d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 459d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 460e54bcde3SZi Shen Lim break; 461e54bcde3SZi Shen Lim case 32: 462e54bcde3SZi Shen Lim emit(A64_REV32(is64, dst, dst), ctx); 463d63903bbSXi Wang /* upper 32 bits already cleared */ 464e54bcde3SZi Shen Lim break; 465e54bcde3SZi Shen Lim case 64: 466e54bcde3SZi Shen Lim emit(A64_REV64(dst, dst), ctx); 467e54bcde3SZi Shen Lim break; 468e54bcde3SZi Shen Lim } 469e54bcde3SZi Shen Lim break; 470d63903bbSXi Wang emit_bswap_uxt: 471d63903bbSXi Wang switch (imm) { 472d63903bbSXi Wang case 16: 473d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 474d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 475d63903bbSXi Wang break; 476d63903bbSXi Wang case 32: 477d63903bbSXi Wang /* zero-extend 32 bits into 64 bits */ 478d63903bbSXi Wang emit(A64_UXTW(is64, dst, dst), ctx); 479d63903bbSXi Wang break; 480d63903bbSXi Wang case 64: 481d63903bbSXi Wang /* nop */ 482d63903bbSXi Wang break; 483d63903bbSXi Wang } 484d63903bbSXi Wang break; 485e54bcde3SZi Shen Lim /* dst = imm */ 486e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_K: 487e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_K: 488e54bcde3SZi Shen Lim emit_a64_mov_i(is64, dst, imm, ctx); 489e54bcde3SZi Shen Lim break; 490e54bcde3SZi Shen Lim /* dst = dst OP imm */ 491e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_K: 492e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_K: 493e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 494e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, tmp), ctx); 495e54bcde3SZi Shen Lim break; 496e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_K: 497e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_K: 498e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 499e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 500e54bcde3SZi Shen Lim break; 501e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_K: 502e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_K: 503e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 504e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, tmp), ctx); 505e54bcde3SZi Shen Lim break; 506e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_K: 507e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_K: 508e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 509e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, tmp), ctx); 510e54bcde3SZi Shen Lim break; 511e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_K: 512e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_K: 513e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 514e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, tmp), ctx); 515e54bcde3SZi Shen Lim break; 516e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_K: 517e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_K: 518e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 519e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, tmp), ctx); 520e54bcde3SZi Shen Lim break; 521e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_K: 522e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_K: 523e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 524e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, tmp), ctx); 525e54bcde3SZi Shen Lim break; 526e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOD | BPF_K: 527e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOD | BPF_K: 528e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp2, imm, ctx); 529e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, tmp2), ctx); 530e54bcde3SZi Shen Lim emit(A64_MUL(is64, tmp, tmp, tmp2), ctx); 531e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 532e54bcde3SZi Shen Lim break; 533e54bcde3SZi Shen Lim case BPF_ALU | BPF_LSH | BPF_K: 534e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_K: 535e54bcde3SZi Shen Lim emit(A64_LSL(is64, dst, dst, imm), ctx); 536e54bcde3SZi Shen Lim break; 537e54bcde3SZi Shen Lim case BPF_ALU | BPF_RSH | BPF_K: 538e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_K: 539e54bcde3SZi Shen Lim emit(A64_LSR(is64, dst, dst, imm), ctx); 540e54bcde3SZi Shen Lim break; 541e54bcde3SZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_K: 542e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_K: 543e54bcde3SZi Shen Lim emit(A64_ASR(is64, dst, dst, imm), ctx); 544e54bcde3SZi Shen Lim break; 545e54bcde3SZi Shen Lim 546e54bcde3SZi Shen Lim /* JUMP off */ 547e54bcde3SZi Shen Lim case BPF_JMP | BPF_JA: 548e54bcde3SZi Shen Lim jmp_offset = bpf2a64_offset(i + off, i, ctx); 549e54bcde3SZi Shen Lim check_imm26(jmp_offset); 550e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 551e54bcde3SZi Shen Lim break; 552e54bcde3SZi Shen Lim /* IF (dst COND src) JUMP off */ 553e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_X: 554e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_X: 555c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_X: 556e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_X: 557c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_X: 558e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_X: 559e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_X: 560c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_X: 561e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_X: 562c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_X: 563e54bcde3SZi Shen Lim emit(A64_CMP(1, dst, src), ctx); 564e54bcde3SZi Shen Lim emit_cond_jmp: 565e54bcde3SZi Shen Lim jmp_offset = bpf2a64_offset(i + off, i, ctx); 566e54bcde3SZi Shen Lim check_imm19(jmp_offset); 567e54bcde3SZi Shen Lim switch (BPF_OP(code)) { 568e54bcde3SZi Shen Lim case BPF_JEQ: 569e54bcde3SZi Shen Lim jmp_cond = A64_COND_EQ; 570e54bcde3SZi Shen Lim break; 571e54bcde3SZi Shen Lim case BPF_JGT: 572e54bcde3SZi Shen Lim jmp_cond = A64_COND_HI; 573e54bcde3SZi Shen Lim break; 574c362b2f3SDaniel Borkmann case BPF_JLT: 575c362b2f3SDaniel Borkmann jmp_cond = A64_COND_CC; 576c362b2f3SDaniel Borkmann break; 577e54bcde3SZi Shen Lim case BPF_JGE: 578e54bcde3SZi Shen Lim jmp_cond = A64_COND_CS; 579e54bcde3SZi Shen Lim break; 580c362b2f3SDaniel Borkmann case BPF_JLE: 581c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LS; 582c362b2f3SDaniel Borkmann break; 58398397fc5SZi Shen Lim case BPF_JSET: 584e54bcde3SZi Shen Lim case BPF_JNE: 585e54bcde3SZi Shen Lim jmp_cond = A64_COND_NE; 586e54bcde3SZi Shen Lim break; 587e54bcde3SZi Shen Lim case BPF_JSGT: 588e54bcde3SZi Shen Lim jmp_cond = A64_COND_GT; 589e54bcde3SZi Shen Lim break; 590c362b2f3SDaniel Borkmann case BPF_JSLT: 591c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LT; 592c362b2f3SDaniel Borkmann break; 593e54bcde3SZi Shen Lim case BPF_JSGE: 594e54bcde3SZi Shen Lim jmp_cond = A64_COND_GE; 595e54bcde3SZi Shen Lim break; 596c362b2f3SDaniel Borkmann case BPF_JSLE: 597c362b2f3SDaniel Borkmann jmp_cond = A64_COND_LE; 598c362b2f3SDaniel Borkmann break; 599e54bcde3SZi Shen Lim default: 600e54bcde3SZi Shen Lim return -EFAULT; 601e54bcde3SZi Shen Lim } 602e54bcde3SZi Shen Lim emit(A64_B_(jmp_cond, jmp_offset), ctx); 603e54bcde3SZi Shen Lim break; 604e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_X: 605e54bcde3SZi Shen Lim emit(A64_TST(1, dst, src), ctx); 606e54bcde3SZi Shen Lim goto emit_cond_jmp; 607e54bcde3SZi Shen Lim /* IF (dst COND imm) JUMP off */ 608e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_K: 609e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_K: 610c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_K: 611e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_K: 612c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_K: 613e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_K: 614e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_K: 615c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_K: 616e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_K: 617c362b2f3SDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_K: 618e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, imm, ctx); 619e54bcde3SZi Shen Lim emit(A64_CMP(1, dst, tmp), ctx); 620e54bcde3SZi Shen Lim goto emit_cond_jmp; 621e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_K: 622e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, imm, ctx); 623e54bcde3SZi Shen Lim emit(A64_TST(1, dst, tmp), ctx); 624e54bcde3SZi Shen Lim goto emit_cond_jmp; 625e54bcde3SZi Shen Lim /* function call */ 626e54bcde3SZi Shen Lim case BPF_JMP | BPF_CALL: 627e54bcde3SZi Shen Lim { 628e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 629*8c11ea5cSDaniel Borkmann bool func_addr_fixed; 630*8c11ea5cSDaniel Borkmann u64 func_addr; 631*8c11ea5cSDaniel Borkmann int ret; 632e54bcde3SZi Shen Lim 633*8c11ea5cSDaniel Borkmann ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 634*8c11ea5cSDaniel Borkmann &func_addr, &func_addr_fixed); 635*8c11ea5cSDaniel Borkmann if (ret < 0) 636*8c11ea5cSDaniel Borkmann return ret; 637*8c11ea5cSDaniel Borkmann if (func_addr_fixed) 638*8c11ea5cSDaniel Borkmann /* We can use optimized emission here. */ 639*8c11ea5cSDaniel Borkmann emit_a64_mov_i64(tmp, func_addr, ctx); 640db496944SAlexei Starovoitov else 641*8c11ea5cSDaniel Borkmann emit_addr_mov_i64(tmp, func_addr, ctx); 642e54bcde3SZi Shen Lim emit(A64_BLR(tmp), ctx); 643e54bcde3SZi Shen Lim emit(A64_MOV(1, r0, A64_R(0)), ctx); 644e54bcde3SZi Shen Lim break; 645e54bcde3SZi Shen Lim } 646ddb55992SZi Shen Lim /* tail call */ 64771189fa9SAlexei Starovoitov case BPF_JMP | BPF_TAIL_CALL: 648ddb55992SZi Shen Lim if (emit_bpf_tail_call(ctx)) 649ddb55992SZi Shen Lim return -EFAULT; 650ddb55992SZi Shen Lim break; 651e54bcde3SZi Shen Lim /* function return */ 652e54bcde3SZi Shen Lim case BPF_JMP | BPF_EXIT: 65351c9fbb1SZi Shen Lim /* Optimization: when last instruction is EXIT, 65451c9fbb1SZi Shen Lim simply fallthrough to epilogue. */ 655e54bcde3SZi Shen Lim if (i == ctx->prog->len - 1) 656e54bcde3SZi Shen Lim break; 657e54bcde3SZi Shen Lim jmp_offset = epilogue_offset(ctx); 658e54bcde3SZi Shen Lim check_imm26(jmp_offset); 659e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 660e54bcde3SZi Shen Lim break; 661e54bcde3SZi Shen Lim 66230d3d94cSZi Shen Lim /* dst = imm64 */ 66330d3d94cSZi Shen Lim case BPF_LD | BPF_IMM | BPF_DW: 66430d3d94cSZi Shen Lim { 66530d3d94cSZi Shen Lim const struct bpf_insn insn1 = insn[1]; 66630d3d94cSZi Shen Lim u64 imm64; 66730d3d94cSZi Shen Lim 6681e4df6b7SXi Wang imm64 = (u64)insn1.imm << 32 | (u32)imm; 66930d3d94cSZi Shen Lim emit_a64_mov_i64(dst, imm64, ctx); 67030d3d94cSZi Shen Lim 67130d3d94cSZi Shen Lim return 1; 67230d3d94cSZi Shen Lim } 67330d3d94cSZi Shen Lim 674e54bcde3SZi Shen Lim /* LDX: dst = *(size *)(src + off) */ 675e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_W: 676e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_H: 677e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_B: 678e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_DW: 679e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 680e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 681e54bcde3SZi Shen Lim case BPF_W: 682e54bcde3SZi Shen Lim emit(A64_LDR32(dst, src, tmp), ctx); 683e54bcde3SZi Shen Lim break; 684e54bcde3SZi Shen Lim case BPF_H: 685e54bcde3SZi Shen Lim emit(A64_LDRH(dst, src, tmp), ctx); 686e54bcde3SZi Shen Lim break; 687e54bcde3SZi Shen Lim case BPF_B: 688e54bcde3SZi Shen Lim emit(A64_LDRB(dst, src, tmp), ctx); 689e54bcde3SZi Shen Lim break; 690e54bcde3SZi Shen Lim case BPF_DW: 691e54bcde3SZi Shen Lim emit(A64_LDR64(dst, src, tmp), ctx); 692e54bcde3SZi Shen Lim break; 693e54bcde3SZi Shen Lim } 694e54bcde3SZi Shen Lim break; 695e54bcde3SZi Shen Lim 696e54bcde3SZi Shen Lim /* ST: *(size *)(dst + off) = imm */ 697e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_W: 698e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_H: 699e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_B: 700e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_DW: 701df849ba3SYang Shi /* Load imm to a register then store it */ 702df849ba3SYang Shi emit_a64_mov_i(1, tmp2, off, ctx); 703df849ba3SYang Shi emit_a64_mov_i(1, tmp, imm, ctx); 704df849ba3SYang Shi switch (BPF_SIZE(code)) { 705df849ba3SYang Shi case BPF_W: 706df849ba3SYang Shi emit(A64_STR32(tmp, dst, tmp2), ctx); 707df849ba3SYang Shi break; 708df849ba3SYang Shi case BPF_H: 709df849ba3SYang Shi emit(A64_STRH(tmp, dst, tmp2), ctx); 710df849ba3SYang Shi break; 711df849ba3SYang Shi case BPF_B: 712df849ba3SYang Shi emit(A64_STRB(tmp, dst, tmp2), ctx); 713df849ba3SYang Shi break; 714df849ba3SYang Shi case BPF_DW: 715df849ba3SYang Shi emit(A64_STR64(tmp, dst, tmp2), ctx); 716df849ba3SYang Shi break; 717df849ba3SYang Shi } 718df849ba3SYang Shi break; 719e54bcde3SZi Shen Lim 720e54bcde3SZi Shen Lim /* STX: *(size *)(dst + off) = src */ 721e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_W: 722e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_H: 723e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_B: 724e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_DW: 725e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 726e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 727e54bcde3SZi Shen Lim case BPF_W: 728e54bcde3SZi Shen Lim emit(A64_STR32(src, dst, tmp), ctx); 729e54bcde3SZi Shen Lim break; 730e54bcde3SZi Shen Lim case BPF_H: 731e54bcde3SZi Shen Lim emit(A64_STRH(src, dst, tmp), ctx); 732e54bcde3SZi Shen Lim break; 733e54bcde3SZi Shen Lim case BPF_B: 734e54bcde3SZi Shen Lim emit(A64_STRB(src, dst, tmp), ctx); 735e54bcde3SZi Shen Lim break; 736e54bcde3SZi Shen Lim case BPF_DW: 737e54bcde3SZi Shen Lim emit(A64_STR64(src, dst, tmp), ctx); 738e54bcde3SZi Shen Lim break; 739e54bcde3SZi Shen Lim } 740e54bcde3SZi Shen Lim break; 741e54bcde3SZi Shen Lim /* STX XADD: lock *(u32 *)(dst + off) += src */ 742e54bcde3SZi Shen Lim case BPF_STX | BPF_XADD | BPF_W: 743e54bcde3SZi Shen Lim /* STX XADD: lock *(u64 *)(dst + off) += src */ 744e54bcde3SZi Shen Lim case BPF_STX | BPF_XADD | BPF_DW: 74585f68fe8SDaniel Borkmann emit_a64_mov_i(1, tmp, off, ctx); 74685f68fe8SDaniel Borkmann emit(A64_ADD(1, tmp, tmp, dst), ctx); 74785f68fe8SDaniel Borkmann emit(A64_PRFM(tmp, PST, L1, STRM), ctx); 74885f68fe8SDaniel Borkmann emit(A64_LDXR(isdw, tmp2, tmp), ctx); 74985f68fe8SDaniel Borkmann emit(A64_ADD(isdw, tmp2, tmp2, src), ctx); 7507005cadeSDaniel Borkmann emit(A64_STXR(isdw, tmp2, tmp, tmp3), ctx); 75185f68fe8SDaniel Borkmann jmp_offset = -3; 75285f68fe8SDaniel Borkmann check_imm19(jmp_offset); 7537005cadeSDaniel Borkmann emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); 75485f68fe8SDaniel Borkmann break; 755e54bcde3SZi Shen Lim 756e54bcde3SZi Shen Lim default: 757e54bcde3SZi Shen Lim pr_err_once("unknown opcode %02x\n", code); 758e54bcde3SZi Shen Lim return -EINVAL; 759e54bcde3SZi Shen Lim } 760e54bcde3SZi Shen Lim 761e54bcde3SZi Shen Lim return 0; 762e54bcde3SZi Shen Lim } 763e54bcde3SZi Shen Lim 764*8c11ea5cSDaniel Borkmann static int build_body(struct jit_ctx *ctx, bool extra_pass) 765e54bcde3SZi Shen Lim { 766e54bcde3SZi Shen Lim const struct bpf_prog *prog = ctx->prog; 767e54bcde3SZi Shen Lim int i; 768e54bcde3SZi Shen Lim 769e54bcde3SZi Shen Lim for (i = 0; i < prog->len; i++) { 770e54bcde3SZi Shen Lim const struct bpf_insn *insn = &prog->insnsi[i]; 771e54bcde3SZi Shen Lim int ret; 772e54bcde3SZi Shen Lim 773*8c11ea5cSDaniel Borkmann ret = build_insn(insn, ctx, extra_pass); 77430d3d94cSZi Shen Lim if (ret > 0) { 77530d3d94cSZi Shen Lim i++; 776ddc665a4SDaniel Borkmann if (ctx->image == NULL) 777ddc665a4SDaniel Borkmann ctx->offset[i] = ctx->idx; 77830d3d94cSZi Shen Lim continue; 77930d3d94cSZi Shen Lim } 780ddc665a4SDaniel Borkmann if (ctx->image == NULL) 781ddc665a4SDaniel Borkmann ctx->offset[i] = ctx->idx; 782e54bcde3SZi Shen Lim if (ret) 783e54bcde3SZi Shen Lim return ret; 784e54bcde3SZi Shen Lim } 785e54bcde3SZi Shen Lim 786e54bcde3SZi Shen Lim return 0; 787e54bcde3SZi Shen Lim } 788e54bcde3SZi Shen Lim 78942ff712bSZi Shen Lim static int validate_code(struct jit_ctx *ctx) 79042ff712bSZi Shen Lim { 79142ff712bSZi Shen Lim int i; 79242ff712bSZi Shen Lim 79342ff712bSZi Shen Lim for (i = 0; i < ctx->idx; i++) { 79442ff712bSZi Shen Lim u32 a64_insn = le32_to_cpu(ctx->image[i]); 79542ff712bSZi Shen Lim 79642ff712bSZi Shen Lim if (a64_insn == AARCH64_BREAK_FAULT) 79742ff712bSZi Shen Lim return -1; 79842ff712bSZi Shen Lim } 79942ff712bSZi Shen Lim 80042ff712bSZi Shen Lim return 0; 80142ff712bSZi Shen Lim } 80242ff712bSZi Shen Lim 803e54bcde3SZi Shen Lim static inline void bpf_flush_icache(void *start, void *end) 804e54bcde3SZi Shen Lim { 805e54bcde3SZi Shen Lim flush_icache_range((unsigned long)start, (unsigned long)end); 806e54bcde3SZi Shen Lim } 807e54bcde3SZi Shen Lim 808db496944SAlexei Starovoitov struct arm64_jit_data { 809db496944SAlexei Starovoitov struct bpf_binary_header *header; 810db496944SAlexei Starovoitov u8 *image; 811db496944SAlexei Starovoitov struct jit_ctx ctx; 812db496944SAlexei Starovoitov }; 813db496944SAlexei Starovoitov 814d1c55ab5SDaniel Borkmann struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 815e54bcde3SZi Shen Lim { 81626eb042eSDaniel Borkmann struct bpf_prog *tmp, *orig_prog = prog; 817b569c1c6SDaniel Borkmann struct bpf_binary_header *header; 818db496944SAlexei Starovoitov struct arm64_jit_data *jit_data; 81956ea6a8bSDaniel Borkmann bool was_classic = bpf_prog_was_classic(prog); 82026eb042eSDaniel Borkmann bool tmp_blinded = false; 821db496944SAlexei Starovoitov bool extra_pass = false; 822e54bcde3SZi Shen Lim struct jit_ctx ctx; 823e54bcde3SZi Shen Lim int image_size; 824b569c1c6SDaniel Borkmann u8 *image_ptr; 825e54bcde3SZi Shen Lim 82660b58afcSAlexei Starovoitov if (!prog->jit_requested) 82726eb042eSDaniel Borkmann return orig_prog; 82826eb042eSDaniel Borkmann 82926eb042eSDaniel Borkmann tmp = bpf_jit_blind_constants(prog); 83026eb042eSDaniel Borkmann /* If blinding was requested and we failed during blinding, 83126eb042eSDaniel Borkmann * we must fall back to the interpreter. 83226eb042eSDaniel Borkmann */ 83326eb042eSDaniel Borkmann if (IS_ERR(tmp)) 83426eb042eSDaniel Borkmann return orig_prog; 83526eb042eSDaniel Borkmann if (tmp != prog) { 83626eb042eSDaniel Borkmann tmp_blinded = true; 83726eb042eSDaniel Borkmann prog = tmp; 83826eb042eSDaniel Borkmann } 839e54bcde3SZi Shen Lim 840db496944SAlexei Starovoitov jit_data = prog->aux->jit_data; 841db496944SAlexei Starovoitov if (!jit_data) { 842db496944SAlexei Starovoitov jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 843db496944SAlexei Starovoitov if (!jit_data) { 844db496944SAlexei Starovoitov prog = orig_prog; 845db496944SAlexei Starovoitov goto out; 846db496944SAlexei Starovoitov } 847db496944SAlexei Starovoitov prog->aux->jit_data = jit_data; 848db496944SAlexei Starovoitov } 849db496944SAlexei Starovoitov if (jit_data->ctx.offset) { 850db496944SAlexei Starovoitov ctx = jit_data->ctx; 851db496944SAlexei Starovoitov image_ptr = jit_data->image; 852db496944SAlexei Starovoitov header = jit_data->header; 853db496944SAlexei Starovoitov extra_pass = true; 8545ee7f784SAlexei Starovoitov image_size = sizeof(u32) * ctx.idx; 855db496944SAlexei Starovoitov goto skip_init_ctx; 856db496944SAlexei Starovoitov } 857e54bcde3SZi Shen Lim memset(&ctx, 0, sizeof(ctx)); 858e54bcde3SZi Shen Lim ctx.prog = prog; 859e54bcde3SZi Shen Lim 860e54bcde3SZi Shen Lim ctx.offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); 86126eb042eSDaniel Borkmann if (ctx.offset == NULL) { 86226eb042eSDaniel Borkmann prog = orig_prog; 863db496944SAlexei Starovoitov goto out_off; 86426eb042eSDaniel Borkmann } 865e54bcde3SZi Shen Lim 866e54bcde3SZi Shen Lim /* 1. Initial fake pass to compute ctx->idx. */ 867e54bcde3SZi Shen Lim 8684c1cd4fdSYang Shi /* Fake pass to fill in ctx->offset. */ 869*8c11ea5cSDaniel Borkmann if (build_body(&ctx, extra_pass)) { 87026eb042eSDaniel Borkmann prog = orig_prog; 87126eb042eSDaniel Borkmann goto out_off; 87226eb042eSDaniel Borkmann } 873e54bcde3SZi Shen Lim 87456ea6a8bSDaniel Borkmann if (build_prologue(&ctx, was_classic)) { 875ddb55992SZi Shen Lim prog = orig_prog; 876ddb55992SZi Shen Lim goto out_off; 877ddb55992SZi Shen Lim } 87851c9fbb1SZi Shen Lim 87951c9fbb1SZi Shen Lim ctx.epilogue_offset = ctx.idx; 880e54bcde3SZi Shen Lim build_epilogue(&ctx); 881e54bcde3SZi Shen Lim 882e54bcde3SZi Shen Lim /* Now we know the actual image size. */ 883e54bcde3SZi Shen Lim image_size = sizeof(u32) * ctx.idx; 884b569c1c6SDaniel Borkmann header = bpf_jit_binary_alloc(image_size, &image_ptr, 885b569c1c6SDaniel Borkmann sizeof(u32), jit_fill_hole); 88626eb042eSDaniel Borkmann if (header == NULL) { 88726eb042eSDaniel Borkmann prog = orig_prog; 88826eb042eSDaniel Borkmann goto out_off; 88926eb042eSDaniel Borkmann } 890e54bcde3SZi Shen Lim 891e54bcde3SZi Shen Lim /* 2. Now, the actual pass. */ 892e54bcde3SZi Shen Lim 893425e1ed7SLuc Van Oostenryck ctx.image = (__le32 *)image_ptr; 894db496944SAlexei Starovoitov skip_init_ctx: 895e54bcde3SZi Shen Lim ctx.idx = 0; 896b569c1c6SDaniel Borkmann 89756ea6a8bSDaniel Borkmann build_prologue(&ctx, was_classic); 898e54bcde3SZi Shen Lim 899*8c11ea5cSDaniel Borkmann if (build_body(&ctx, extra_pass)) { 900b569c1c6SDaniel Borkmann bpf_jit_binary_free(header); 90126eb042eSDaniel Borkmann prog = orig_prog; 90226eb042eSDaniel Borkmann goto out_off; 90360ef0494SDaniel Borkmann } 904e54bcde3SZi Shen Lim 905e54bcde3SZi Shen Lim build_epilogue(&ctx); 906e54bcde3SZi Shen Lim 90742ff712bSZi Shen Lim /* 3. Extra pass to validate JITed code. */ 90842ff712bSZi Shen Lim if (validate_code(&ctx)) { 90942ff712bSZi Shen Lim bpf_jit_binary_free(header); 91026eb042eSDaniel Borkmann prog = orig_prog; 91126eb042eSDaniel Borkmann goto out_off; 91242ff712bSZi Shen Lim } 91342ff712bSZi Shen Lim 914e54bcde3SZi Shen Lim /* And we're done. */ 915e54bcde3SZi Shen Lim if (bpf_jit_enable > 1) 916e54bcde3SZi Shen Lim bpf_jit_dump(prog->len, image_size, 2, ctx.image); 917e54bcde3SZi Shen Lim 918c3d4c682SDaniel Borkmann bpf_flush_icache(header, ctx.image + ctx.idx); 919b569c1c6SDaniel Borkmann 920db496944SAlexei Starovoitov if (!prog->is_func || extra_pass) { 921db496944SAlexei Starovoitov if (extra_pass && ctx.idx != jit_data->ctx.idx) { 922db496944SAlexei Starovoitov pr_err_once("multi-func JIT bug %d != %d\n", 923db496944SAlexei Starovoitov ctx.idx, jit_data->ctx.idx); 924db496944SAlexei Starovoitov bpf_jit_binary_free(header); 925db496944SAlexei Starovoitov prog->bpf_func = NULL; 926db496944SAlexei Starovoitov prog->jited = 0; 927db496944SAlexei Starovoitov goto out_off; 928db496944SAlexei Starovoitov } 9299d876e79SDaniel Borkmann bpf_jit_binary_lock_ro(header); 930db496944SAlexei Starovoitov } else { 931db496944SAlexei Starovoitov jit_data->ctx = ctx; 932db496944SAlexei Starovoitov jit_data->image = image_ptr; 933db496944SAlexei Starovoitov jit_data->header = header; 934db496944SAlexei Starovoitov } 935e54bcde3SZi Shen Lim prog->bpf_func = (void *)ctx.image; 936a91263d5SDaniel Borkmann prog->jited = 1; 937783d28ddSMartin KaFai Lau prog->jited_len = image_size; 93826eb042eSDaniel Borkmann 939db496944SAlexei Starovoitov if (!prog->is_func || extra_pass) { 94026eb042eSDaniel Borkmann out_off: 941e54bcde3SZi Shen Lim kfree(ctx.offset); 942db496944SAlexei Starovoitov kfree(jit_data); 943db496944SAlexei Starovoitov prog->aux->jit_data = NULL; 944db496944SAlexei Starovoitov } 94526eb042eSDaniel Borkmann out: 94626eb042eSDaniel Borkmann if (tmp_blinded) 94726eb042eSDaniel Borkmann bpf_jit_prog_release_other(prog, prog == orig_prog ? 94826eb042eSDaniel Borkmann tmp : orig_prog); 949d1c55ab5SDaniel Borkmann return prog; 950e54bcde3SZi Shen Lim } 951