1e54bcde3SZi Shen Lim /* 2e54bcde3SZi Shen Lim * BPF JIT compiler for ARM64 3e54bcde3SZi Shen Lim * 442ff712bSZi Shen Lim * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com> 5e54bcde3SZi Shen Lim * 6e54bcde3SZi Shen Lim * This program is free software; you can redistribute it and/or modify 7e54bcde3SZi Shen Lim * it under the terms of the GNU General Public License version 2 as 8e54bcde3SZi Shen Lim * published by the Free Software Foundation. 9e54bcde3SZi Shen Lim * 10e54bcde3SZi Shen Lim * This program is distributed in the hope that it will be useful, 11e54bcde3SZi Shen Lim * but WITHOUT ANY WARRANTY; without even the implied warranty of 12e54bcde3SZi Shen Lim * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13e54bcde3SZi Shen Lim * GNU General Public License for more details. 14e54bcde3SZi Shen Lim * 15e54bcde3SZi Shen Lim * You should have received a copy of the GNU General Public License 16e54bcde3SZi Shen Lim * along with this program. If not, see <http://www.gnu.org/licenses/>. 17e54bcde3SZi Shen Lim */ 18e54bcde3SZi Shen Lim 19e54bcde3SZi Shen Lim #define pr_fmt(fmt) "bpf_jit: " fmt 20e54bcde3SZi Shen Lim 21ddb55992SZi Shen Lim #include <linux/bpf.h> 22e54bcde3SZi Shen Lim #include <linux/filter.h> 23e54bcde3SZi Shen Lim #include <linux/printk.h> 24e54bcde3SZi Shen Lim #include <linux/skbuff.h> 25e54bcde3SZi Shen Lim #include <linux/slab.h> 26b569c1c6SDaniel Borkmann 27e54bcde3SZi Shen Lim #include <asm/byteorder.h> 28e54bcde3SZi Shen Lim #include <asm/cacheflush.h> 29b569c1c6SDaniel Borkmann #include <asm/debug-monitors.h> 30e54bcde3SZi Shen Lim 31e54bcde3SZi Shen Lim #include "bpf_jit.h" 32e54bcde3SZi Shen Lim 33e54bcde3SZi Shen Lim int bpf_jit_enable __read_mostly; 34e54bcde3SZi Shen Lim 3526eb042eSDaniel Borkmann #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) 3626eb042eSDaniel Borkmann #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 37ddb55992SZi Shen Lim #define TCALL_CNT (MAX_BPF_JIT_REG + 2) 38e54bcde3SZi Shen Lim 39e54bcde3SZi Shen Lim /* Map BPF registers to A64 registers */ 40e54bcde3SZi Shen Lim static const int bpf2a64[] = { 41e54bcde3SZi Shen Lim /* return value from in-kernel function, and exit value from eBPF */ 42e54bcde3SZi Shen Lim [BPF_REG_0] = A64_R(7), 43e54bcde3SZi Shen Lim /* arguments from eBPF program to in-kernel function */ 44e54bcde3SZi Shen Lim [BPF_REG_1] = A64_R(0), 45e54bcde3SZi Shen Lim [BPF_REG_2] = A64_R(1), 46e54bcde3SZi Shen Lim [BPF_REG_3] = A64_R(2), 47e54bcde3SZi Shen Lim [BPF_REG_4] = A64_R(3), 48e54bcde3SZi Shen Lim [BPF_REG_5] = A64_R(4), 49e54bcde3SZi Shen Lim /* callee saved registers that in-kernel function will preserve */ 50e54bcde3SZi Shen Lim [BPF_REG_6] = A64_R(19), 51e54bcde3SZi Shen Lim [BPF_REG_7] = A64_R(20), 52e54bcde3SZi Shen Lim [BPF_REG_8] = A64_R(21), 53e54bcde3SZi Shen Lim [BPF_REG_9] = A64_R(22), 54e54bcde3SZi Shen Lim /* read-only frame pointer to access stack */ 55ec0738dbSYang Shi [BPF_REG_FP] = A64_R(25), 564c1cd4fdSYang Shi /* temporary registers for internal BPF JIT */ 574c1cd4fdSYang Shi [TMP_REG_1] = A64_R(10), 584c1cd4fdSYang Shi [TMP_REG_2] = A64_R(11), 59ddb55992SZi Shen Lim /* tail_call_cnt */ 60ddb55992SZi Shen Lim [TCALL_CNT] = A64_R(26), 6126eb042eSDaniel Borkmann /* temporary register for blinding constants */ 6226eb042eSDaniel Borkmann [BPF_REG_AX] = A64_R(9), 63e54bcde3SZi Shen Lim }; 64e54bcde3SZi Shen Lim 65e54bcde3SZi Shen Lim struct jit_ctx { 66e54bcde3SZi Shen Lim const struct bpf_prog *prog; 67e54bcde3SZi Shen Lim int idx; 6851c9fbb1SZi Shen Lim int epilogue_offset; 69e54bcde3SZi Shen Lim int *offset; 70e54bcde3SZi Shen Lim u32 *image; 71e54bcde3SZi Shen Lim }; 72e54bcde3SZi Shen Lim 73e54bcde3SZi Shen Lim static inline void emit(const u32 insn, struct jit_ctx *ctx) 74e54bcde3SZi Shen Lim { 75e54bcde3SZi Shen Lim if (ctx->image != NULL) 76e54bcde3SZi Shen Lim ctx->image[ctx->idx] = cpu_to_le32(insn); 77e54bcde3SZi Shen Lim 78e54bcde3SZi Shen Lim ctx->idx++; 79e54bcde3SZi Shen Lim } 80e54bcde3SZi Shen Lim 81e54bcde3SZi Shen Lim static inline void emit_a64_mov_i64(const int reg, const u64 val, 82e54bcde3SZi Shen Lim struct jit_ctx *ctx) 83e54bcde3SZi Shen Lim { 84e54bcde3SZi Shen Lim u64 tmp = val; 85e54bcde3SZi Shen Lim int shift = 0; 86e54bcde3SZi Shen Lim 87e54bcde3SZi Shen Lim emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx); 88e54bcde3SZi Shen Lim tmp >>= 16; 89e54bcde3SZi Shen Lim shift += 16; 90e54bcde3SZi Shen Lim while (tmp) { 91e54bcde3SZi Shen Lim if (tmp & 0xffff) 92e54bcde3SZi Shen Lim emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx); 93e54bcde3SZi Shen Lim tmp >>= 16; 94e54bcde3SZi Shen Lim shift += 16; 95e54bcde3SZi Shen Lim } 96e54bcde3SZi Shen Lim } 97e54bcde3SZi Shen Lim 98e54bcde3SZi Shen Lim static inline void emit_a64_mov_i(const int is64, const int reg, 99e54bcde3SZi Shen Lim const s32 val, struct jit_ctx *ctx) 100e54bcde3SZi Shen Lim { 101e54bcde3SZi Shen Lim u16 hi = val >> 16; 102e54bcde3SZi Shen Lim u16 lo = val & 0xffff; 103e54bcde3SZi Shen Lim 104e54bcde3SZi Shen Lim if (hi & 0x8000) { 105e54bcde3SZi Shen Lim if (hi == 0xffff) { 106e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx); 107e54bcde3SZi Shen Lim } else { 108e54bcde3SZi Shen Lim emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx); 109e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, lo, 0), ctx); 110e54bcde3SZi Shen Lim } 111e54bcde3SZi Shen Lim } else { 112e54bcde3SZi Shen Lim emit(A64_MOVZ(is64, reg, lo, 0), ctx); 113e54bcde3SZi Shen Lim if (hi) 114e54bcde3SZi Shen Lim emit(A64_MOVK(is64, reg, hi, 16), ctx); 115e54bcde3SZi Shen Lim } 116e54bcde3SZi Shen Lim } 117e54bcde3SZi Shen Lim 118e54bcde3SZi Shen Lim static inline int bpf2a64_offset(int bpf_to, int bpf_from, 119e54bcde3SZi Shen Lim const struct jit_ctx *ctx) 120e54bcde3SZi Shen Lim { 1218eee539dSXi Wang int to = ctx->offset[bpf_to]; 122e54bcde3SZi Shen Lim /* -1 to account for the Branch instruction */ 1238eee539dSXi Wang int from = ctx->offset[bpf_from] - 1; 124e54bcde3SZi Shen Lim 125e54bcde3SZi Shen Lim return to - from; 126e54bcde3SZi Shen Lim } 127e54bcde3SZi Shen Lim 128b569c1c6SDaniel Borkmann static void jit_fill_hole(void *area, unsigned int size) 129b569c1c6SDaniel Borkmann { 130b569c1c6SDaniel Borkmann u32 *ptr; 131b569c1c6SDaniel Borkmann /* We are guaranteed to have aligned memory. */ 132b569c1c6SDaniel Borkmann for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 133b569c1c6SDaniel Borkmann *ptr++ = cpu_to_le32(AARCH64_BREAK_FAULT); 134b569c1c6SDaniel Borkmann } 135b569c1c6SDaniel Borkmann 136e54bcde3SZi Shen Lim static inline int epilogue_offset(const struct jit_ctx *ctx) 137e54bcde3SZi Shen Lim { 13851c9fbb1SZi Shen Lim int to = ctx->epilogue_offset; 13951c9fbb1SZi Shen Lim int from = ctx->idx; 140e54bcde3SZi Shen Lim 141e54bcde3SZi Shen Lim return to - from; 142e54bcde3SZi Shen Lim } 143e54bcde3SZi Shen Lim 144e54bcde3SZi Shen Lim /* Stack must be multiples of 16B */ 145e54bcde3SZi Shen Lim #define STACK_ALIGN(sz) (((sz) + 15) & ~15) 146e54bcde3SZi Shen Lim 147f4b16fceSZi Shen Lim #define _STACK_SIZE \ 148f4b16fceSZi Shen Lim (MAX_BPF_STACK \ 149f4b16fceSZi Shen Lim + 4 /* extra for skb_copy_bits buffer */) 150f4b16fceSZi Shen Lim 151f4b16fceSZi Shen Lim #define STACK_SIZE STACK_ALIGN(_STACK_SIZE) 152f4b16fceSZi Shen Lim 153ddb55992SZi Shen Lim #define PROLOGUE_OFFSET 8 154ddb55992SZi Shen Lim 155ddb55992SZi Shen Lim static int build_prologue(struct jit_ctx *ctx) 156e54bcde3SZi Shen Lim { 157e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 158e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 159e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 160e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 161e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 162ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 163ddb55992SZi Shen Lim const int idx0 = ctx->idx; 164ddb55992SZi Shen Lim int cur_offset; 165e54bcde3SZi Shen Lim 166ec0738dbSYang Shi /* 167ec0738dbSYang Shi * BPF prog stack layout 168ec0738dbSYang Shi * 169ec0738dbSYang Shi * high 170ec0738dbSYang Shi * original A64_SP => 0:+-----+ BPF prologue 171ec0738dbSYang Shi * |FP/LR| 172ec0738dbSYang Shi * current A64_FP => -16:+-----+ 173ec0738dbSYang Shi * | ... | callee saved registers 1744c1cd4fdSYang Shi * BPF fp register => -64:+-----+ <= (BPF_FP) 175ec0738dbSYang Shi * | | 176ec0738dbSYang Shi * | ... | BPF prog stack 177ec0738dbSYang Shi * | | 178f4b16fceSZi Shen Lim * +-----+ <= (BPF_FP - MAX_BPF_STACK) 179f4b16fceSZi Shen Lim * |RSVD | JIT scratchpad 180f4b16fceSZi Shen Lim * current A64_SP => +-----+ <= (BPF_FP - STACK_SIZE) 181ec0738dbSYang Shi * | | 182ec0738dbSYang Shi * | ... | Function call stack 183ec0738dbSYang Shi * | | 184ec0738dbSYang Shi * +-----+ 185ec0738dbSYang Shi * low 186ec0738dbSYang Shi * 187ec0738dbSYang Shi */ 188ec0738dbSYang Shi 189ec0738dbSYang Shi /* Save FP and LR registers to stay align with ARM64 AAPCS */ 190ec0738dbSYang Shi emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); 191ec0738dbSYang Shi emit(A64_MOV(1, A64_FP, A64_SP), ctx); 192ec0738dbSYang Shi 193ddb55992SZi Shen Lim /* Save callee-saved registers */ 194e54bcde3SZi Shen Lim emit(A64_PUSH(r6, r7, A64_SP), ctx); 195e54bcde3SZi Shen Lim emit(A64_PUSH(r8, r9, A64_SP), ctx); 196ddb55992SZi Shen Lim emit(A64_PUSH(fp, tcc, A64_SP), ctx); 197e54bcde3SZi Shen Lim 198ddb55992SZi Shen Lim /* Set up BPF prog stack base register */ 199e54bcde3SZi Shen Lim emit(A64_MOV(1, fp, A64_SP), ctx); 200e54bcde3SZi Shen Lim 201ddb55992SZi Shen Lim /* Initialize tail_call_cnt */ 202ddb55992SZi Shen Lim emit(A64_MOVZ(1, tcc, 0, 0), ctx); 203ddb55992SZi Shen Lim 204ec0738dbSYang Shi /* Set up function call stack */ 205f4b16fceSZi Shen Lim emit(A64_SUB_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); 206ddb55992SZi Shen Lim 207ddb55992SZi Shen Lim cur_offset = ctx->idx - idx0; 208ddb55992SZi Shen Lim if (cur_offset != PROLOGUE_OFFSET) { 209ddb55992SZi Shen Lim pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n", 210ddb55992SZi Shen Lim cur_offset, PROLOGUE_OFFSET); 211ddb55992SZi Shen Lim return -1; 212ddb55992SZi Shen Lim } 213ddb55992SZi Shen Lim return 0; 214ddb55992SZi Shen Lim } 215ddb55992SZi Shen Lim 216ddb55992SZi Shen Lim static int out_offset = -1; /* initialized on the first pass of build_body() */ 217ddb55992SZi Shen Lim static int emit_bpf_tail_call(struct jit_ctx *ctx) 218ddb55992SZi Shen Lim { 219ddb55992SZi Shen Lim /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ 220ddb55992SZi Shen Lim const u8 r2 = bpf2a64[BPF_REG_2]; 221ddb55992SZi Shen Lim const u8 r3 = bpf2a64[BPF_REG_3]; 222ddb55992SZi Shen Lim 223ddb55992SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 224ddb55992SZi Shen Lim const u8 prg = bpf2a64[TMP_REG_2]; 225ddb55992SZi Shen Lim const u8 tcc = bpf2a64[TCALL_CNT]; 226ddb55992SZi Shen Lim const int idx0 = ctx->idx; 227ddb55992SZi Shen Lim #define cur_offset (ctx->idx - idx0) 228ddb55992SZi Shen Lim #define jmp_offset (out_offset - (cur_offset)) 229ddb55992SZi Shen Lim size_t off; 230ddb55992SZi Shen Lim 231ddb55992SZi Shen Lim /* if (index >= array->map.max_entries) 232ddb55992SZi Shen Lim * goto out; 233ddb55992SZi Shen Lim */ 234ddb55992SZi Shen Lim off = offsetof(struct bpf_array, map.max_entries); 235ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 236ddb55992SZi Shen Lim emit(A64_LDR32(tmp, r2, tmp), ctx); 237ddb55992SZi Shen Lim emit(A64_CMP(0, r3, tmp), ctx); 238ddb55992SZi Shen Lim emit(A64_B_(A64_COND_GE, jmp_offset), ctx); 239ddb55992SZi Shen Lim 240ddb55992SZi Shen Lim /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) 241ddb55992SZi Shen Lim * goto out; 242ddb55992SZi Shen Lim * tail_call_cnt++; 243ddb55992SZi Shen Lim */ 244ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx); 245ddb55992SZi Shen Lim emit(A64_CMP(1, tcc, tmp), ctx); 246ddb55992SZi Shen Lim emit(A64_B_(A64_COND_GT, jmp_offset), ctx); 247ddb55992SZi Shen Lim emit(A64_ADD_I(1, tcc, tcc, 1), ctx); 248ddb55992SZi Shen Lim 249ddb55992SZi Shen Lim /* prog = array->ptrs[index]; 250ddb55992SZi Shen Lim * if (prog == NULL) 251ddb55992SZi Shen Lim * goto out; 252ddb55992SZi Shen Lim */ 253ddb55992SZi Shen Lim off = offsetof(struct bpf_array, ptrs); 254ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 255ddb55992SZi Shen Lim emit(A64_LDR64(tmp, r2, tmp), ctx); 256ddb55992SZi Shen Lim emit(A64_LDR64(prg, tmp, r3), ctx); 257ddb55992SZi Shen Lim emit(A64_CBZ(1, prg, jmp_offset), ctx); 258ddb55992SZi Shen Lim 259ddb55992SZi Shen Lim /* goto *(prog->bpf_func + prologue_size); */ 260ddb55992SZi Shen Lim off = offsetof(struct bpf_prog, bpf_func); 261ddb55992SZi Shen Lim emit_a64_mov_i64(tmp, off, ctx); 262ddb55992SZi Shen Lim emit(A64_LDR64(tmp, prg, tmp), ctx); 263ddb55992SZi Shen Lim emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx); 264ddb55992SZi Shen Lim emit(A64_BR(tmp), ctx); 265ddb55992SZi Shen Lim 266ddb55992SZi Shen Lim /* out: */ 267ddb55992SZi Shen Lim if (out_offset == -1) 268ddb55992SZi Shen Lim out_offset = cur_offset; 269ddb55992SZi Shen Lim if (cur_offset != out_offset) { 270ddb55992SZi Shen Lim pr_err_once("tail_call out_offset = %d, expected %d!\n", 271ddb55992SZi Shen Lim cur_offset, out_offset); 272ddb55992SZi Shen Lim return -1; 273ddb55992SZi Shen Lim } 274ddb55992SZi Shen Lim return 0; 275ddb55992SZi Shen Lim #undef cur_offset 276ddb55992SZi Shen Lim #undef jmp_offset 277e54bcde3SZi Shen Lim } 278e54bcde3SZi Shen Lim 279e54bcde3SZi Shen Lim static void build_epilogue(struct jit_ctx *ctx) 280e54bcde3SZi Shen Lim { 281e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 282e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; 283e54bcde3SZi Shen Lim const u8 r7 = bpf2a64[BPF_REG_7]; 284e54bcde3SZi Shen Lim const u8 r8 = bpf2a64[BPF_REG_8]; 285e54bcde3SZi Shen Lim const u8 r9 = bpf2a64[BPF_REG_9]; 286e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 287e54bcde3SZi Shen Lim 288e54bcde3SZi Shen Lim /* We're done with BPF stack */ 289f4b16fceSZi Shen Lim emit(A64_ADD_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); 290e54bcde3SZi Shen Lim 291ec0738dbSYang Shi /* Restore fs (x25) and x26 */ 292ec0738dbSYang Shi emit(A64_POP(fp, A64_R(26), A64_SP), ctx); 293ec0738dbSYang Shi 294e54bcde3SZi Shen Lim /* Restore callee-saved register */ 295e54bcde3SZi Shen Lim emit(A64_POP(r8, r9, A64_SP), ctx); 296e54bcde3SZi Shen Lim emit(A64_POP(r6, r7, A64_SP), ctx); 297e54bcde3SZi Shen Lim 298ec0738dbSYang Shi /* Restore FP/LR registers */ 299ec0738dbSYang Shi emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); 300e54bcde3SZi Shen Lim 301e54bcde3SZi Shen Lim /* Set return value */ 302e54bcde3SZi Shen Lim emit(A64_MOV(1, A64_R(0), r0), ctx); 303e54bcde3SZi Shen Lim 304e54bcde3SZi Shen Lim emit(A64_RET(A64_LR), ctx); 305e54bcde3SZi Shen Lim } 306e54bcde3SZi Shen Lim 30730d3d94cSZi Shen Lim /* JITs an eBPF instruction. 30830d3d94cSZi Shen Lim * Returns: 30930d3d94cSZi Shen Lim * 0 - successfully JITed an 8-byte eBPF instruction. 31030d3d94cSZi Shen Lim * >0 - successfully JITed a 16-byte eBPF instruction. 31130d3d94cSZi Shen Lim * <0 - failed to JIT. 31230d3d94cSZi Shen Lim */ 313e54bcde3SZi Shen Lim static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 314e54bcde3SZi Shen Lim { 315e54bcde3SZi Shen Lim const u8 code = insn->code; 316e54bcde3SZi Shen Lim const u8 dst = bpf2a64[insn->dst_reg]; 317e54bcde3SZi Shen Lim const u8 src = bpf2a64[insn->src_reg]; 318e54bcde3SZi Shen Lim const u8 tmp = bpf2a64[TMP_REG_1]; 319e54bcde3SZi Shen Lim const u8 tmp2 = bpf2a64[TMP_REG_2]; 320e54bcde3SZi Shen Lim const s16 off = insn->off; 321e54bcde3SZi Shen Lim const s32 imm = insn->imm; 322e54bcde3SZi Shen Lim const int i = insn - ctx->prog->insnsi; 323e54bcde3SZi Shen Lim const bool is64 = BPF_CLASS(code) == BPF_ALU64; 324*85f68fe8SDaniel Borkmann const bool isdw = BPF_SIZE(code) == BPF_DW; 325e54bcde3SZi Shen Lim u8 jmp_cond; 326e54bcde3SZi Shen Lim s32 jmp_offset; 327e54bcde3SZi Shen Lim 328251599e1SZi Shen Lim #define check_imm(bits, imm) do { \ 329251599e1SZi Shen Lim if ((((imm) > 0) && ((imm) >> (bits))) || \ 330251599e1SZi Shen Lim (((imm) < 0) && (~(imm) >> (bits)))) { \ 331251599e1SZi Shen Lim pr_info("[%2d] imm=%d(0x%x) out of range\n", \ 332251599e1SZi Shen Lim i, imm, imm); \ 333251599e1SZi Shen Lim return -EINVAL; \ 334251599e1SZi Shen Lim } \ 335251599e1SZi Shen Lim } while (0) 336251599e1SZi Shen Lim #define check_imm19(imm) check_imm(19, imm) 337251599e1SZi Shen Lim #define check_imm26(imm) check_imm(26, imm) 338251599e1SZi Shen Lim 339e54bcde3SZi Shen Lim switch (code) { 340e54bcde3SZi Shen Lim /* dst = src */ 341e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_X: 342e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_X: 343e54bcde3SZi Shen Lim emit(A64_MOV(is64, dst, src), ctx); 344e54bcde3SZi Shen Lim break; 345e54bcde3SZi Shen Lim /* dst = dst OP src */ 346e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_X: 347e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_X: 348e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, src), ctx); 349e54bcde3SZi Shen Lim break; 350e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_X: 351e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_X: 352e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, src), ctx); 353e54bcde3SZi Shen Lim break; 354e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_X: 355e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_X: 356e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, src), ctx); 357e54bcde3SZi Shen Lim break; 358e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_X: 359e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_X: 360e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, src), ctx); 361e54bcde3SZi Shen Lim break; 362e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_X: 363e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_X: 364e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, src), ctx); 365e54bcde3SZi Shen Lim break; 366e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_X: 367e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_X: 368e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, src), ctx); 369e54bcde3SZi Shen Lim break; 370e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_X: 371e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_X: 372e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOD | BPF_X: 373e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOD | BPF_X: 374251599e1SZi Shen Lim { 375251599e1SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 376251599e1SZi Shen Lim 377251599e1SZi Shen Lim /* if (src == 0) return 0 */ 378251599e1SZi Shen Lim jmp_offset = 3; /* skip ahead to else path */ 379251599e1SZi Shen Lim check_imm19(jmp_offset); 380251599e1SZi Shen Lim emit(A64_CBNZ(is64, src, jmp_offset), ctx); 381251599e1SZi Shen Lim emit(A64_MOVZ(1, r0, 0, 0), ctx); 382251599e1SZi Shen Lim jmp_offset = epilogue_offset(ctx); 383251599e1SZi Shen Lim check_imm26(jmp_offset); 384251599e1SZi Shen Lim emit(A64_B(jmp_offset), ctx); 385251599e1SZi Shen Lim /* else */ 38614e589ffSZi Shen Lim switch (BPF_OP(code)) { 38714e589ffSZi Shen Lim case BPF_DIV: 388e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, src), ctx); 389e54bcde3SZi Shen Lim break; 39014e589ffSZi Shen Lim case BPF_MOD: 391e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, src), ctx); 392e54bcde3SZi Shen Lim emit(A64_MUL(is64, tmp, tmp, src), ctx); 393e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 394e54bcde3SZi Shen Lim break; 39514e589ffSZi Shen Lim } 39614e589ffSZi Shen Lim break; 39714e589ffSZi Shen Lim } 398d65a634aSZi Shen Lim case BPF_ALU | BPF_LSH | BPF_X: 399d65a634aSZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_X: 400d65a634aSZi Shen Lim emit(A64_LSLV(is64, dst, dst, src), ctx); 401d65a634aSZi Shen Lim break; 402d65a634aSZi Shen Lim case BPF_ALU | BPF_RSH | BPF_X: 403d65a634aSZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_X: 404d65a634aSZi Shen Lim emit(A64_LSRV(is64, dst, dst, src), ctx); 405d65a634aSZi Shen Lim break; 406d65a634aSZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_X: 407d65a634aSZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_X: 408d65a634aSZi Shen Lim emit(A64_ASRV(is64, dst, dst, src), ctx); 409d65a634aSZi Shen Lim break; 410e54bcde3SZi Shen Lim /* dst = -dst */ 411e54bcde3SZi Shen Lim case BPF_ALU | BPF_NEG: 412e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_NEG: 413e54bcde3SZi Shen Lim emit(A64_NEG(is64, dst, dst), ctx); 414e54bcde3SZi Shen Lim break; 415e54bcde3SZi Shen Lim /* dst = BSWAP##imm(dst) */ 416e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_LE: 417e54bcde3SZi Shen Lim case BPF_ALU | BPF_END | BPF_FROM_BE: 418e54bcde3SZi Shen Lim #ifdef CONFIG_CPU_BIG_ENDIAN 419e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_BE) 420d63903bbSXi Wang goto emit_bswap_uxt; 421e54bcde3SZi Shen Lim #else /* !CONFIG_CPU_BIG_ENDIAN */ 422e54bcde3SZi Shen Lim if (BPF_SRC(code) == BPF_FROM_LE) 423d63903bbSXi Wang goto emit_bswap_uxt; 424e54bcde3SZi Shen Lim #endif 425e54bcde3SZi Shen Lim switch (imm) { 426e54bcde3SZi Shen Lim case 16: 427e54bcde3SZi Shen Lim emit(A64_REV16(is64, dst, dst), ctx); 428d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 429d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 430e54bcde3SZi Shen Lim break; 431e54bcde3SZi Shen Lim case 32: 432e54bcde3SZi Shen Lim emit(A64_REV32(is64, dst, dst), ctx); 433d63903bbSXi Wang /* upper 32 bits already cleared */ 434e54bcde3SZi Shen Lim break; 435e54bcde3SZi Shen Lim case 64: 436e54bcde3SZi Shen Lim emit(A64_REV64(dst, dst), ctx); 437e54bcde3SZi Shen Lim break; 438e54bcde3SZi Shen Lim } 439e54bcde3SZi Shen Lim break; 440d63903bbSXi Wang emit_bswap_uxt: 441d63903bbSXi Wang switch (imm) { 442d63903bbSXi Wang case 16: 443d63903bbSXi Wang /* zero-extend 16 bits into 64 bits */ 444d63903bbSXi Wang emit(A64_UXTH(is64, dst, dst), ctx); 445d63903bbSXi Wang break; 446d63903bbSXi Wang case 32: 447d63903bbSXi Wang /* zero-extend 32 bits into 64 bits */ 448d63903bbSXi Wang emit(A64_UXTW(is64, dst, dst), ctx); 449d63903bbSXi Wang break; 450d63903bbSXi Wang case 64: 451d63903bbSXi Wang /* nop */ 452d63903bbSXi Wang break; 453d63903bbSXi Wang } 454d63903bbSXi Wang break; 455e54bcde3SZi Shen Lim /* dst = imm */ 456e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOV | BPF_K: 457e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOV | BPF_K: 458e54bcde3SZi Shen Lim emit_a64_mov_i(is64, dst, imm, ctx); 459e54bcde3SZi Shen Lim break; 460e54bcde3SZi Shen Lim /* dst = dst OP imm */ 461e54bcde3SZi Shen Lim case BPF_ALU | BPF_ADD | BPF_K: 462e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ADD | BPF_K: 463e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 464e54bcde3SZi Shen Lim emit(A64_ADD(is64, dst, dst, tmp), ctx); 465e54bcde3SZi Shen Lim break; 466e54bcde3SZi Shen Lim case BPF_ALU | BPF_SUB | BPF_K: 467e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_SUB | BPF_K: 468e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 469e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 470e54bcde3SZi Shen Lim break; 471e54bcde3SZi Shen Lim case BPF_ALU | BPF_AND | BPF_K: 472e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_AND | BPF_K: 473e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 474e54bcde3SZi Shen Lim emit(A64_AND(is64, dst, dst, tmp), ctx); 475e54bcde3SZi Shen Lim break; 476e54bcde3SZi Shen Lim case BPF_ALU | BPF_OR | BPF_K: 477e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_OR | BPF_K: 478e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 479e54bcde3SZi Shen Lim emit(A64_ORR(is64, dst, dst, tmp), ctx); 480e54bcde3SZi Shen Lim break; 481e54bcde3SZi Shen Lim case BPF_ALU | BPF_XOR | BPF_K: 482e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_XOR | BPF_K: 483e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 484e54bcde3SZi Shen Lim emit(A64_EOR(is64, dst, dst, tmp), ctx); 485e54bcde3SZi Shen Lim break; 486e54bcde3SZi Shen Lim case BPF_ALU | BPF_MUL | BPF_K: 487e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MUL | BPF_K: 488e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 489e54bcde3SZi Shen Lim emit(A64_MUL(is64, dst, dst, tmp), ctx); 490e54bcde3SZi Shen Lim break; 491e54bcde3SZi Shen Lim case BPF_ALU | BPF_DIV | BPF_K: 492e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_DIV | BPF_K: 493e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp, imm, ctx); 494e54bcde3SZi Shen Lim emit(A64_UDIV(is64, dst, dst, tmp), ctx); 495e54bcde3SZi Shen Lim break; 496e54bcde3SZi Shen Lim case BPF_ALU | BPF_MOD | BPF_K: 497e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_MOD | BPF_K: 498e54bcde3SZi Shen Lim emit_a64_mov_i(is64, tmp2, imm, ctx); 499e54bcde3SZi Shen Lim emit(A64_UDIV(is64, tmp, dst, tmp2), ctx); 500e54bcde3SZi Shen Lim emit(A64_MUL(is64, tmp, tmp, tmp2), ctx); 501e54bcde3SZi Shen Lim emit(A64_SUB(is64, dst, dst, tmp), ctx); 502e54bcde3SZi Shen Lim break; 503e54bcde3SZi Shen Lim case BPF_ALU | BPF_LSH | BPF_K: 504e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_LSH | BPF_K: 505e54bcde3SZi Shen Lim emit(A64_LSL(is64, dst, dst, imm), ctx); 506e54bcde3SZi Shen Lim break; 507e54bcde3SZi Shen Lim case BPF_ALU | BPF_RSH | BPF_K: 508e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_RSH | BPF_K: 509e54bcde3SZi Shen Lim emit(A64_LSR(is64, dst, dst, imm), ctx); 510e54bcde3SZi Shen Lim break; 511e54bcde3SZi Shen Lim case BPF_ALU | BPF_ARSH | BPF_K: 512e54bcde3SZi Shen Lim case BPF_ALU64 | BPF_ARSH | BPF_K: 513e54bcde3SZi Shen Lim emit(A64_ASR(is64, dst, dst, imm), ctx); 514e54bcde3SZi Shen Lim break; 515e54bcde3SZi Shen Lim 516e54bcde3SZi Shen Lim /* JUMP off */ 517e54bcde3SZi Shen Lim case BPF_JMP | BPF_JA: 518e54bcde3SZi Shen Lim jmp_offset = bpf2a64_offset(i + off, i, ctx); 519e54bcde3SZi Shen Lim check_imm26(jmp_offset); 520e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 521e54bcde3SZi Shen Lim break; 522e54bcde3SZi Shen Lim /* IF (dst COND src) JUMP off */ 523e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_X: 524e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_X: 525e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_X: 526e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_X: 527e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_X: 528e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_X: 529e54bcde3SZi Shen Lim emit(A64_CMP(1, dst, src), ctx); 530e54bcde3SZi Shen Lim emit_cond_jmp: 531e54bcde3SZi Shen Lim jmp_offset = bpf2a64_offset(i + off, i, ctx); 532e54bcde3SZi Shen Lim check_imm19(jmp_offset); 533e54bcde3SZi Shen Lim switch (BPF_OP(code)) { 534e54bcde3SZi Shen Lim case BPF_JEQ: 535e54bcde3SZi Shen Lim jmp_cond = A64_COND_EQ; 536e54bcde3SZi Shen Lim break; 537e54bcde3SZi Shen Lim case BPF_JGT: 538e54bcde3SZi Shen Lim jmp_cond = A64_COND_HI; 539e54bcde3SZi Shen Lim break; 540e54bcde3SZi Shen Lim case BPF_JGE: 541e54bcde3SZi Shen Lim jmp_cond = A64_COND_CS; 542e54bcde3SZi Shen Lim break; 54398397fc5SZi Shen Lim case BPF_JSET: 544e54bcde3SZi Shen Lim case BPF_JNE: 545e54bcde3SZi Shen Lim jmp_cond = A64_COND_NE; 546e54bcde3SZi Shen Lim break; 547e54bcde3SZi Shen Lim case BPF_JSGT: 548e54bcde3SZi Shen Lim jmp_cond = A64_COND_GT; 549e54bcde3SZi Shen Lim break; 550e54bcde3SZi Shen Lim case BPF_JSGE: 551e54bcde3SZi Shen Lim jmp_cond = A64_COND_GE; 552e54bcde3SZi Shen Lim break; 553e54bcde3SZi Shen Lim default: 554e54bcde3SZi Shen Lim return -EFAULT; 555e54bcde3SZi Shen Lim } 556e54bcde3SZi Shen Lim emit(A64_B_(jmp_cond, jmp_offset), ctx); 557e54bcde3SZi Shen Lim break; 558e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_X: 559e54bcde3SZi Shen Lim emit(A64_TST(1, dst, src), ctx); 560e54bcde3SZi Shen Lim goto emit_cond_jmp; 561e54bcde3SZi Shen Lim /* IF (dst COND imm) JUMP off */ 562e54bcde3SZi Shen Lim case BPF_JMP | BPF_JEQ | BPF_K: 563e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGT | BPF_K: 564e54bcde3SZi Shen Lim case BPF_JMP | BPF_JGE | BPF_K: 565e54bcde3SZi Shen Lim case BPF_JMP | BPF_JNE | BPF_K: 566e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGT | BPF_K: 567e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSGE | BPF_K: 568e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, imm, ctx); 569e54bcde3SZi Shen Lim emit(A64_CMP(1, dst, tmp), ctx); 570e54bcde3SZi Shen Lim goto emit_cond_jmp; 571e54bcde3SZi Shen Lim case BPF_JMP | BPF_JSET | BPF_K: 572e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, imm, ctx); 573e54bcde3SZi Shen Lim emit(A64_TST(1, dst, tmp), ctx); 574e54bcde3SZi Shen Lim goto emit_cond_jmp; 575e54bcde3SZi Shen Lim /* function call */ 576e54bcde3SZi Shen Lim case BPF_JMP | BPF_CALL: 577e54bcde3SZi Shen Lim { 578e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; 579e54bcde3SZi Shen Lim const u64 func = (u64)__bpf_call_base + imm; 580e54bcde3SZi Shen Lim 581e54bcde3SZi Shen Lim emit_a64_mov_i64(tmp, func, ctx); 582e54bcde3SZi Shen Lim emit(A64_BLR(tmp), ctx); 583e54bcde3SZi Shen Lim emit(A64_MOV(1, r0, A64_R(0)), ctx); 584e54bcde3SZi Shen Lim break; 585e54bcde3SZi Shen Lim } 586ddb55992SZi Shen Lim /* tail call */ 587ddb55992SZi Shen Lim case BPF_JMP | BPF_CALL | BPF_X: 588ddb55992SZi Shen Lim if (emit_bpf_tail_call(ctx)) 589ddb55992SZi Shen Lim return -EFAULT; 590ddb55992SZi Shen Lim break; 591e54bcde3SZi Shen Lim /* function return */ 592e54bcde3SZi Shen Lim case BPF_JMP | BPF_EXIT: 59351c9fbb1SZi Shen Lim /* Optimization: when last instruction is EXIT, 59451c9fbb1SZi Shen Lim simply fallthrough to epilogue. */ 595e54bcde3SZi Shen Lim if (i == ctx->prog->len - 1) 596e54bcde3SZi Shen Lim break; 597e54bcde3SZi Shen Lim jmp_offset = epilogue_offset(ctx); 598e54bcde3SZi Shen Lim check_imm26(jmp_offset); 599e54bcde3SZi Shen Lim emit(A64_B(jmp_offset), ctx); 600e54bcde3SZi Shen Lim break; 601e54bcde3SZi Shen Lim 60230d3d94cSZi Shen Lim /* dst = imm64 */ 60330d3d94cSZi Shen Lim case BPF_LD | BPF_IMM | BPF_DW: 60430d3d94cSZi Shen Lim { 60530d3d94cSZi Shen Lim const struct bpf_insn insn1 = insn[1]; 60630d3d94cSZi Shen Lim u64 imm64; 60730d3d94cSZi Shen Lim 6081e4df6b7SXi Wang imm64 = (u64)insn1.imm << 32 | (u32)imm; 60930d3d94cSZi Shen Lim emit_a64_mov_i64(dst, imm64, ctx); 61030d3d94cSZi Shen Lim 61130d3d94cSZi Shen Lim return 1; 61230d3d94cSZi Shen Lim } 61330d3d94cSZi Shen Lim 614e54bcde3SZi Shen Lim /* LDX: dst = *(size *)(src + off) */ 615e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_W: 616e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_H: 617e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_B: 618e54bcde3SZi Shen Lim case BPF_LDX | BPF_MEM | BPF_DW: 619e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 620e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 621e54bcde3SZi Shen Lim case BPF_W: 622e54bcde3SZi Shen Lim emit(A64_LDR32(dst, src, tmp), ctx); 623e54bcde3SZi Shen Lim break; 624e54bcde3SZi Shen Lim case BPF_H: 625e54bcde3SZi Shen Lim emit(A64_LDRH(dst, src, tmp), ctx); 626e54bcde3SZi Shen Lim break; 627e54bcde3SZi Shen Lim case BPF_B: 628e54bcde3SZi Shen Lim emit(A64_LDRB(dst, src, tmp), ctx); 629e54bcde3SZi Shen Lim break; 630e54bcde3SZi Shen Lim case BPF_DW: 631e54bcde3SZi Shen Lim emit(A64_LDR64(dst, src, tmp), ctx); 632e54bcde3SZi Shen Lim break; 633e54bcde3SZi Shen Lim } 634e54bcde3SZi Shen Lim break; 635e54bcde3SZi Shen Lim 636e54bcde3SZi Shen Lim /* ST: *(size *)(dst + off) = imm */ 637e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_W: 638e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_H: 639e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_B: 640e54bcde3SZi Shen Lim case BPF_ST | BPF_MEM | BPF_DW: 641df849ba3SYang Shi /* Load imm to a register then store it */ 642df849ba3SYang Shi emit_a64_mov_i(1, tmp2, off, ctx); 643df849ba3SYang Shi emit_a64_mov_i(1, tmp, imm, ctx); 644df849ba3SYang Shi switch (BPF_SIZE(code)) { 645df849ba3SYang Shi case BPF_W: 646df849ba3SYang Shi emit(A64_STR32(tmp, dst, tmp2), ctx); 647df849ba3SYang Shi break; 648df849ba3SYang Shi case BPF_H: 649df849ba3SYang Shi emit(A64_STRH(tmp, dst, tmp2), ctx); 650df849ba3SYang Shi break; 651df849ba3SYang Shi case BPF_B: 652df849ba3SYang Shi emit(A64_STRB(tmp, dst, tmp2), ctx); 653df849ba3SYang Shi break; 654df849ba3SYang Shi case BPF_DW: 655df849ba3SYang Shi emit(A64_STR64(tmp, dst, tmp2), ctx); 656df849ba3SYang Shi break; 657df849ba3SYang Shi } 658df849ba3SYang Shi break; 659e54bcde3SZi Shen Lim 660e54bcde3SZi Shen Lim /* STX: *(size *)(dst + off) = src */ 661e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_W: 662e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_H: 663e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_B: 664e54bcde3SZi Shen Lim case BPF_STX | BPF_MEM | BPF_DW: 665e54bcde3SZi Shen Lim emit_a64_mov_i(1, tmp, off, ctx); 666e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 667e54bcde3SZi Shen Lim case BPF_W: 668e54bcde3SZi Shen Lim emit(A64_STR32(src, dst, tmp), ctx); 669e54bcde3SZi Shen Lim break; 670e54bcde3SZi Shen Lim case BPF_H: 671e54bcde3SZi Shen Lim emit(A64_STRH(src, dst, tmp), ctx); 672e54bcde3SZi Shen Lim break; 673e54bcde3SZi Shen Lim case BPF_B: 674e54bcde3SZi Shen Lim emit(A64_STRB(src, dst, tmp), ctx); 675e54bcde3SZi Shen Lim break; 676e54bcde3SZi Shen Lim case BPF_DW: 677e54bcde3SZi Shen Lim emit(A64_STR64(src, dst, tmp), ctx); 678e54bcde3SZi Shen Lim break; 679e54bcde3SZi Shen Lim } 680e54bcde3SZi Shen Lim break; 681e54bcde3SZi Shen Lim /* STX XADD: lock *(u32 *)(dst + off) += src */ 682e54bcde3SZi Shen Lim case BPF_STX | BPF_XADD | BPF_W: 683e54bcde3SZi Shen Lim /* STX XADD: lock *(u64 *)(dst + off) += src */ 684e54bcde3SZi Shen Lim case BPF_STX | BPF_XADD | BPF_DW: 685*85f68fe8SDaniel Borkmann emit_a64_mov_i(1, tmp, off, ctx); 686*85f68fe8SDaniel Borkmann emit(A64_ADD(1, tmp, tmp, dst), ctx); 687*85f68fe8SDaniel Borkmann emit(A64_PRFM(tmp, PST, L1, STRM), ctx); 688*85f68fe8SDaniel Borkmann emit(A64_LDXR(isdw, tmp2, tmp), ctx); 689*85f68fe8SDaniel Borkmann emit(A64_ADD(isdw, tmp2, tmp2, src), ctx); 690*85f68fe8SDaniel Borkmann emit(A64_STXR(isdw, tmp2, tmp, tmp2), ctx); 691*85f68fe8SDaniel Borkmann jmp_offset = -3; 692*85f68fe8SDaniel Borkmann check_imm19(jmp_offset); 693*85f68fe8SDaniel Borkmann emit(A64_CBNZ(0, tmp2, jmp_offset), ctx); 694*85f68fe8SDaniel Borkmann break; 695e54bcde3SZi Shen Lim 696e54bcde3SZi Shen Lim /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */ 697e54bcde3SZi Shen Lim case BPF_LD | BPF_ABS | BPF_W: 698e54bcde3SZi Shen Lim case BPF_LD | BPF_ABS | BPF_H: 699e54bcde3SZi Shen Lim case BPF_LD | BPF_ABS | BPF_B: 700e54bcde3SZi Shen Lim /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */ 701e54bcde3SZi Shen Lim case BPF_LD | BPF_IND | BPF_W: 702e54bcde3SZi Shen Lim case BPF_LD | BPF_IND | BPF_H: 703e54bcde3SZi Shen Lim case BPF_LD | BPF_IND | BPF_B: 704e54bcde3SZi Shen Lim { 705e54bcde3SZi Shen Lim const u8 r0 = bpf2a64[BPF_REG_0]; /* r0 = return value */ 706e54bcde3SZi Shen Lim const u8 r6 = bpf2a64[BPF_REG_6]; /* r6 = pointer to sk_buff */ 707e54bcde3SZi Shen Lim const u8 fp = bpf2a64[BPF_REG_FP]; 708e54bcde3SZi Shen Lim const u8 r1 = bpf2a64[BPF_REG_1]; /* r1: struct sk_buff *skb */ 709e54bcde3SZi Shen Lim const u8 r2 = bpf2a64[BPF_REG_2]; /* r2: int k */ 710e54bcde3SZi Shen Lim const u8 r3 = bpf2a64[BPF_REG_3]; /* r3: unsigned int size */ 711e54bcde3SZi Shen Lim const u8 r4 = bpf2a64[BPF_REG_4]; /* r4: void *buffer */ 712e54bcde3SZi Shen Lim const u8 r5 = bpf2a64[BPF_REG_5]; /* r5: void *(*func)(...) */ 713e54bcde3SZi Shen Lim int size; 714e54bcde3SZi Shen Lim 715e54bcde3SZi Shen Lim emit(A64_MOV(1, r1, r6), ctx); 716e54bcde3SZi Shen Lim emit_a64_mov_i(0, r2, imm, ctx); 717e54bcde3SZi Shen Lim if (BPF_MODE(code) == BPF_IND) 718e54bcde3SZi Shen Lim emit(A64_ADD(0, r2, r2, src), ctx); 719e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 720e54bcde3SZi Shen Lim case BPF_W: 721e54bcde3SZi Shen Lim size = 4; 722e54bcde3SZi Shen Lim break; 723e54bcde3SZi Shen Lim case BPF_H: 724e54bcde3SZi Shen Lim size = 2; 725e54bcde3SZi Shen Lim break; 726e54bcde3SZi Shen Lim case BPF_B: 727e54bcde3SZi Shen Lim size = 1; 728e54bcde3SZi Shen Lim break; 729e54bcde3SZi Shen Lim default: 730e54bcde3SZi Shen Lim return -EINVAL; 731e54bcde3SZi Shen Lim } 732e54bcde3SZi Shen Lim emit_a64_mov_i64(r3, size, ctx); 733f4b16fceSZi Shen Lim emit(A64_SUB_I(1, r4, fp, STACK_SIZE), ctx); 734e54bcde3SZi Shen Lim emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx); 735e54bcde3SZi Shen Lim emit(A64_BLR(r5), ctx); 736e54bcde3SZi Shen Lim emit(A64_MOV(1, r0, A64_R(0)), ctx); 737e54bcde3SZi Shen Lim 738e54bcde3SZi Shen Lim jmp_offset = epilogue_offset(ctx); 739e54bcde3SZi Shen Lim check_imm19(jmp_offset); 740e54bcde3SZi Shen Lim emit(A64_CBZ(1, r0, jmp_offset), ctx); 741e54bcde3SZi Shen Lim emit(A64_MOV(1, r5, r0), ctx); 742e54bcde3SZi Shen Lim switch (BPF_SIZE(code)) { 743e54bcde3SZi Shen Lim case BPF_W: 744e54bcde3SZi Shen Lim emit(A64_LDR32(r0, r5, A64_ZR), ctx); 745e54bcde3SZi Shen Lim #ifndef CONFIG_CPU_BIG_ENDIAN 746e54bcde3SZi Shen Lim emit(A64_REV32(0, r0, r0), ctx); 747e54bcde3SZi Shen Lim #endif 748e54bcde3SZi Shen Lim break; 749e54bcde3SZi Shen Lim case BPF_H: 750e54bcde3SZi Shen Lim emit(A64_LDRH(r0, r5, A64_ZR), ctx); 751e54bcde3SZi Shen Lim #ifndef CONFIG_CPU_BIG_ENDIAN 752e54bcde3SZi Shen Lim emit(A64_REV16(0, r0, r0), ctx); 753e54bcde3SZi Shen Lim #endif 754e54bcde3SZi Shen Lim break; 755e54bcde3SZi Shen Lim case BPF_B: 756e54bcde3SZi Shen Lim emit(A64_LDRB(r0, r5, A64_ZR), ctx); 757e54bcde3SZi Shen Lim break; 758e54bcde3SZi Shen Lim } 759e54bcde3SZi Shen Lim break; 760e54bcde3SZi Shen Lim } 761e54bcde3SZi Shen Lim default: 762e54bcde3SZi Shen Lim pr_err_once("unknown opcode %02x\n", code); 763e54bcde3SZi Shen Lim return -EINVAL; 764e54bcde3SZi Shen Lim } 765e54bcde3SZi Shen Lim 766e54bcde3SZi Shen Lim return 0; 767e54bcde3SZi Shen Lim } 768e54bcde3SZi Shen Lim 769e54bcde3SZi Shen Lim static int build_body(struct jit_ctx *ctx) 770e54bcde3SZi Shen Lim { 771e54bcde3SZi Shen Lim const struct bpf_prog *prog = ctx->prog; 772e54bcde3SZi Shen Lim int i; 773e54bcde3SZi Shen Lim 774e54bcde3SZi Shen Lim for (i = 0; i < prog->len; i++) { 775e54bcde3SZi Shen Lim const struct bpf_insn *insn = &prog->insnsi[i]; 776e54bcde3SZi Shen Lim int ret; 777e54bcde3SZi Shen Lim 7788eee539dSXi Wang ret = build_insn(insn, ctx); 7798eee539dSXi Wang 780e54bcde3SZi Shen Lim if (ctx->image == NULL) 781e54bcde3SZi Shen Lim ctx->offset[i] = ctx->idx; 782e54bcde3SZi Shen Lim 78330d3d94cSZi Shen Lim if (ret > 0) { 78430d3d94cSZi Shen Lim i++; 78530d3d94cSZi Shen Lim continue; 78630d3d94cSZi Shen Lim } 787e54bcde3SZi Shen Lim if (ret) 788e54bcde3SZi Shen Lim return ret; 789e54bcde3SZi Shen Lim } 790e54bcde3SZi Shen Lim 791e54bcde3SZi Shen Lim return 0; 792e54bcde3SZi Shen Lim } 793e54bcde3SZi Shen Lim 79442ff712bSZi Shen Lim static int validate_code(struct jit_ctx *ctx) 79542ff712bSZi Shen Lim { 79642ff712bSZi Shen Lim int i; 79742ff712bSZi Shen Lim 79842ff712bSZi Shen Lim for (i = 0; i < ctx->idx; i++) { 79942ff712bSZi Shen Lim u32 a64_insn = le32_to_cpu(ctx->image[i]); 80042ff712bSZi Shen Lim 80142ff712bSZi Shen Lim if (a64_insn == AARCH64_BREAK_FAULT) 80242ff712bSZi Shen Lim return -1; 80342ff712bSZi Shen Lim } 80442ff712bSZi Shen Lim 80542ff712bSZi Shen Lim return 0; 80642ff712bSZi Shen Lim } 80742ff712bSZi Shen Lim 808e54bcde3SZi Shen Lim static inline void bpf_flush_icache(void *start, void *end) 809e54bcde3SZi Shen Lim { 810e54bcde3SZi Shen Lim flush_icache_range((unsigned long)start, (unsigned long)end); 811e54bcde3SZi Shen Lim } 812e54bcde3SZi Shen Lim 813d1c55ab5SDaniel Borkmann struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 814e54bcde3SZi Shen Lim { 81526eb042eSDaniel Borkmann struct bpf_prog *tmp, *orig_prog = prog; 816b569c1c6SDaniel Borkmann struct bpf_binary_header *header; 81726eb042eSDaniel Borkmann bool tmp_blinded = false; 818e54bcde3SZi Shen Lim struct jit_ctx ctx; 819e54bcde3SZi Shen Lim int image_size; 820b569c1c6SDaniel Borkmann u8 *image_ptr; 821e54bcde3SZi Shen Lim 822e54bcde3SZi Shen Lim if (!bpf_jit_enable) 82326eb042eSDaniel Borkmann return orig_prog; 82426eb042eSDaniel Borkmann 82526eb042eSDaniel Borkmann tmp = bpf_jit_blind_constants(prog); 82626eb042eSDaniel Borkmann /* If blinding was requested and we failed during blinding, 82726eb042eSDaniel Borkmann * we must fall back to the interpreter. 82826eb042eSDaniel Borkmann */ 82926eb042eSDaniel Borkmann if (IS_ERR(tmp)) 83026eb042eSDaniel Borkmann return orig_prog; 83126eb042eSDaniel Borkmann if (tmp != prog) { 83226eb042eSDaniel Borkmann tmp_blinded = true; 83326eb042eSDaniel Borkmann prog = tmp; 83426eb042eSDaniel Borkmann } 835e54bcde3SZi Shen Lim 836e54bcde3SZi Shen Lim memset(&ctx, 0, sizeof(ctx)); 837e54bcde3SZi Shen Lim ctx.prog = prog; 838e54bcde3SZi Shen Lim 839e54bcde3SZi Shen Lim ctx.offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); 84026eb042eSDaniel Borkmann if (ctx.offset == NULL) { 84126eb042eSDaniel Borkmann prog = orig_prog; 84226eb042eSDaniel Borkmann goto out; 84326eb042eSDaniel Borkmann } 844e54bcde3SZi Shen Lim 845e54bcde3SZi Shen Lim /* 1. Initial fake pass to compute ctx->idx. */ 846e54bcde3SZi Shen Lim 8474c1cd4fdSYang Shi /* Fake pass to fill in ctx->offset. */ 84826eb042eSDaniel Borkmann if (build_body(&ctx)) { 84926eb042eSDaniel Borkmann prog = orig_prog; 85026eb042eSDaniel Borkmann goto out_off; 85126eb042eSDaniel Borkmann } 852e54bcde3SZi Shen Lim 853ddb55992SZi Shen Lim if (build_prologue(&ctx)) { 854ddb55992SZi Shen Lim prog = orig_prog; 855ddb55992SZi Shen Lim goto out_off; 856ddb55992SZi Shen Lim } 85751c9fbb1SZi Shen Lim 85851c9fbb1SZi Shen Lim ctx.epilogue_offset = ctx.idx; 859e54bcde3SZi Shen Lim build_epilogue(&ctx); 860e54bcde3SZi Shen Lim 861e54bcde3SZi Shen Lim /* Now we know the actual image size. */ 862e54bcde3SZi Shen Lim image_size = sizeof(u32) * ctx.idx; 863b569c1c6SDaniel Borkmann header = bpf_jit_binary_alloc(image_size, &image_ptr, 864b569c1c6SDaniel Borkmann sizeof(u32), jit_fill_hole); 86526eb042eSDaniel Borkmann if (header == NULL) { 86626eb042eSDaniel Borkmann prog = orig_prog; 86726eb042eSDaniel Borkmann goto out_off; 86826eb042eSDaniel Borkmann } 869e54bcde3SZi Shen Lim 870e54bcde3SZi Shen Lim /* 2. Now, the actual pass. */ 871e54bcde3SZi Shen Lim 872b569c1c6SDaniel Borkmann ctx.image = (u32 *)image_ptr; 873e54bcde3SZi Shen Lim ctx.idx = 0; 874b569c1c6SDaniel Borkmann 875e54bcde3SZi Shen Lim build_prologue(&ctx); 876e54bcde3SZi Shen Lim 87760ef0494SDaniel Borkmann if (build_body(&ctx)) { 878b569c1c6SDaniel Borkmann bpf_jit_binary_free(header); 87926eb042eSDaniel Borkmann prog = orig_prog; 88026eb042eSDaniel Borkmann goto out_off; 88160ef0494SDaniel Borkmann } 882e54bcde3SZi Shen Lim 883e54bcde3SZi Shen Lim build_epilogue(&ctx); 884e54bcde3SZi Shen Lim 88542ff712bSZi Shen Lim /* 3. Extra pass to validate JITed code. */ 88642ff712bSZi Shen Lim if (validate_code(&ctx)) { 88742ff712bSZi Shen Lim bpf_jit_binary_free(header); 88826eb042eSDaniel Borkmann prog = orig_prog; 88926eb042eSDaniel Borkmann goto out_off; 89042ff712bSZi Shen Lim } 89142ff712bSZi Shen Lim 892e54bcde3SZi Shen Lim /* And we're done. */ 893e54bcde3SZi Shen Lim if (bpf_jit_enable > 1) 894e54bcde3SZi Shen Lim bpf_jit_dump(prog->len, image_size, 2, ctx.image); 895e54bcde3SZi Shen Lim 896c3d4c682SDaniel Borkmann bpf_flush_icache(header, ctx.image + ctx.idx); 897b569c1c6SDaniel Borkmann 8989d876e79SDaniel Borkmann bpf_jit_binary_lock_ro(header); 899e54bcde3SZi Shen Lim prog->bpf_func = (void *)ctx.image; 900a91263d5SDaniel Borkmann prog->jited = 1; 90126eb042eSDaniel Borkmann 90226eb042eSDaniel Borkmann out_off: 903e54bcde3SZi Shen Lim kfree(ctx.offset); 90426eb042eSDaniel Borkmann out: 90526eb042eSDaniel Borkmann if (tmp_blinded) 90626eb042eSDaniel Borkmann bpf_jit_prog_release_other(prog, prog == orig_prog ? 90726eb042eSDaniel Borkmann tmp : orig_prog); 908d1c55ab5SDaniel Borkmann return prog; 909e54bcde3SZi Shen Lim } 910