1b886d83cSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2156d0e29SNaveen N. Rao /* 3156d0e29SNaveen N. Rao * bpf_jit_comp64.c: eBPF JIT compiler 4156d0e29SNaveen N. Rao * 5156d0e29SNaveen N. Rao * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6156d0e29SNaveen N. Rao * IBM Corporation 7156d0e29SNaveen N. Rao * 8156d0e29SNaveen N. Rao * Based on the powerpc classic BPF JIT compiler by Matt Evans 9156d0e29SNaveen N. Rao */ 10156d0e29SNaveen N. Rao #include <linux/moduleloader.h> 11156d0e29SNaveen N. Rao #include <asm/cacheflush.h> 12ec0c464cSChristophe Leroy #include <asm/asm-compat.h> 13156d0e29SNaveen N. Rao #include <linux/netdevice.h> 14156d0e29SNaveen N. Rao #include <linux/filter.h> 15156d0e29SNaveen N. Rao #include <linux/if_vlan.h> 16156d0e29SNaveen N. Rao #include <asm/kprobes.h> 17ce076141SNaveen N. Rao #include <linux/bpf.h> 18156d0e29SNaveen N. Rao 19156d0e29SNaveen N. Rao #include "bpf_jit64.h" 20156d0e29SNaveen N. Rao 21156d0e29SNaveen N. Rao static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 22156d0e29SNaveen N. Rao { 236acdc9a6SNaveen N. Rao memset32(area, BREAKPOINT_INSTRUCTION, size/4); 24156d0e29SNaveen N. Rao } 25156d0e29SNaveen N. Rao 26156d0e29SNaveen N. Rao static inline void bpf_flush_icache(void *start, void *end) 27156d0e29SNaveen N. Rao { 28156d0e29SNaveen N. Rao smp_wmb(); 29156d0e29SNaveen N. Rao flush_icache_range((unsigned long)start, (unsigned long)end); 30156d0e29SNaveen N. Rao } 31156d0e29SNaveen N. Rao 32156d0e29SNaveen N. Rao static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) 33156d0e29SNaveen N. Rao { 34*ed573b57SChristophe Leroy return ctx->seen & (1 << (31 - i)); 35156d0e29SNaveen N. Rao } 36156d0e29SNaveen N. Rao 37156d0e29SNaveen N. Rao static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) 38156d0e29SNaveen N. Rao { 39*ed573b57SChristophe Leroy ctx->seen |= 1 << (31 - i); 40156d0e29SNaveen N. Rao } 41156d0e29SNaveen N. Rao 42156d0e29SNaveen N. Rao static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 43156d0e29SNaveen N. Rao { 44156d0e29SNaveen N. Rao /* 45156d0e29SNaveen N. Rao * We only need a stack frame if: 46156d0e29SNaveen N. Rao * - we call other functions (kernel helpers), or 47156d0e29SNaveen N. Rao * - the bpf program uses its stack area 48156d0e29SNaveen N. Rao * The latter condition is deduced from the usage of BPF_REG_FP 49156d0e29SNaveen N. Rao */ 50*ed573b57SChristophe Leroy return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, b2p[BPF_REG_FP]); 51156d0e29SNaveen N. Rao } 52156d0e29SNaveen N. Rao 537b847f52SNaveen N. Rao /* 547b847f52SNaveen N. Rao * When not setting up our own stackframe, the redzone usage is: 557b847f52SNaveen N. Rao * 567b847f52SNaveen N. Rao * [ prev sp ] <------------- 577b847f52SNaveen N. Rao * [ ... ] | 587b847f52SNaveen N. Rao * sp (r1) ---> [ stack pointer ] -------------- 59dbf44dafSDaniel Borkmann * [ nv gpr save area ] 6*8 607b847f52SNaveen N. Rao * [ tail_call_cnt ] 8 617b847f52SNaveen N. Rao * [ local_tmp_var ] 8 627b847f52SNaveen N. Rao * [ unused red zone ] 208 bytes protected 637b847f52SNaveen N. Rao */ 647b847f52SNaveen N. Rao static int bpf_jit_stack_local(struct codegen_context *ctx) 657b847f52SNaveen N. Rao { 667b847f52SNaveen N. Rao if (bpf_has_stack_frame(ctx)) 67ac0761ebSSandipan Das return STACK_FRAME_MIN_SIZE + ctx->stack_size; 687b847f52SNaveen N. Rao else 697b847f52SNaveen N. Rao return -(BPF_PPC_STACK_SAVE + 16); 707b847f52SNaveen N. Rao } 717b847f52SNaveen N. Rao 72ce076141SNaveen N. Rao static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) 73ce076141SNaveen N. Rao { 74ce076141SNaveen N. Rao return bpf_jit_stack_local(ctx) + 8; 75ce076141SNaveen N. Rao } 76ce076141SNaveen N. Rao 777b847f52SNaveen N. Rao static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 787b847f52SNaveen N. Rao { 797b847f52SNaveen N. Rao if (reg >= BPF_PPC_NVR_MIN && reg < 32) 80ac0761ebSSandipan Das return (bpf_has_stack_frame(ctx) ? 81ac0761ebSSandipan Das (BPF_PPC_STACKFRAME + ctx->stack_size) : 0) 827b847f52SNaveen N. Rao - (8 * (32 - reg)); 837b847f52SNaveen N. Rao 847b847f52SNaveen N. Rao pr_err("BPF JIT is asking about unknown registers"); 857b847f52SNaveen N. Rao BUG(); 867b847f52SNaveen N. Rao } 877b847f52SNaveen N. Rao 88156d0e29SNaveen N. Rao static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 89156d0e29SNaveen N. Rao { 90156d0e29SNaveen N. Rao int i; 91156d0e29SNaveen N. Rao 92ce076141SNaveen N. Rao /* 93ce076141SNaveen N. Rao * Initialize tail_call_cnt if we do tail calls. 94ce076141SNaveen N. Rao * Otherwise, put in NOPs so that it can be skipped when we are 95ce076141SNaveen N. Rao * invoked through a tail call. 96ce076141SNaveen N. Rao */ 97ce076141SNaveen N. Rao if (ctx->seen & SEEN_TAILCALL) { 983a181237SBalamuruhan S EMIT(PPC_RAW_LI(b2p[TMP_REG_1], 0)); 99ce076141SNaveen N. Rao /* this goes in the redzone */ 100ce076141SNaveen N. Rao PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8)); 101ce076141SNaveen N. Rao } else { 1023a181237SBalamuruhan S EMIT(PPC_RAW_NOP()); 1033a181237SBalamuruhan S EMIT(PPC_RAW_NOP()); 104ce076141SNaveen N. Rao } 105ce076141SNaveen N. Rao 106ce076141SNaveen N. Rao #define BPF_TAILCALL_PROLOGUE_SIZE 8 107ce076141SNaveen N. Rao 1087b847f52SNaveen N. Rao if (bpf_has_stack_frame(ctx)) { 109156d0e29SNaveen N. Rao /* 110156d0e29SNaveen N. Rao * We need a stack frame, but we don't necessarily need to 111156d0e29SNaveen N. Rao * save/restore LR unless we call other functions 112156d0e29SNaveen N. Rao */ 113156d0e29SNaveen N. Rao if (ctx->seen & SEEN_FUNC) { 114156d0e29SNaveen N. Rao EMIT(PPC_INST_MFLR | __PPC_RT(R0)); 115156d0e29SNaveen N. Rao PPC_BPF_STL(0, 1, PPC_LR_STKOFF); 116156d0e29SNaveen N. Rao } 117156d0e29SNaveen N. Rao 118ac0761ebSSandipan Das PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size)); 119156d0e29SNaveen N. Rao } 120156d0e29SNaveen N. Rao 121156d0e29SNaveen N. Rao /* 122156d0e29SNaveen N. Rao * Back up non-volatile regs -- BPF registers 6-10 123156d0e29SNaveen N. Rao * If we haven't created our own stack frame, we save these 124156d0e29SNaveen N. Rao * in the protected zone below the previous stack frame 125156d0e29SNaveen N. Rao */ 126156d0e29SNaveen N. Rao for (i = BPF_REG_6; i <= BPF_REG_10; i++) 127*ed573b57SChristophe Leroy if (bpf_is_seen_register(ctx, b2p[i])) 1287b847f52SNaveen N. Rao PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 129156d0e29SNaveen N. Rao 130156d0e29SNaveen N. Rao /* Setup frame pointer to point to the bpf stack area */ 131*ed573b57SChristophe Leroy if (bpf_is_seen_register(ctx, b2p[BPF_REG_FP])) 1323a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1, 1333a181237SBalamuruhan S STACK_FRAME_MIN_SIZE + ctx->stack_size)); 134156d0e29SNaveen N. Rao } 135156d0e29SNaveen N. Rao 136ce076141SNaveen N. Rao static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 137156d0e29SNaveen N. Rao { 138156d0e29SNaveen N. Rao int i; 139156d0e29SNaveen N. Rao 140156d0e29SNaveen N. Rao /* Restore NVRs */ 141156d0e29SNaveen N. Rao for (i = BPF_REG_6; i <= BPF_REG_10; i++) 142*ed573b57SChristophe Leroy if (bpf_is_seen_register(ctx, b2p[i])) 1437b847f52SNaveen N. Rao PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 144156d0e29SNaveen N. Rao 145156d0e29SNaveen N. Rao /* Tear down our stack frame */ 1467b847f52SNaveen N. Rao if (bpf_has_stack_frame(ctx)) { 1473a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size)); 148156d0e29SNaveen N. Rao if (ctx->seen & SEEN_FUNC) { 149156d0e29SNaveen N. Rao PPC_BPF_LL(0, 1, PPC_LR_STKOFF); 1503a181237SBalamuruhan S EMIT(PPC_RAW_MTLR(0)); 151156d0e29SNaveen N. Rao } 152156d0e29SNaveen N. Rao } 153ce076141SNaveen N. Rao } 154ce076141SNaveen N. Rao 155ce076141SNaveen N. Rao static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 156ce076141SNaveen N. Rao { 157ce076141SNaveen N. Rao bpf_jit_emit_common_epilogue(image, ctx); 158ce076141SNaveen N. Rao 159ce076141SNaveen N. Rao /* Move result to r3 */ 1603a181237SBalamuruhan S EMIT(PPC_RAW_MR(3, b2p[BPF_REG_0])); 161156d0e29SNaveen N. Rao 1623a181237SBalamuruhan S EMIT(PPC_RAW_BLR()); 163156d0e29SNaveen N. Rao } 164156d0e29SNaveen N. Rao 165e2c95a61SDaniel Borkmann static void bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, 166e2c95a61SDaniel Borkmann u64 func) 167e2c95a61SDaniel Borkmann { 168e2c95a61SDaniel Borkmann #ifdef PPC64_ELF_ABI_v1 169e2c95a61SDaniel Borkmann /* func points to the function descriptor */ 170e2c95a61SDaniel Borkmann PPC_LI64(b2p[TMP_REG_2], func); 171e2c95a61SDaniel Borkmann /* Load actual entry point from function descriptor */ 172e2c95a61SDaniel Borkmann PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0); 173e2c95a61SDaniel Borkmann /* ... and move it to LR */ 1743a181237SBalamuruhan S EMIT(PPC_RAW_MTLR(b2p[TMP_REG_1])); 175e2c95a61SDaniel Borkmann /* 176e2c95a61SDaniel Borkmann * Load TOC from function descriptor at offset 8. 177e2c95a61SDaniel Borkmann * We can clobber r2 since we get called through a 178e2c95a61SDaniel Borkmann * function pointer (so caller will save/restore r2) 179e2c95a61SDaniel Borkmann * and since we don't use a TOC ourself. 180e2c95a61SDaniel Borkmann */ 181e2c95a61SDaniel Borkmann PPC_BPF_LL(2, b2p[TMP_REG_2], 8); 182e2c95a61SDaniel Borkmann #else 183e2c95a61SDaniel Borkmann /* We can clobber r12 */ 184e2c95a61SDaniel Borkmann PPC_FUNC_ADDR(12, func); 1853a181237SBalamuruhan S EMIT(PPC_RAW_MTLR(12)); 186e2c95a61SDaniel Borkmann #endif 1873a181237SBalamuruhan S EMIT(PPC_RAW_BLRL()); 188e2c95a61SDaniel Borkmann } 189e2c95a61SDaniel Borkmann 190e2c95a61SDaniel Borkmann static void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, 191e2c95a61SDaniel Borkmann u64 func) 192ce076141SNaveen N. Rao { 1934ea69b2fSSandipan Das unsigned int i, ctx_idx = ctx->idx; 1944ea69b2fSSandipan Das 1954ea69b2fSSandipan Das /* Load function address into r12 */ 1964ea69b2fSSandipan Das PPC_LI64(12, func); 1974ea69b2fSSandipan Das 1984ea69b2fSSandipan Das /* For bpf-to-bpf function calls, the callee's address is unknown 1994ea69b2fSSandipan Das * until the last extra pass. As seen above, we use PPC_LI64() to 2004ea69b2fSSandipan Das * load the callee's address, but this may optimize the number of 2014ea69b2fSSandipan Das * instructions required based on the nature of the address. 2024ea69b2fSSandipan Das * 2034ea69b2fSSandipan Das * Since we don't want the number of instructions emitted to change, 2044ea69b2fSSandipan Das * we pad the optimized PPC_LI64() call with NOPs to guarantee that 2054ea69b2fSSandipan Das * we always have a five-instruction sequence, which is the maximum 2064ea69b2fSSandipan Das * that PPC_LI64() can emit. 2074ea69b2fSSandipan Das */ 2084ea69b2fSSandipan Das for (i = ctx->idx - ctx_idx; i < 5; i++) 2093a181237SBalamuruhan S EMIT(PPC_RAW_NOP()); 2104ea69b2fSSandipan Das 211ce076141SNaveen N. Rao #ifdef PPC64_ELF_ABI_v1 212ce076141SNaveen N. Rao /* 213ce076141SNaveen N. Rao * Load TOC from function descriptor at offset 8. 214ce076141SNaveen N. Rao * We can clobber r2 since we get called through a 215ce076141SNaveen N. Rao * function pointer (so caller will save/restore r2) 216ce076141SNaveen N. Rao * and since we don't use a TOC ourself. 217ce076141SNaveen N. Rao */ 2184ea69b2fSSandipan Das PPC_BPF_LL(2, 12, 8); 2194ea69b2fSSandipan Das /* Load actual entry point from function descriptor */ 2204ea69b2fSSandipan Das PPC_BPF_LL(12, 12, 0); 221ce076141SNaveen N. Rao #endif 2224ea69b2fSSandipan Das 2233a181237SBalamuruhan S EMIT(PPC_RAW_MTLR(12)); 2243a181237SBalamuruhan S EMIT(PPC_RAW_BLRL()); 225ce076141SNaveen N. Rao } 226ce076141SNaveen N. Rao 227ce076141SNaveen N. Rao static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 228ce076141SNaveen N. Rao { 229ce076141SNaveen N. Rao /* 230ce076141SNaveen N. Rao * By now, the eBPF program has already setup parameters in r3, r4 and r5 231ce076141SNaveen N. Rao * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 232ce076141SNaveen N. Rao * r4/BPF_REG_2 - pointer to bpf_array 233ce076141SNaveen N. Rao * r5/BPF_REG_3 - index in bpf_array 234ce076141SNaveen N. Rao */ 235ce076141SNaveen N. Rao int b2p_bpf_array = b2p[BPF_REG_2]; 236ce076141SNaveen N. Rao int b2p_index = b2p[BPF_REG_3]; 237ce076141SNaveen N. Rao 238ce076141SNaveen N. Rao /* 239ce076141SNaveen N. Rao * if (index >= array->map.max_entries) 240ce076141SNaveen N. Rao * goto out; 241ce076141SNaveen N. Rao */ 24206541865SBalamuruhan S EMIT(PPC_RAW_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries))); 2433a181237SBalamuruhan S EMIT(PPC_RAW_RLWINM(b2p_index, b2p_index, 0, 0, 31)); 2443a181237SBalamuruhan S EMIT(PPC_RAW_CMPLW(b2p_index, b2p[TMP_REG_1])); 245ce076141SNaveen N. Rao PPC_BCC(COND_GE, out); 246ce076141SNaveen N. Rao 247ce076141SNaveen N. Rao /* 248ce076141SNaveen N. Rao * if (tail_call_cnt > MAX_TAIL_CALL_CNT) 249ce076141SNaveen N. Rao * goto out; 250ce076141SNaveen N. Rao */ 25186be36f6SNaveen N. Rao PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 2523a181237SBalamuruhan S EMIT(PPC_RAW_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT)); 253ce076141SNaveen N. Rao PPC_BCC(COND_GT, out); 254ce076141SNaveen N. Rao 255ce076141SNaveen N. Rao /* 256ce076141SNaveen N. Rao * tail_call_cnt++; 257ce076141SNaveen N. Rao */ 2583a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1)); 259ce076141SNaveen N. Rao PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 260ce076141SNaveen N. Rao 261ce076141SNaveen N. Rao /* prog = array->ptrs[index]; */ 2623a181237SBalamuruhan S EMIT(PPC_RAW_MULI(b2p[TMP_REG_1], b2p_index, 8)); 26306541865SBalamuruhan S EMIT(PPC_RAW_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array)); 26486be36f6SNaveen N. Rao PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); 265ce076141SNaveen N. Rao 266ce076141SNaveen N. Rao /* 267ce076141SNaveen N. Rao * if (prog == NULL) 268ce076141SNaveen N. Rao * goto out; 269ce076141SNaveen N. Rao */ 2703a181237SBalamuruhan S EMIT(PPC_RAW_CMPLDI(b2p[TMP_REG_1], 0)); 271ce076141SNaveen N. Rao PPC_BCC(COND_EQ, out); 272ce076141SNaveen N. Rao 273ce076141SNaveen N. Rao /* goto *(prog->bpf_func + prologue_size); */ 27486be36f6SNaveen N. Rao PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); 275ce076141SNaveen N. Rao #ifdef PPC64_ELF_ABI_v1 276ce076141SNaveen N. Rao /* skip past the function descriptor */ 2773a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 2783a181237SBalamuruhan S FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE)); 279ce076141SNaveen N. Rao #else 2803a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE)); 281ce076141SNaveen N. Rao #endif 2823a181237SBalamuruhan S EMIT(PPC_RAW_MTCTR(b2p[TMP_REG_1])); 283ce076141SNaveen N. Rao 284ce076141SNaveen N. Rao /* tear down stack, restore NVRs, ... */ 285ce076141SNaveen N. Rao bpf_jit_emit_common_epilogue(image, ctx); 286ce076141SNaveen N. Rao 2873a181237SBalamuruhan S EMIT(PPC_RAW_BCTR()); 288ce076141SNaveen N. Rao /* out: */ 289ce076141SNaveen N. Rao } 290ce076141SNaveen N. Rao 291156d0e29SNaveen N. Rao /* Assemble the body code between the prologue & epilogue */ 292156d0e29SNaveen N. Rao static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 293156d0e29SNaveen N. Rao struct codegen_context *ctx, 2948484ce83SSandipan Das u32 *addrs, bool extra_pass) 295156d0e29SNaveen N. Rao { 296156d0e29SNaveen N. Rao const struct bpf_insn *insn = fp->insnsi; 297156d0e29SNaveen N. Rao int flen = fp->len; 298e2c95a61SDaniel Borkmann int i, ret; 299156d0e29SNaveen N. Rao 300156d0e29SNaveen N. Rao /* Start of epilogue code - will only be valid 2nd pass onwards */ 301156d0e29SNaveen N. Rao u32 exit_addr = addrs[flen]; 302156d0e29SNaveen N. Rao 303156d0e29SNaveen N. Rao for (i = 0; i < flen; i++) { 304156d0e29SNaveen N. Rao u32 code = insn[i].code; 305156d0e29SNaveen N. Rao u32 dst_reg = b2p[insn[i].dst_reg]; 306156d0e29SNaveen N. Rao u32 src_reg = b2p[insn[i].src_reg]; 307156d0e29SNaveen N. Rao s16 off = insn[i].off; 308156d0e29SNaveen N. Rao s32 imm = insn[i].imm; 309e2c95a61SDaniel Borkmann bool func_addr_fixed; 310e2c95a61SDaniel Borkmann u64 func_addr; 311156d0e29SNaveen N. Rao u64 imm64; 312156d0e29SNaveen N. Rao u32 true_cond; 313b9c1e60eSDaniel Borkmann u32 tmp_idx; 314156d0e29SNaveen N. Rao 315156d0e29SNaveen N. Rao /* 316156d0e29SNaveen N. Rao * addrs[] maps a BPF bytecode address into a real offset from 317156d0e29SNaveen N. Rao * the start of the body code. 318156d0e29SNaveen N. Rao */ 319156d0e29SNaveen N. Rao addrs[i] = ctx->idx * 4; 320156d0e29SNaveen N. Rao 321156d0e29SNaveen N. Rao /* 322156d0e29SNaveen N. Rao * As an optimization, we note down which non-volatile registers 323156d0e29SNaveen N. Rao * are used so that we can only save/restore those in our 324156d0e29SNaveen N. Rao * prologue and epilogue. We do this here regardless of whether 325156d0e29SNaveen N. Rao * the actual BPF instruction uses src/dst registers or not 326156d0e29SNaveen N. Rao * (for instance, BPF_CALL does not use them). The expectation 327156d0e29SNaveen N. Rao * is that those instructions will have src_reg/dst_reg set to 328156d0e29SNaveen N. Rao * 0. Even otherwise, we just lose some prologue/epilogue 329156d0e29SNaveen N. Rao * optimization but everything else should work without 330156d0e29SNaveen N. Rao * any issues. 331156d0e29SNaveen N. Rao */ 3327b847f52SNaveen N. Rao if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32) 333*ed573b57SChristophe Leroy bpf_set_seen_register(ctx, dst_reg); 3347b847f52SNaveen N. Rao if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32) 335*ed573b57SChristophe Leroy bpf_set_seen_register(ctx, src_reg); 336156d0e29SNaveen N. Rao 337156d0e29SNaveen N. Rao switch (code) { 338156d0e29SNaveen N. Rao /* 339156d0e29SNaveen N. Rao * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 340156d0e29SNaveen N. Rao */ 341156d0e29SNaveen N. Rao case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 342156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 34306541865SBalamuruhan S EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg)); 344156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 345156d0e29SNaveen N. Rao case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 346156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 3473a181237SBalamuruhan S EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg)); 348156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 349156d0e29SNaveen N. Rao case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 350156d0e29SNaveen N. Rao case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 351156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 352156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 353156d0e29SNaveen N. Rao if (BPF_OP(code) == BPF_SUB) 354156d0e29SNaveen N. Rao imm = -imm; 355156d0e29SNaveen N. Rao if (imm) { 356156d0e29SNaveen N. Rao if (imm >= -32768 && imm < 32768) 3573a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm))); 358156d0e29SNaveen N. Rao else { 359156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 36006541865SBalamuruhan S EMIT(PPC_RAW_ADD(dst_reg, dst_reg, b2p[TMP_REG_1])); 361156d0e29SNaveen N. Rao } 362156d0e29SNaveen N. Rao } 363156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 364156d0e29SNaveen N. Rao case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 365156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 366156d0e29SNaveen N. Rao if (BPF_CLASS(code) == BPF_ALU) 3673a181237SBalamuruhan S EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg)); 368156d0e29SNaveen N. Rao else 3693a181237SBalamuruhan S EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg)); 370156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 371156d0e29SNaveen N. Rao case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 372156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 373156d0e29SNaveen N. Rao if (imm >= -32768 && imm < 32768) 3743a181237SBalamuruhan S EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm))); 375156d0e29SNaveen N. Rao else { 376156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 377156d0e29SNaveen N. Rao if (BPF_CLASS(code) == BPF_ALU) 3783a181237SBalamuruhan S EMIT(PPC_RAW_MULW(dst_reg, dst_reg, 3793a181237SBalamuruhan S b2p[TMP_REG_1])); 380156d0e29SNaveen N. Rao else 3813a181237SBalamuruhan S EMIT(PPC_RAW_MULD(dst_reg, dst_reg, 3823a181237SBalamuruhan S b2p[TMP_REG_1])); 383156d0e29SNaveen N. Rao } 384156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 385156d0e29SNaveen N. Rao case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 386156d0e29SNaveen N. Rao case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 387156d0e29SNaveen N. Rao if (BPF_OP(code) == BPF_MOD) { 3883a181237SBalamuruhan S EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg)); 3893a181237SBalamuruhan S EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], src_reg, 3903a181237SBalamuruhan S b2p[TMP_REG_1])); 3913a181237SBalamuruhan S EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1])); 392156d0e29SNaveen N. Rao } else 3933a181237SBalamuruhan S EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg)); 394156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 395156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 396156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 397156d0e29SNaveen N. Rao if (BPF_OP(code) == BPF_MOD) { 3983a181237SBalamuruhan S EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg)); 3993a181237SBalamuruhan S EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], src_reg, 4003a181237SBalamuruhan S b2p[TMP_REG_1])); 4013a181237SBalamuruhan S EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1])); 402156d0e29SNaveen N. Rao } else 4033a181237SBalamuruhan S EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg)); 404156d0e29SNaveen N. Rao break; 405156d0e29SNaveen N. Rao case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 406156d0e29SNaveen N. Rao case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 407156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 408156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 409156d0e29SNaveen N. Rao if (imm == 0) 410156d0e29SNaveen N. Rao return -EINVAL; 411156d0e29SNaveen N. Rao else if (imm == 1) 412156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 413156d0e29SNaveen N. Rao 414156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 415156d0e29SNaveen N. Rao switch (BPF_CLASS(code)) { 416156d0e29SNaveen N. Rao case BPF_ALU: 417156d0e29SNaveen N. Rao if (BPF_OP(code) == BPF_MOD) { 4183a181237SBalamuruhan S EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_2], 4193a181237SBalamuruhan S dst_reg, 4203a181237SBalamuruhan S b2p[TMP_REG_1])); 4213a181237SBalamuruhan S EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], 422156d0e29SNaveen N. Rao b2p[TMP_REG_1], 4233a181237SBalamuruhan S b2p[TMP_REG_2])); 4243a181237SBalamuruhan S EMIT(PPC_RAW_SUB(dst_reg, dst_reg, 4253a181237SBalamuruhan S b2p[TMP_REG_1])); 426156d0e29SNaveen N. Rao } else 4273a181237SBalamuruhan S EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, 4283a181237SBalamuruhan S b2p[TMP_REG_1])); 429156d0e29SNaveen N. Rao break; 430156d0e29SNaveen N. Rao case BPF_ALU64: 431156d0e29SNaveen N. Rao if (BPF_OP(code) == BPF_MOD) { 4323a181237SBalamuruhan S EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_2], 4333a181237SBalamuruhan S dst_reg, 4343a181237SBalamuruhan S b2p[TMP_REG_1])); 4353a181237SBalamuruhan S EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], 436156d0e29SNaveen N. Rao b2p[TMP_REG_1], 4373a181237SBalamuruhan S b2p[TMP_REG_2])); 4383a181237SBalamuruhan S EMIT(PPC_RAW_SUB(dst_reg, dst_reg, 4393a181237SBalamuruhan S b2p[TMP_REG_1])); 440156d0e29SNaveen N. Rao } else 4413a181237SBalamuruhan S EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, 4423a181237SBalamuruhan S b2p[TMP_REG_1])); 443156d0e29SNaveen N. Rao break; 444156d0e29SNaveen N. Rao } 445156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 446156d0e29SNaveen N. Rao case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 447156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 4483a181237SBalamuruhan S EMIT(PPC_RAW_NEG(dst_reg, dst_reg)); 449156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 450156d0e29SNaveen N. Rao 451156d0e29SNaveen N. Rao /* 452156d0e29SNaveen N. Rao * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 453156d0e29SNaveen N. Rao */ 454156d0e29SNaveen N. Rao case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 455156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 4563a181237SBalamuruhan S EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg)); 457156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 458156d0e29SNaveen N. Rao case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 459156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 460156d0e29SNaveen N. Rao if (!IMM_H(imm)) 4613a181237SBalamuruhan S EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm))); 462156d0e29SNaveen N. Rao else { 463156d0e29SNaveen N. Rao /* Sign-extended */ 464156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 4653a181237SBalamuruhan S EMIT(PPC_RAW_AND(dst_reg, dst_reg, b2p[TMP_REG_1])); 466156d0e29SNaveen N. Rao } 467156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 468156d0e29SNaveen N. Rao case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 469156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 4703a181237SBalamuruhan S EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg)); 471156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 472156d0e29SNaveen N. Rao case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 473156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 474156d0e29SNaveen N. Rao if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 475156d0e29SNaveen N. Rao /* Sign-extended */ 476156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 4773a181237SBalamuruhan S EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_1])); 478156d0e29SNaveen N. Rao } else { 479156d0e29SNaveen N. Rao if (IMM_L(imm)) 4803a181237SBalamuruhan S EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm))); 481156d0e29SNaveen N. Rao if (IMM_H(imm)) 4823a181237SBalamuruhan S EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm))); 483156d0e29SNaveen N. Rao } 484156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 485156d0e29SNaveen N. Rao case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 486156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 4873a181237SBalamuruhan S EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg)); 488156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 489156d0e29SNaveen N. Rao case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 490156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 491156d0e29SNaveen N. Rao if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 492156d0e29SNaveen N. Rao /* Sign-extended */ 493156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 4943a181237SBalamuruhan S EMIT(PPC_RAW_XOR(dst_reg, dst_reg, b2p[TMP_REG_1])); 495156d0e29SNaveen N. Rao } else { 496156d0e29SNaveen N. Rao if (IMM_L(imm)) 4973a181237SBalamuruhan S EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm))); 498156d0e29SNaveen N. Rao if (IMM_H(imm)) 4993a181237SBalamuruhan S EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm))); 500156d0e29SNaveen N. Rao } 501156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 502156d0e29SNaveen N. Rao case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 503156d0e29SNaveen N. Rao /* slw clears top 32 bits */ 5043a181237SBalamuruhan S EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg)); 505a4c92773SJiong Wang /* skip zero extension move, but set address map. */ 506a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 507a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 508156d0e29SNaveen N. Rao break; 509156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 5103a181237SBalamuruhan S EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg)); 511156d0e29SNaveen N. Rao break; 512156d0e29SNaveen N. Rao case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */ 513156d0e29SNaveen N. Rao /* with imm 0, we still need to clear top 32 bits */ 5143a181237SBalamuruhan S EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm)); 515a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 516a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 517156d0e29SNaveen N. Rao break; 518156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */ 519156d0e29SNaveen N. Rao if (imm != 0) 5203a181237SBalamuruhan S EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm)); 521156d0e29SNaveen N. Rao break; 522156d0e29SNaveen N. Rao case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 5233a181237SBalamuruhan S EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 524a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 525a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 526156d0e29SNaveen N. Rao break; 527156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 5283a181237SBalamuruhan S EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg)); 529156d0e29SNaveen N. Rao break; 530156d0e29SNaveen N. Rao case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 5313a181237SBalamuruhan S EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm)); 532a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 533a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 534156d0e29SNaveen N. Rao break; 535156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 536156d0e29SNaveen N. Rao if (imm != 0) 5373a181237SBalamuruhan S EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm)); 538156d0e29SNaveen N. Rao break; 53944cf43c0SJiong Wang case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ 5403a181237SBalamuruhan S EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg)); 54144cf43c0SJiong Wang goto bpf_alu32_trunc; 542156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 5433a181237SBalamuruhan S EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg)); 544156d0e29SNaveen N. Rao break; 54544cf43c0SJiong Wang case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ 5463a181237SBalamuruhan S EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm)); 54744cf43c0SJiong Wang goto bpf_alu32_trunc; 548156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 549156d0e29SNaveen N. Rao if (imm != 0) 5503a181237SBalamuruhan S EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm)); 551156d0e29SNaveen N. Rao break; 552156d0e29SNaveen N. Rao 553156d0e29SNaveen N. Rao /* 554156d0e29SNaveen N. Rao * MOV 555156d0e29SNaveen N. Rao */ 556156d0e29SNaveen N. Rao case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 557156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 558a4c92773SJiong Wang if (imm == 1) { 559a4c92773SJiong Wang /* special mov32 for zext */ 5603a181237SBalamuruhan S EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31)); 561a4c92773SJiong Wang break; 562a4c92773SJiong Wang } 5633a181237SBalamuruhan S EMIT(PPC_RAW_MR(dst_reg, src_reg)); 564156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 565156d0e29SNaveen N. Rao case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 566156d0e29SNaveen N. Rao case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 567156d0e29SNaveen N. Rao PPC_LI32(dst_reg, imm); 568156d0e29SNaveen N. Rao if (imm < 0) 569156d0e29SNaveen N. Rao goto bpf_alu32_trunc; 570a4c92773SJiong Wang else if (insn_is_zext(&insn[i + 1])) 571a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 572156d0e29SNaveen N. Rao break; 573156d0e29SNaveen N. Rao 574156d0e29SNaveen N. Rao bpf_alu32_trunc: 575156d0e29SNaveen N. Rao /* Truncate to 32-bits */ 576a4c92773SJiong Wang if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext) 5773a181237SBalamuruhan S EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31)); 578156d0e29SNaveen N. Rao break; 579156d0e29SNaveen N. Rao 580156d0e29SNaveen N. Rao /* 581156d0e29SNaveen N. Rao * BPF_FROM_BE/LE 582156d0e29SNaveen N. Rao */ 583156d0e29SNaveen N. Rao case BPF_ALU | BPF_END | BPF_FROM_LE: 584156d0e29SNaveen N. Rao case BPF_ALU | BPF_END | BPF_FROM_BE: 585156d0e29SNaveen N. Rao #ifdef __BIG_ENDIAN__ 586156d0e29SNaveen N. Rao if (BPF_SRC(code) == BPF_FROM_BE) 587156d0e29SNaveen N. Rao goto emit_clear; 588156d0e29SNaveen N. Rao #else /* !__BIG_ENDIAN__ */ 589156d0e29SNaveen N. Rao if (BPF_SRC(code) == BPF_FROM_LE) 590156d0e29SNaveen N. Rao goto emit_clear; 591156d0e29SNaveen N. Rao #endif 592156d0e29SNaveen N. Rao switch (imm) { 593156d0e29SNaveen N. Rao case 16: 594156d0e29SNaveen N. Rao /* Rotate 8 bits left & mask with 0x0000ff00 */ 5953a181237SBalamuruhan S EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23)); 596156d0e29SNaveen N. Rao /* Rotate 8 bits right & insert LSB to reg */ 5973a181237SBalamuruhan S EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31)); 598156d0e29SNaveen N. Rao /* Move result back to dst_reg */ 5993a181237SBalamuruhan S EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1])); 600156d0e29SNaveen N. Rao break; 601156d0e29SNaveen N. Rao case 32: 602156d0e29SNaveen N. Rao /* 603156d0e29SNaveen N. Rao * Rotate word left by 8 bits: 604156d0e29SNaveen N. Rao * 2 bytes are already in their final position 605156d0e29SNaveen N. Rao * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 606156d0e29SNaveen N. Rao */ 6073a181237SBalamuruhan S EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31)); 608156d0e29SNaveen N. Rao /* Rotate 24 bits and insert byte 1 */ 6093a181237SBalamuruhan S EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7)); 610156d0e29SNaveen N. Rao /* Rotate 24 bits and insert byte 3 */ 6113a181237SBalamuruhan S EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23)); 6123a181237SBalamuruhan S EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1])); 613156d0e29SNaveen N. Rao break; 614156d0e29SNaveen N. Rao case 64: 615156d0e29SNaveen N. Rao /* 616156d0e29SNaveen N. Rao * Way easier and faster(?) to store the value 617156d0e29SNaveen N. Rao * into stack and then use ldbrx 618156d0e29SNaveen N. Rao * 619156d0e29SNaveen N. Rao * ctx->seen will be reliable in pass2, but 620156d0e29SNaveen N. Rao * the instructions generated will remain the 621156d0e29SNaveen N. Rao * same across all passes 622156d0e29SNaveen N. Rao */ 62386be36f6SNaveen N. Rao PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx)); 6243a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx))); 6253a181237SBalamuruhan S EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1])); 626156d0e29SNaveen N. Rao break; 627156d0e29SNaveen N. Rao } 628156d0e29SNaveen N. Rao break; 629156d0e29SNaveen N. Rao 630156d0e29SNaveen N. Rao emit_clear: 631156d0e29SNaveen N. Rao switch (imm) { 632156d0e29SNaveen N. Rao case 16: 633156d0e29SNaveen N. Rao /* zero-extend 16 bits into 64 bits */ 6343a181237SBalamuruhan S EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48)); 635a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 636a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 637156d0e29SNaveen N. Rao break; 638156d0e29SNaveen N. Rao case 32: 639a4c92773SJiong Wang if (!fp->aux->verifier_zext) 640156d0e29SNaveen N. Rao /* zero-extend 32 bits into 64 bits */ 6413a181237SBalamuruhan S EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32)); 642156d0e29SNaveen N. Rao break; 643156d0e29SNaveen N. Rao case 64: 644156d0e29SNaveen N. Rao /* nop */ 645156d0e29SNaveen N. Rao break; 646156d0e29SNaveen N. Rao } 647156d0e29SNaveen N. Rao break; 648156d0e29SNaveen N. Rao 649156d0e29SNaveen N. Rao /* 650156d0e29SNaveen N. Rao * BPF_ST(X) 651156d0e29SNaveen N. Rao */ 652156d0e29SNaveen N. Rao case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 653156d0e29SNaveen N. Rao case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 654156d0e29SNaveen N. Rao if (BPF_CLASS(code) == BPF_ST) { 6553a181237SBalamuruhan S EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm)); 656156d0e29SNaveen N. Rao src_reg = b2p[TMP_REG_1]; 657156d0e29SNaveen N. Rao } 6583a181237SBalamuruhan S EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); 659156d0e29SNaveen N. Rao break; 660156d0e29SNaveen N. Rao case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 661156d0e29SNaveen N. Rao case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 662156d0e29SNaveen N. Rao if (BPF_CLASS(code) == BPF_ST) { 6633a181237SBalamuruhan S EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm)); 664156d0e29SNaveen N. Rao src_reg = b2p[TMP_REG_1]; 665156d0e29SNaveen N. Rao } 6663a181237SBalamuruhan S EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); 667156d0e29SNaveen N. Rao break; 668156d0e29SNaveen N. Rao case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 669156d0e29SNaveen N. Rao case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 670156d0e29SNaveen N. Rao if (BPF_CLASS(code) == BPF_ST) { 671156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 672156d0e29SNaveen N. Rao src_reg = b2p[TMP_REG_1]; 673156d0e29SNaveen N. Rao } 6743a181237SBalamuruhan S EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); 675156d0e29SNaveen N. Rao break; 676156d0e29SNaveen N. Rao case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 677156d0e29SNaveen N. Rao case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 678156d0e29SNaveen N. Rao if (BPF_CLASS(code) == BPF_ST) { 679156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 680156d0e29SNaveen N. Rao src_reg = b2p[TMP_REG_1]; 681156d0e29SNaveen N. Rao } 68286be36f6SNaveen N. Rao PPC_BPF_STL(src_reg, dst_reg, off); 683156d0e29SNaveen N. Rao break; 684156d0e29SNaveen N. Rao 685156d0e29SNaveen N. Rao /* 68691c960b0SBrendan Jackman * BPF_STX ATOMIC (atomic ops) 687156d0e29SNaveen N. Rao */ 68891c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_W: 68991c960b0SBrendan Jackman if (insn->imm != BPF_ADD) { 69091c960b0SBrendan Jackman pr_err_ratelimited( 69191c960b0SBrendan Jackman "eBPF filter atomic op code %02x (@%d) unsupported\n", 69291c960b0SBrendan Jackman code, i); 69391c960b0SBrendan Jackman return -ENOTSUPP; 69491c960b0SBrendan Jackman } 69591c960b0SBrendan Jackman 696156d0e29SNaveen N. Rao /* *(u32 *)(dst + off) += src */ 69791c960b0SBrendan Jackman 698156d0e29SNaveen N. Rao /* Get EA into TMP_REG_1 */ 6993a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off)); 700b9c1e60eSDaniel Borkmann tmp_idx = ctx->idx * 4; 701156d0e29SNaveen N. Rao /* load value from memory into TMP_REG_2 */ 70206541865SBalamuruhan S EMIT(PPC_RAW_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0)); 703156d0e29SNaveen N. Rao /* add value from src_reg into this */ 70406541865SBalamuruhan S EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg)); 705156d0e29SNaveen N. Rao /* store result back */ 7063a181237SBalamuruhan S EMIT(PPC_RAW_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1])); 707156d0e29SNaveen N. Rao /* we're done if this succeeded */ 708b9c1e60eSDaniel Borkmann PPC_BCC_SHORT(COND_NE, tmp_idx); 709156d0e29SNaveen N. Rao break; 71091c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_DW: 71191c960b0SBrendan Jackman if (insn->imm != BPF_ADD) { 71291c960b0SBrendan Jackman pr_err_ratelimited( 71391c960b0SBrendan Jackman "eBPF filter atomic op code %02x (@%d) unsupported\n", 71491c960b0SBrendan Jackman code, i); 71591c960b0SBrendan Jackman return -ENOTSUPP; 71691c960b0SBrendan Jackman } 717156d0e29SNaveen N. Rao /* *(u64 *)(dst + off) += src */ 71891c960b0SBrendan Jackman 7193a181237SBalamuruhan S EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off)); 720b9c1e60eSDaniel Borkmann tmp_idx = ctx->idx * 4; 72106541865SBalamuruhan S EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0)); 72206541865SBalamuruhan S EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg)); 72306541865SBalamuruhan S EMIT(PPC_RAW_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1])); 724b9c1e60eSDaniel Borkmann PPC_BCC_SHORT(COND_NE, tmp_idx); 725156d0e29SNaveen N. Rao break; 726156d0e29SNaveen N. Rao 727156d0e29SNaveen N. Rao /* 728156d0e29SNaveen N. Rao * BPF_LDX 729156d0e29SNaveen N. Rao */ 730156d0e29SNaveen N. Rao /* dst = *(u8 *)(ul) (src + off) */ 731156d0e29SNaveen N. Rao case BPF_LDX | BPF_MEM | BPF_B: 7323a181237SBalamuruhan S EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 733a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 734a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 735156d0e29SNaveen N. Rao break; 736156d0e29SNaveen N. Rao /* dst = *(u16 *)(ul) (src + off) */ 737156d0e29SNaveen N. Rao case BPF_LDX | BPF_MEM | BPF_H: 7383a181237SBalamuruhan S EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); 739a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 740a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 741156d0e29SNaveen N. Rao break; 742156d0e29SNaveen N. Rao /* dst = *(u32 *)(ul) (src + off) */ 743156d0e29SNaveen N. Rao case BPF_LDX | BPF_MEM | BPF_W: 74406541865SBalamuruhan S EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); 745a4c92773SJiong Wang if (insn_is_zext(&insn[i + 1])) 746a4c92773SJiong Wang addrs[++i] = ctx->idx * 4; 747156d0e29SNaveen N. Rao break; 748156d0e29SNaveen N. Rao /* dst = *(u64 *)(ul) (src + off) */ 749156d0e29SNaveen N. Rao case BPF_LDX | BPF_MEM | BPF_DW: 75086be36f6SNaveen N. Rao PPC_BPF_LL(dst_reg, src_reg, off); 751156d0e29SNaveen N. Rao break; 752156d0e29SNaveen N. Rao 753156d0e29SNaveen N. Rao /* 754156d0e29SNaveen N. Rao * Doubleword load 755156d0e29SNaveen N. Rao * 16 byte instruction that uses two 'struct bpf_insn' 756156d0e29SNaveen N. Rao */ 757156d0e29SNaveen N. Rao case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 758156d0e29SNaveen N. Rao imm64 = ((u64)(u32) insn[i].imm) | 759156d0e29SNaveen N. Rao (((u64)(u32) insn[i+1].imm) << 32); 760156d0e29SNaveen N. Rao /* Adjust for two bpf instructions */ 761156d0e29SNaveen N. Rao addrs[++i] = ctx->idx * 4; 762156d0e29SNaveen N. Rao PPC_LI64(dst_reg, imm64); 763156d0e29SNaveen N. Rao break; 764156d0e29SNaveen N. Rao 765156d0e29SNaveen N. Rao /* 766156d0e29SNaveen N. Rao * Return/Exit 767156d0e29SNaveen N. Rao */ 768156d0e29SNaveen N. Rao case BPF_JMP | BPF_EXIT: 769156d0e29SNaveen N. Rao /* 770156d0e29SNaveen N. Rao * If this isn't the very last instruction, branch to 771156d0e29SNaveen N. Rao * the epilogue. If we _are_ the last instruction, 772156d0e29SNaveen N. Rao * we'll just fall through to the epilogue. 773156d0e29SNaveen N. Rao */ 774156d0e29SNaveen N. Rao if (i != flen - 1) 775156d0e29SNaveen N. Rao PPC_JMP(exit_addr); 776156d0e29SNaveen N. Rao /* else fall through to the epilogue */ 777156d0e29SNaveen N. Rao break; 778156d0e29SNaveen N. Rao 779156d0e29SNaveen N. Rao /* 7808484ce83SSandipan Das * Call kernel helper or bpf function 781156d0e29SNaveen N. Rao */ 782156d0e29SNaveen N. Rao case BPF_JMP | BPF_CALL: 783156d0e29SNaveen N. Rao ctx->seen |= SEEN_FUNC; 7848484ce83SSandipan Das 785e2c95a61SDaniel Borkmann ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, 786e2c95a61SDaniel Borkmann &func_addr, &func_addr_fixed); 787e2c95a61SDaniel Borkmann if (ret < 0) 788e2c95a61SDaniel Borkmann return ret; 789156d0e29SNaveen N. Rao 790e2c95a61SDaniel Borkmann if (func_addr_fixed) 791e2c95a61SDaniel Borkmann bpf_jit_emit_func_call_hlp(image, ctx, func_addr); 792e2c95a61SDaniel Borkmann else 793e2c95a61SDaniel Borkmann bpf_jit_emit_func_call_rel(image, ctx, func_addr); 794156d0e29SNaveen N. Rao /* move return value from r3 to BPF_REG_0 */ 7953a181237SBalamuruhan S EMIT(PPC_RAW_MR(b2p[BPF_REG_0], 3)); 796156d0e29SNaveen N. Rao break; 797156d0e29SNaveen N. Rao 798156d0e29SNaveen N. Rao /* 799156d0e29SNaveen N. Rao * Jumps and branches 800156d0e29SNaveen N. Rao */ 801156d0e29SNaveen N. Rao case BPF_JMP | BPF_JA: 802156d0e29SNaveen N. Rao PPC_JMP(addrs[i + 1 + off]); 803156d0e29SNaveen N. Rao break; 804156d0e29SNaveen N. Rao 805156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGT | BPF_K: 806156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGT | BPF_X: 807156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGT | BPF_K: 808156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGT | BPF_X: 8095f645996SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_K: 8105f645996SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_X: 8115f645996SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_K: 8125f645996SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_X: 813156d0e29SNaveen N. Rao true_cond = COND_GT; 814156d0e29SNaveen N. Rao goto cond_branch; 81520dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_K: 81620dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_X: 81720dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_K: 81820dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_X: 8195f645996SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_K: 8205f645996SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_X: 8215f645996SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_K: 8225f645996SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_X: 82320dbf5ccSDaniel Borkmann true_cond = COND_LT; 82420dbf5ccSDaniel Borkmann goto cond_branch; 825156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGE | BPF_K: 826156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGE | BPF_X: 827156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGE | BPF_K: 828156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGE | BPF_X: 8295f645996SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_K: 8305f645996SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_X: 8315f645996SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_K: 8325f645996SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_X: 833156d0e29SNaveen N. Rao true_cond = COND_GE; 834156d0e29SNaveen N. Rao goto cond_branch; 83520dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_K: 83620dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_X: 83720dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_K: 83820dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_X: 8395f645996SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_K: 8405f645996SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_X: 8415f645996SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_K: 8425f645996SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_X: 84320dbf5ccSDaniel Borkmann true_cond = COND_LE; 84420dbf5ccSDaniel Borkmann goto cond_branch; 845156d0e29SNaveen N. Rao case BPF_JMP | BPF_JEQ | BPF_K: 846156d0e29SNaveen N. Rao case BPF_JMP | BPF_JEQ | BPF_X: 8475f645996SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_K: 8485f645996SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_X: 849156d0e29SNaveen N. Rao true_cond = COND_EQ; 850156d0e29SNaveen N. Rao goto cond_branch; 851156d0e29SNaveen N. Rao case BPF_JMP | BPF_JNE | BPF_K: 852156d0e29SNaveen N. Rao case BPF_JMP | BPF_JNE | BPF_X: 8535f645996SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_K: 8545f645996SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_X: 855156d0e29SNaveen N. Rao true_cond = COND_NE; 856156d0e29SNaveen N. Rao goto cond_branch; 857156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSET | BPF_K: 858156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSET | BPF_X: 8595f645996SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_K: 8605f645996SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_X: 861156d0e29SNaveen N. Rao true_cond = COND_NE; 862156d0e29SNaveen N. Rao /* Fall through */ 863156d0e29SNaveen N. Rao 864156d0e29SNaveen N. Rao cond_branch: 865156d0e29SNaveen N. Rao switch (code) { 866156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGT | BPF_X: 86720dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_X: 868156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGE | BPF_X: 86920dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_X: 870156d0e29SNaveen N. Rao case BPF_JMP | BPF_JEQ | BPF_X: 871156d0e29SNaveen N. Rao case BPF_JMP | BPF_JNE | BPF_X: 8725f645996SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_X: 8735f645996SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_X: 8745f645996SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_X: 8755f645996SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_X: 8765f645996SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_X: 8775f645996SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_X: 878156d0e29SNaveen N. Rao /* unsigned comparison */ 8795f645996SJiong Wang if (BPF_CLASS(code) == BPF_JMP32) 8803a181237SBalamuruhan S EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 8815f645996SJiong Wang else 8823a181237SBalamuruhan S EMIT(PPC_RAW_CMPLD(dst_reg, src_reg)); 883156d0e29SNaveen N. Rao break; 884156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGT | BPF_X: 88520dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_X: 886156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGE | BPF_X: 88720dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_X: 8885f645996SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_X: 8895f645996SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_X: 8905f645996SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_X: 8915f645996SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_X: 892156d0e29SNaveen N. Rao /* signed comparison */ 8935f645996SJiong Wang if (BPF_CLASS(code) == BPF_JMP32) 8943a181237SBalamuruhan S EMIT(PPC_RAW_CMPW(dst_reg, src_reg)); 8955f645996SJiong Wang else 8963a181237SBalamuruhan S EMIT(PPC_RAW_CMPD(dst_reg, src_reg)); 897156d0e29SNaveen N. Rao break; 898156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSET | BPF_X: 8995f645996SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_X: 9005f645996SJiong Wang if (BPF_CLASS(code) == BPF_JMP) { 9013a181237SBalamuruhan S EMIT(PPC_RAW_AND_DOT(b2p[TMP_REG_1], dst_reg, 9023a181237SBalamuruhan S src_reg)); 9035f645996SJiong Wang } else { 9045f645996SJiong Wang int tmp_reg = b2p[TMP_REG_1]; 9055f645996SJiong Wang 9063a181237SBalamuruhan S EMIT(PPC_RAW_AND(tmp_reg, dst_reg, src_reg)); 9073a181237SBalamuruhan S EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0, 9083a181237SBalamuruhan S 31)); 9095f645996SJiong Wang } 910156d0e29SNaveen N. Rao break; 911156d0e29SNaveen N. Rao case BPF_JMP | BPF_JNE | BPF_K: 912156d0e29SNaveen N. Rao case BPF_JMP | BPF_JEQ | BPF_K: 913156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGT | BPF_K: 91420dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLT | BPF_K: 915156d0e29SNaveen N. Rao case BPF_JMP | BPF_JGE | BPF_K: 91620dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JLE | BPF_K: 9175f645996SJiong Wang case BPF_JMP32 | BPF_JNE | BPF_K: 9185f645996SJiong Wang case BPF_JMP32 | BPF_JEQ | BPF_K: 9195f645996SJiong Wang case BPF_JMP32 | BPF_JGT | BPF_K: 9205f645996SJiong Wang case BPF_JMP32 | BPF_JLT | BPF_K: 9215f645996SJiong Wang case BPF_JMP32 | BPF_JGE | BPF_K: 9225f645996SJiong Wang case BPF_JMP32 | BPF_JLE | BPF_K: 9235f645996SJiong Wang { 9245f645996SJiong Wang bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32; 9255f645996SJiong Wang 926156d0e29SNaveen N. Rao /* 927156d0e29SNaveen N. Rao * Need sign-extended load, so only positive 928156d0e29SNaveen N. Rao * values can be used as imm in cmpldi 929156d0e29SNaveen N. Rao */ 9305f645996SJiong Wang if (imm >= 0 && imm < 32768) { 9315f645996SJiong Wang if (is_jmp32) 9323a181237SBalamuruhan S EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 9335f645996SJiong Wang else 9343a181237SBalamuruhan S EMIT(PPC_RAW_CMPLDI(dst_reg, imm)); 9355f645996SJiong Wang } else { 936156d0e29SNaveen N. Rao /* sign-extending load */ 937156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 938156d0e29SNaveen N. Rao /* ... but unsigned comparison */ 9395f645996SJiong Wang if (is_jmp32) 9403a181237SBalamuruhan S EMIT(PPC_RAW_CMPLW(dst_reg, 9413a181237SBalamuruhan S b2p[TMP_REG_1])); 9425f645996SJiong Wang else 9433a181237SBalamuruhan S EMIT(PPC_RAW_CMPLD(dst_reg, 9443a181237SBalamuruhan S b2p[TMP_REG_1])); 945156d0e29SNaveen N. Rao } 946156d0e29SNaveen N. Rao break; 9475f645996SJiong Wang } 948156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGT | BPF_K: 94920dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLT | BPF_K: 950156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSGE | BPF_K: 95120dbf5ccSDaniel Borkmann case BPF_JMP | BPF_JSLE | BPF_K: 9525f645996SJiong Wang case BPF_JMP32 | BPF_JSGT | BPF_K: 9535f645996SJiong Wang case BPF_JMP32 | BPF_JSLT | BPF_K: 9545f645996SJiong Wang case BPF_JMP32 | BPF_JSGE | BPF_K: 9555f645996SJiong Wang case BPF_JMP32 | BPF_JSLE | BPF_K: 9565f645996SJiong Wang { 9575f645996SJiong Wang bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32; 9585f645996SJiong Wang 959156d0e29SNaveen N. Rao /* 960156d0e29SNaveen N. Rao * signed comparison, so any 16-bit value 961156d0e29SNaveen N. Rao * can be used in cmpdi 962156d0e29SNaveen N. Rao */ 9635f645996SJiong Wang if (imm >= -32768 && imm < 32768) { 9645f645996SJiong Wang if (is_jmp32) 9653a181237SBalamuruhan S EMIT(PPC_RAW_CMPWI(dst_reg, imm)); 9665f645996SJiong Wang else 9673a181237SBalamuruhan S EMIT(PPC_RAW_CMPDI(dst_reg, imm)); 9685f645996SJiong Wang } else { 969156d0e29SNaveen N. Rao PPC_LI32(b2p[TMP_REG_1], imm); 9705f645996SJiong Wang if (is_jmp32) 9713a181237SBalamuruhan S EMIT(PPC_RAW_CMPW(dst_reg, 9723a181237SBalamuruhan S b2p[TMP_REG_1])); 9735f645996SJiong Wang else 9743a181237SBalamuruhan S EMIT(PPC_RAW_CMPD(dst_reg, 9753a181237SBalamuruhan S b2p[TMP_REG_1])); 976156d0e29SNaveen N. Rao } 977156d0e29SNaveen N. Rao break; 9785f645996SJiong Wang } 979156d0e29SNaveen N. Rao case BPF_JMP | BPF_JSET | BPF_K: 9805f645996SJiong Wang case BPF_JMP32 | BPF_JSET | BPF_K: 981156d0e29SNaveen N. Rao /* andi does not sign-extend the immediate */ 982156d0e29SNaveen N. Rao if (imm >= 0 && imm < 32768) 983156d0e29SNaveen N. Rao /* PPC_ANDI is _only/always_ dot-form */ 9843a181237SBalamuruhan S EMIT(PPC_RAW_ANDI(b2p[TMP_REG_1], dst_reg, imm)); 985156d0e29SNaveen N. Rao else { 9865f645996SJiong Wang int tmp_reg = b2p[TMP_REG_1]; 9875f645996SJiong Wang 9885f645996SJiong Wang PPC_LI32(tmp_reg, imm); 9895f645996SJiong Wang if (BPF_CLASS(code) == BPF_JMP) { 9903a181237SBalamuruhan S EMIT(PPC_RAW_AND_DOT(tmp_reg, dst_reg, 9913a181237SBalamuruhan S tmp_reg)); 9925f645996SJiong Wang } else { 9933a181237SBalamuruhan S EMIT(PPC_RAW_AND(tmp_reg, dst_reg, 9943a181237SBalamuruhan S tmp_reg)); 9953a181237SBalamuruhan S EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 9963a181237SBalamuruhan S 0, 0, 31)); 9975f645996SJiong Wang } 998156d0e29SNaveen N. Rao } 999156d0e29SNaveen N. Rao break; 1000156d0e29SNaveen N. Rao } 1001156d0e29SNaveen N. Rao PPC_BCC(true_cond, addrs[i + 1 + off]); 1002156d0e29SNaveen N. Rao break; 1003156d0e29SNaveen N. Rao 1004156d0e29SNaveen N. Rao /* 1005ce076141SNaveen N. Rao * Tail call 1006156d0e29SNaveen N. Rao */ 100771189fa9SAlexei Starovoitov case BPF_JMP | BPF_TAIL_CALL: 1008ce076141SNaveen N. Rao ctx->seen |= SEEN_TAILCALL; 1009ce076141SNaveen N. Rao bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 1010ce076141SNaveen N. Rao break; 1011156d0e29SNaveen N. Rao 1012156d0e29SNaveen N. Rao default: 1013156d0e29SNaveen N. Rao /* 1014156d0e29SNaveen N. Rao * The filter contains something cruel & unusual. 1015156d0e29SNaveen N. Rao * We don't handle it, but also there shouldn't be 1016156d0e29SNaveen N. Rao * anything missing from our list. 1017156d0e29SNaveen N. Rao */ 1018156d0e29SNaveen N. Rao pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", 1019156d0e29SNaveen N. Rao code, i); 1020156d0e29SNaveen N. Rao return -ENOTSUPP; 1021156d0e29SNaveen N. Rao } 1022156d0e29SNaveen N. Rao } 1023156d0e29SNaveen N. Rao 1024156d0e29SNaveen N. Rao /* Set end-of-body-code address for exit. */ 1025156d0e29SNaveen N. Rao addrs[i] = ctx->idx * 4; 1026156d0e29SNaveen N. Rao 1027156d0e29SNaveen N. Rao return 0; 1028156d0e29SNaveen N. Rao } 1029156d0e29SNaveen N. Rao 1030025dceb0SSandipan Das /* Fix the branch target addresses for subprog calls */ 1031025dceb0SSandipan Das static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image, 1032025dceb0SSandipan Das struct codegen_context *ctx, u32 *addrs) 1033025dceb0SSandipan Das { 1034025dceb0SSandipan Das const struct bpf_insn *insn = fp->insnsi; 1035025dceb0SSandipan Das bool func_addr_fixed; 1036025dceb0SSandipan Das u64 func_addr; 1037025dceb0SSandipan Das u32 tmp_idx; 1038025dceb0SSandipan Das int i, ret; 1039025dceb0SSandipan Das 1040025dceb0SSandipan Das for (i = 0; i < fp->len; i++) { 1041025dceb0SSandipan Das /* 1042025dceb0SSandipan Das * During the extra pass, only the branch target addresses for 1043025dceb0SSandipan Das * the subprog calls need to be fixed. All other instructions 1044025dceb0SSandipan Das * can left untouched. 1045025dceb0SSandipan Das * 1046025dceb0SSandipan Das * The JITed image length does not change because we already 1047025dceb0SSandipan Das * ensure that the JITed instruction sequence for these calls 1048025dceb0SSandipan Das * are of fixed length by padding them with NOPs. 1049025dceb0SSandipan Das */ 1050025dceb0SSandipan Das if (insn[i].code == (BPF_JMP | BPF_CALL) && 1051025dceb0SSandipan Das insn[i].src_reg == BPF_PSEUDO_CALL) { 1052025dceb0SSandipan Das ret = bpf_jit_get_func_addr(fp, &insn[i], true, 1053025dceb0SSandipan Das &func_addr, 1054025dceb0SSandipan Das &func_addr_fixed); 1055025dceb0SSandipan Das if (ret < 0) 1056025dceb0SSandipan Das return ret; 1057025dceb0SSandipan Das 1058025dceb0SSandipan Das /* 1059025dceb0SSandipan Das * Save ctx->idx as this would currently point to the 1060025dceb0SSandipan Das * end of the JITed image and set it to the offset of 1061025dceb0SSandipan Das * the instruction sequence corresponding to the 1062025dceb0SSandipan Das * subprog call temporarily. 1063025dceb0SSandipan Das */ 1064025dceb0SSandipan Das tmp_idx = ctx->idx; 1065025dceb0SSandipan Das ctx->idx = addrs[i] / 4; 1066025dceb0SSandipan Das bpf_jit_emit_func_call_rel(image, ctx, func_addr); 1067025dceb0SSandipan Das 1068025dceb0SSandipan Das /* 1069025dceb0SSandipan Das * Restore ctx->idx here. This is safe as the length 1070025dceb0SSandipan Das * of the JITed sequence remains unchanged. 1071025dceb0SSandipan Das */ 1072025dceb0SSandipan Das ctx->idx = tmp_idx; 1073025dceb0SSandipan Das } 1074025dceb0SSandipan Das } 1075025dceb0SSandipan Das 1076025dceb0SSandipan Das return 0; 1077025dceb0SSandipan Das } 1078025dceb0SSandipan Das 10798484ce83SSandipan Das struct powerpc64_jit_data { 10808484ce83SSandipan Das struct bpf_binary_header *header; 10818484ce83SSandipan Das u32 *addrs; 10828484ce83SSandipan Das u8 *image; 10838484ce83SSandipan Das u32 proglen; 10848484ce83SSandipan Das struct codegen_context ctx; 10858484ce83SSandipan Das }; 10868484ce83SSandipan Das 1087a4c92773SJiong Wang bool bpf_jit_needs_zext(void) 1088a4c92773SJiong Wang { 1089a4c92773SJiong Wang return true; 1090a4c92773SJiong Wang } 1091a4c92773SJiong Wang 1092156d0e29SNaveen N. Rao struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 1093156d0e29SNaveen N. Rao { 1094156d0e29SNaveen N. Rao u32 proglen; 1095156d0e29SNaveen N. Rao u32 alloclen; 1096156d0e29SNaveen N. Rao u8 *image = NULL; 1097156d0e29SNaveen N. Rao u32 *code_base; 1098156d0e29SNaveen N. Rao u32 *addrs; 10998484ce83SSandipan Das struct powerpc64_jit_data *jit_data; 1100156d0e29SNaveen N. Rao struct codegen_context cgctx; 1101156d0e29SNaveen N. Rao int pass; 1102156d0e29SNaveen N. Rao int flen; 1103156d0e29SNaveen N. Rao struct bpf_binary_header *bpf_hdr; 1104b7b7013cSNaveen N. Rao struct bpf_prog *org_fp = fp; 1105b7b7013cSNaveen N. Rao struct bpf_prog *tmp_fp; 1106b7b7013cSNaveen N. Rao bool bpf_blinded = false; 11078484ce83SSandipan Das bool extra_pass = false; 1108156d0e29SNaveen N. Rao 110960b58afcSAlexei Starovoitov if (!fp->jit_requested) 1110b7b7013cSNaveen N. Rao return org_fp; 1111b7b7013cSNaveen N. Rao 1112b7b7013cSNaveen N. Rao tmp_fp = bpf_jit_blind_constants(org_fp); 1113b7b7013cSNaveen N. Rao if (IS_ERR(tmp_fp)) 1114b7b7013cSNaveen N. Rao return org_fp; 1115b7b7013cSNaveen N. Rao 1116b7b7013cSNaveen N. Rao if (tmp_fp != org_fp) { 1117b7b7013cSNaveen N. Rao bpf_blinded = true; 1118b7b7013cSNaveen N. Rao fp = tmp_fp; 1119b7b7013cSNaveen N. Rao } 1120156d0e29SNaveen N. Rao 11218484ce83SSandipan Das jit_data = fp->aux->jit_data; 11228484ce83SSandipan Das if (!jit_data) { 11238484ce83SSandipan Das jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 11248484ce83SSandipan Das if (!jit_data) { 11258484ce83SSandipan Das fp = org_fp; 11268484ce83SSandipan Das goto out; 11278484ce83SSandipan Das } 11288484ce83SSandipan Das fp->aux->jit_data = jit_data; 11298484ce83SSandipan Das } 11308484ce83SSandipan Das 1131156d0e29SNaveen N. Rao flen = fp->len; 11328484ce83SSandipan Das addrs = jit_data->addrs; 11338484ce83SSandipan Das if (addrs) { 11348484ce83SSandipan Das cgctx = jit_data->ctx; 11358484ce83SSandipan Das image = jit_data->image; 11368484ce83SSandipan Das bpf_hdr = jit_data->header; 11378484ce83SSandipan Das proglen = jit_data->proglen; 11388484ce83SSandipan Das alloclen = proglen + FUNCTION_DESCR_SIZE; 11398484ce83SSandipan Das extra_pass = true; 11408484ce83SSandipan Das goto skip_init_ctx; 11418484ce83SSandipan Das } 11428484ce83SSandipan Das 11436396bb22SKees Cook addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 1144b7b7013cSNaveen N. Rao if (addrs == NULL) { 1145b7b7013cSNaveen N. Rao fp = org_fp; 11468484ce83SSandipan Das goto out_addrs; 1147b7b7013cSNaveen N. Rao } 1148b7b7013cSNaveen N. Rao 1149b7b7013cSNaveen N. Rao memset(&cgctx, 0, sizeof(struct codegen_context)); 1150b7b7013cSNaveen N. Rao 1151ac0761ebSSandipan Das /* Make sure that the stack is quadword aligned. */ 1152ac0761ebSSandipan Das cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 1153ac0761ebSSandipan Das 1154b7b7013cSNaveen N. Rao /* Scouting faux-generate pass 0 */ 11558484ce83SSandipan Das if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 1156b7b7013cSNaveen N. Rao /* We hit something illegal or unsupported. */ 1157b7b7013cSNaveen N. Rao fp = org_fp; 11588484ce83SSandipan Das goto out_addrs; 1159b7b7013cSNaveen N. Rao } 1160156d0e29SNaveen N. Rao 1161156d0e29SNaveen N. Rao /* 11627de08690SEric Dumazet * If we have seen a tail call, we need a second pass. 11637de08690SEric Dumazet * This is because bpf_jit_emit_common_epilogue() is called 11647de08690SEric Dumazet * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 11657de08690SEric Dumazet */ 11667de08690SEric Dumazet if (cgctx.seen & SEEN_TAILCALL) { 11677de08690SEric Dumazet cgctx.idx = 0; 11687de08690SEric Dumazet if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 11697de08690SEric Dumazet fp = org_fp; 11707de08690SEric Dumazet goto out_addrs; 11717de08690SEric Dumazet } 11727de08690SEric Dumazet } 11737de08690SEric Dumazet 11747de08690SEric Dumazet /* 1175156d0e29SNaveen N. Rao * Pretend to build prologue, given the features we've seen. This will 1176156d0e29SNaveen N. Rao * update ctgtx.idx as it pretends to output instructions, then we can 1177156d0e29SNaveen N. Rao * calculate total size from idx. 1178156d0e29SNaveen N. Rao */ 1179156d0e29SNaveen N. Rao bpf_jit_build_prologue(0, &cgctx); 1180156d0e29SNaveen N. Rao bpf_jit_build_epilogue(0, &cgctx); 1181156d0e29SNaveen N. Rao 1182156d0e29SNaveen N. Rao proglen = cgctx.idx * 4; 1183156d0e29SNaveen N. Rao alloclen = proglen + FUNCTION_DESCR_SIZE; 1184156d0e29SNaveen N. Rao 1185156d0e29SNaveen N. Rao bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, 1186156d0e29SNaveen N. Rao bpf_jit_fill_ill_insns); 1187b7b7013cSNaveen N. Rao if (!bpf_hdr) { 1188b7b7013cSNaveen N. Rao fp = org_fp; 11898484ce83SSandipan Das goto out_addrs; 1190b7b7013cSNaveen N. Rao } 1191156d0e29SNaveen N. Rao 11928484ce83SSandipan Das skip_init_ctx: 1193156d0e29SNaveen N. Rao code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 1194156d0e29SNaveen N. Rao 1195025dceb0SSandipan Das if (extra_pass) { 1196025dceb0SSandipan Das /* 1197025dceb0SSandipan Das * Do not touch the prologue and epilogue as they will remain 1198025dceb0SSandipan Das * unchanged. Only fix the branch target address for subprog 1199025dceb0SSandipan Das * calls in the body. 1200025dceb0SSandipan Das * 1201025dceb0SSandipan Das * This does not change the offsets and lengths of the subprog 1202025dceb0SSandipan Das * call instruction sequences and hence, the size of the JITed 1203025dceb0SSandipan Das * image as well. 1204025dceb0SSandipan Das */ 1205025dceb0SSandipan Das bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs); 1206025dceb0SSandipan Das 1207025dceb0SSandipan Das /* There is no need to perform the usual passes. */ 1208025dceb0SSandipan Das goto skip_codegen_passes; 1209025dceb0SSandipan Das } 1210025dceb0SSandipan Das 1211156d0e29SNaveen N. Rao /* Code generation passes 1-2 */ 1212156d0e29SNaveen N. Rao for (pass = 1; pass < 3; pass++) { 1213156d0e29SNaveen N. Rao /* Now build the prologue, body code & epilogue for real. */ 1214156d0e29SNaveen N. Rao cgctx.idx = 0; 1215156d0e29SNaveen N. Rao bpf_jit_build_prologue(code_base, &cgctx); 12168484ce83SSandipan Das bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass); 1217156d0e29SNaveen N. Rao bpf_jit_build_epilogue(code_base, &cgctx); 1218156d0e29SNaveen N. Rao 1219156d0e29SNaveen N. Rao if (bpf_jit_enable > 1) 1220156d0e29SNaveen N. Rao pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 1221156d0e29SNaveen N. Rao proglen - (cgctx.idx * 4), cgctx.seen); 1222156d0e29SNaveen N. Rao } 1223156d0e29SNaveen N. Rao 1224025dceb0SSandipan Das skip_codegen_passes: 1225156d0e29SNaveen N. Rao if (bpf_jit_enable > 1) 1226156d0e29SNaveen N. Rao /* 1227156d0e29SNaveen N. Rao * Note that we output the base address of the code_base 1228156d0e29SNaveen N. Rao * rather than image, since opcodes are in code_base. 1229156d0e29SNaveen N. Rao */ 1230156d0e29SNaveen N. Rao bpf_jit_dump(flen, proglen, pass, code_base); 1231156d0e29SNaveen N. Rao 1232156d0e29SNaveen N. Rao #ifdef PPC64_ELF_ABI_v1 1233156d0e29SNaveen N. Rao /* Function descriptor nastiness: Address + TOC */ 1234156d0e29SNaveen N. Rao ((u64 *)image)[0] = (u64)code_base; 1235156d0e29SNaveen N. Rao ((u64 *)image)[1] = local_paca->kernel_toc; 1236156d0e29SNaveen N. Rao #endif 1237052de33cSDaniel Borkmann 1238156d0e29SNaveen N. Rao fp->bpf_func = (void *)image; 1239156d0e29SNaveen N. Rao fp->jited = 1; 1240783d28ddSMartin KaFai Lau fp->jited_len = alloclen; 1241156d0e29SNaveen N. Rao 124210528b9cSNaveen N. Rao bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); 12438484ce83SSandipan Das if (!fp->is_func || extra_pass) { 12446f20c71dSSandipan Das bpf_prog_fill_jited_linfo(fp, addrs); 12458484ce83SSandipan Das out_addrs: 12468484ce83SSandipan Das kfree(addrs); 12478484ce83SSandipan Das kfree(jit_data); 12488484ce83SSandipan Das fp->aux->jit_data = NULL; 12498484ce83SSandipan Das } else { 12508484ce83SSandipan Das jit_data->addrs = addrs; 12518484ce83SSandipan Das jit_data->ctx = cgctx; 12528484ce83SSandipan Das jit_data->proglen = proglen; 12538484ce83SSandipan Das jit_data->image = image; 12548484ce83SSandipan Das jit_data->header = bpf_hdr; 12558484ce83SSandipan Das } 1256156d0e29SNaveen N. Rao 1257156d0e29SNaveen N. Rao out: 1258b7b7013cSNaveen N. Rao if (bpf_blinded) 1259b7b7013cSNaveen N. Rao bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 1260b7b7013cSNaveen N. Rao 1261156d0e29SNaveen N. Rao return fp; 1262156d0e29SNaveen N. Rao } 1263156d0e29SNaveen N. Rao 126474451e66SDaniel Borkmann /* Overriding bpf_jit_free() as we don't set images read-only. */ 1265156d0e29SNaveen N. Rao void bpf_jit_free(struct bpf_prog *fp) 1266156d0e29SNaveen N. Rao { 1267156d0e29SNaveen N. Rao unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 1268156d0e29SNaveen N. Rao struct bpf_binary_header *bpf_hdr = (void *)addr; 1269156d0e29SNaveen N. Rao 1270156d0e29SNaveen N. Rao if (fp->jited) 1271156d0e29SNaveen N. Rao bpf_jit_binary_free(bpf_hdr); 1272156d0e29SNaveen N. Rao 1273156d0e29SNaveen N. Rao bpf_prog_unlock_free(fp); 1274156d0e29SNaveen N. Rao } 1275