1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <asm/kprobes.h> 17 #include <linux/bpf.h> 18 19 #include "bpf_jit.h" 20 21 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 22 { 23 memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 24 } 25 26 /* Fix updated addresses (for subprog calls, ldimm64, et al) during extra pass */ 27 static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image, 28 struct codegen_context *ctx, u32 *addrs) 29 { 30 const struct bpf_insn *insn = fp->insnsi; 31 bool func_addr_fixed; 32 u64 func_addr; 33 u32 tmp_idx; 34 int i, j, ret; 35 36 for (i = 0; i < fp->len; i++) { 37 /* 38 * During the extra pass, only the branch target addresses for 39 * the subprog calls need to be fixed. All other instructions 40 * can left untouched. 41 * 42 * The JITed image length does not change because we already 43 * ensure that the JITed instruction sequence for these calls 44 * are of fixed length by padding them with NOPs. 45 */ 46 if (insn[i].code == (BPF_JMP | BPF_CALL) && 47 insn[i].src_reg == BPF_PSEUDO_CALL) { 48 ret = bpf_jit_get_func_addr(fp, &insn[i], true, 49 &func_addr, 50 &func_addr_fixed); 51 if (ret < 0) 52 return ret; 53 54 /* 55 * Save ctx->idx as this would currently point to the 56 * end of the JITed image and set it to the offset of 57 * the instruction sequence corresponding to the 58 * subprog call temporarily. 59 */ 60 tmp_idx = ctx->idx; 61 ctx->idx = addrs[i] / 4; 62 ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr); 63 if (ret) 64 return ret; 65 66 /* 67 * Restore ctx->idx here. This is safe as the length 68 * of the JITed sequence remains unchanged. 69 */ 70 ctx->idx = tmp_idx; 71 } else if (insn[i].code == (BPF_LD | BPF_IMM | BPF_DW)) { 72 tmp_idx = ctx->idx; 73 ctx->idx = addrs[i] / 4; 74 #ifdef CONFIG_PPC32 75 PPC_LI32(bpf_to_ppc(insn[i].dst_reg) - 1, (u32)insn[i + 1].imm); 76 PPC_LI32(bpf_to_ppc(insn[i].dst_reg), (u32)insn[i].imm); 77 for (j = ctx->idx - addrs[i] / 4; j < 4; j++) 78 EMIT(PPC_RAW_NOP()); 79 #else 80 func_addr = ((u64)(u32)insn[i].imm) | (((u64)(u32)insn[i + 1].imm) << 32); 81 PPC_LI64(bpf_to_ppc(insn[i].dst_reg), func_addr); 82 /* overwrite rest with nops */ 83 for (j = ctx->idx - addrs[i] / 4; j < 5; j++) 84 EMIT(PPC_RAW_NOP()); 85 #endif 86 ctx->idx = tmp_idx; 87 i++; 88 } 89 } 90 91 return 0; 92 } 93 94 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr) 95 { 96 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) { 97 PPC_JMP(exit_addr); 98 } else if (ctx->alt_exit_addr) { 99 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4)))) 100 return -1; 101 PPC_JMP(ctx->alt_exit_addr); 102 } else { 103 ctx->alt_exit_addr = ctx->idx * 4; 104 bpf_jit_build_epilogue(image, ctx); 105 } 106 107 return 0; 108 } 109 110 struct powerpc64_jit_data { 111 struct bpf_binary_header *header; 112 u32 *addrs; 113 u8 *image; 114 u32 proglen; 115 struct codegen_context ctx; 116 }; 117 118 bool bpf_jit_needs_zext(void) 119 { 120 return true; 121 } 122 123 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 124 { 125 u32 proglen; 126 u32 alloclen; 127 u8 *image = NULL; 128 u32 *code_base; 129 u32 *addrs; 130 struct powerpc64_jit_data *jit_data; 131 struct codegen_context cgctx; 132 int pass; 133 int flen; 134 struct bpf_binary_header *bpf_hdr; 135 struct bpf_prog *org_fp = fp; 136 struct bpf_prog *tmp_fp; 137 bool bpf_blinded = false; 138 bool extra_pass = false; 139 u32 extable_len; 140 u32 fixup_len; 141 142 if (!fp->jit_requested) 143 return org_fp; 144 145 tmp_fp = bpf_jit_blind_constants(org_fp); 146 if (IS_ERR(tmp_fp)) 147 return org_fp; 148 149 if (tmp_fp != org_fp) { 150 bpf_blinded = true; 151 fp = tmp_fp; 152 } 153 154 jit_data = fp->aux->jit_data; 155 if (!jit_data) { 156 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 157 if (!jit_data) { 158 fp = org_fp; 159 goto out; 160 } 161 fp->aux->jit_data = jit_data; 162 } 163 164 flen = fp->len; 165 addrs = jit_data->addrs; 166 if (addrs) { 167 cgctx = jit_data->ctx; 168 image = jit_data->image; 169 bpf_hdr = jit_data->header; 170 proglen = jit_data->proglen; 171 extra_pass = true; 172 goto skip_init_ctx; 173 } 174 175 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 176 if (addrs == NULL) { 177 fp = org_fp; 178 goto out_addrs; 179 } 180 181 memset(&cgctx, 0, sizeof(struct codegen_context)); 182 bpf_jit_init_reg_mapping(&cgctx); 183 184 /* Make sure that the stack is quadword aligned. */ 185 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 186 187 /* Scouting faux-generate pass 0 */ 188 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) { 189 /* We hit something illegal or unsupported. */ 190 fp = org_fp; 191 goto out_addrs; 192 } 193 194 /* 195 * If we have seen a tail call, we need a second pass. 196 * This is because bpf_jit_emit_common_epilogue() is called 197 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 198 * We also need a second pass if we ended up with too large 199 * a program so as to ensure BPF_EXIT branches are in range. 200 */ 201 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 202 cgctx.idx = 0; 203 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) { 204 fp = org_fp; 205 goto out_addrs; 206 } 207 } 208 209 bpf_jit_realloc_regs(&cgctx); 210 /* 211 * Pretend to build prologue, given the features we've seen. This will 212 * update ctgtx.idx as it pretends to output instructions, then we can 213 * calculate total size from idx. 214 */ 215 bpf_jit_build_prologue(0, &cgctx); 216 addrs[fp->len] = cgctx.idx * 4; 217 bpf_jit_build_epilogue(0, &cgctx); 218 219 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; 220 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); 221 222 proglen = cgctx.idx * 4; 223 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; 224 225 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns); 226 if (!bpf_hdr) { 227 fp = org_fp; 228 goto out_addrs; 229 } 230 231 if (extable_len) 232 fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len; 233 234 skip_init_ctx: 235 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 236 237 if (extra_pass) { 238 /* 239 * Do not touch the prologue and epilogue as they will remain 240 * unchanged. Only fix the branch target address for subprog 241 * calls in the body, and ldimm64 instructions. 242 * 243 * This does not change the offsets and lengths of the subprog 244 * call instruction sequences and hence, the size of the JITed 245 * image as well. 246 */ 247 bpf_jit_fixup_addresses(fp, code_base, &cgctx, addrs); 248 249 /* There is no need to perform the usual passes. */ 250 goto skip_codegen_passes; 251 } 252 253 /* Code generation passes 1-2 */ 254 for (pass = 1; pass < 3; pass++) { 255 /* Now build the prologue, body code & epilogue for real. */ 256 cgctx.idx = 0; 257 cgctx.alt_exit_addr = 0; 258 bpf_jit_build_prologue(code_base, &cgctx); 259 if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) { 260 bpf_jit_binary_free(bpf_hdr); 261 fp = org_fp; 262 goto out_addrs; 263 } 264 bpf_jit_build_epilogue(code_base, &cgctx); 265 266 if (bpf_jit_enable > 1) 267 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 268 proglen - (cgctx.idx * 4), cgctx.seen); 269 } 270 271 skip_codegen_passes: 272 if (bpf_jit_enable > 1) 273 /* 274 * Note that we output the base address of the code_base 275 * rather than image, since opcodes are in code_base. 276 */ 277 bpf_jit_dump(flen, proglen, pass, code_base); 278 279 #ifdef PPC64_ELF_ABI_v1 280 /* Function descriptor nastiness: Address + TOC */ 281 ((u64 *)image)[0] = (u64)code_base; 282 ((u64 *)image)[1] = local_paca->kernel_toc; 283 #endif 284 285 fp->bpf_func = (void *)image; 286 fp->jited = 1; 287 fp->jited_len = proglen + FUNCTION_DESCR_SIZE; 288 289 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size); 290 if (!fp->is_func || extra_pass) { 291 bpf_jit_binary_lock_ro(bpf_hdr); 292 bpf_prog_fill_jited_linfo(fp, addrs); 293 out_addrs: 294 kfree(addrs); 295 kfree(jit_data); 296 fp->aux->jit_data = NULL; 297 } else { 298 jit_data->addrs = addrs; 299 jit_data->ctx = cgctx; 300 jit_data->proglen = proglen; 301 jit_data->image = image; 302 jit_data->header = bpf_hdr; 303 } 304 305 out: 306 if (bpf_blinded) 307 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 308 309 return fp; 310 } 311 312 /* 313 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling 314 * this function, as this only applies to BPF_PROBE_MEM, for now. 315 */ 316 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx, 317 int insn_idx, int jmp_off, int dst_reg) 318 { 319 off_t offset; 320 unsigned long pc; 321 struct exception_table_entry *ex; 322 u32 *fixup; 323 324 /* Populate extable entries only in the last pass */ 325 if (pass != 2) 326 return 0; 327 328 if (!fp->aux->extable || 329 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) 330 return -EINVAL; 331 332 pc = (unsigned long)&image[insn_idx]; 333 334 fixup = (void *)fp->aux->extable - 335 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + 336 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); 337 338 fixup[0] = PPC_RAW_LI(dst_reg, 0); 339 if (IS_ENABLED(CONFIG_PPC32)) 340 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ 341 342 fixup[BPF_FIXUP_LEN - 1] = 343 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); 344 345 ex = &fp->aux->extable[ctx->exentry_idx]; 346 347 offset = pc - (long)&ex->insn; 348 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 349 return -ERANGE; 350 ex->insn = offset; 351 352 offset = (long)fixup - (long)&ex->fixup; 353 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 354 return -ERANGE; 355 ex->fixup = offset; 356 357 ctx->exentry_idx++; 358 return 0; 359 } 360