1 /* 2 * bpf_jit_comp64.c: eBPF JIT compiler 3 * 4 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 5 * IBM Corporation 6 * 7 * Based on the powerpc classic BPF JIT compiler by Matt Evans 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; version 2 12 * of the License. 13 */ 14 #include <linux/moduleloader.h> 15 #include <asm/cacheflush.h> 16 #include <asm/asm-compat.h> 17 #include <linux/netdevice.h> 18 #include <linux/filter.h> 19 #include <linux/if_vlan.h> 20 #include <asm/kprobes.h> 21 #include <linux/bpf.h> 22 23 #include "bpf_jit64.h" 24 25 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 26 { 27 memset32(area, BREAKPOINT_INSTRUCTION, size/4); 28 } 29 30 static inline void bpf_flush_icache(void *start, void *end) 31 { 32 smp_wmb(); 33 flush_icache_range((unsigned long)start, (unsigned long)end); 34 } 35 36 static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) 37 { 38 return (ctx->seen & (1 << (31 - b2p[i]))); 39 } 40 41 static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) 42 { 43 ctx->seen |= (1 << (31 - b2p[i])); 44 } 45 46 static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 47 { 48 /* 49 * We only need a stack frame if: 50 * - we call other functions (kernel helpers), or 51 * - the bpf program uses its stack area 52 * The latter condition is deduced from the usage of BPF_REG_FP 53 */ 54 return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP); 55 } 56 57 /* 58 * When not setting up our own stackframe, the redzone usage is: 59 * 60 * [ prev sp ] <------------- 61 * [ ... ] | 62 * sp (r1) ---> [ stack pointer ] -------------- 63 * [ nv gpr save area ] 6*8 64 * [ tail_call_cnt ] 8 65 * [ local_tmp_var ] 8 66 * [ unused red zone ] 208 bytes protected 67 */ 68 static int bpf_jit_stack_local(struct codegen_context *ctx) 69 { 70 if (bpf_has_stack_frame(ctx)) 71 return STACK_FRAME_MIN_SIZE + ctx->stack_size; 72 else 73 return -(BPF_PPC_STACK_SAVE + 16); 74 } 75 76 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) 77 { 78 return bpf_jit_stack_local(ctx) + 8; 79 } 80 81 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 82 { 83 if (reg >= BPF_PPC_NVR_MIN && reg < 32) 84 return (bpf_has_stack_frame(ctx) ? 85 (BPF_PPC_STACKFRAME + ctx->stack_size) : 0) 86 - (8 * (32 - reg)); 87 88 pr_err("BPF JIT is asking about unknown registers"); 89 BUG(); 90 } 91 92 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 93 { 94 int i; 95 96 /* 97 * Initialize tail_call_cnt if we do tail calls. 98 * Otherwise, put in NOPs so that it can be skipped when we are 99 * invoked through a tail call. 100 */ 101 if (ctx->seen & SEEN_TAILCALL) { 102 PPC_LI(b2p[TMP_REG_1], 0); 103 /* this goes in the redzone */ 104 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8)); 105 } else { 106 PPC_NOP(); 107 PPC_NOP(); 108 } 109 110 #define BPF_TAILCALL_PROLOGUE_SIZE 8 111 112 if (bpf_has_stack_frame(ctx)) { 113 /* 114 * We need a stack frame, but we don't necessarily need to 115 * save/restore LR unless we call other functions 116 */ 117 if (ctx->seen & SEEN_FUNC) { 118 EMIT(PPC_INST_MFLR | __PPC_RT(R0)); 119 PPC_BPF_STL(0, 1, PPC_LR_STKOFF); 120 } 121 122 PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size)); 123 } 124 125 /* 126 * Back up non-volatile regs -- BPF registers 6-10 127 * If we haven't created our own stack frame, we save these 128 * in the protected zone below the previous stack frame 129 */ 130 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 131 if (bpf_is_seen_register(ctx, i)) 132 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 133 134 /* Setup frame pointer to point to the bpf stack area */ 135 if (bpf_is_seen_register(ctx, BPF_REG_FP)) 136 PPC_ADDI(b2p[BPF_REG_FP], 1, 137 STACK_FRAME_MIN_SIZE + ctx->stack_size); 138 } 139 140 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 141 { 142 int i; 143 144 /* Restore NVRs */ 145 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 146 if (bpf_is_seen_register(ctx, i)) 147 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 148 149 /* Tear down our stack frame */ 150 if (bpf_has_stack_frame(ctx)) { 151 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size); 152 if (ctx->seen & SEEN_FUNC) { 153 PPC_BPF_LL(0, 1, PPC_LR_STKOFF); 154 PPC_MTLR(0); 155 } 156 } 157 } 158 159 static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 160 { 161 bpf_jit_emit_common_epilogue(image, ctx); 162 163 /* Move result to r3 */ 164 PPC_MR(3, b2p[BPF_REG_0]); 165 166 PPC_BLR(); 167 } 168 169 static void bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, 170 u64 func) 171 { 172 #ifdef PPC64_ELF_ABI_v1 173 /* func points to the function descriptor */ 174 PPC_LI64(b2p[TMP_REG_2], func); 175 /* Load actual entry point from function descriptor */ 176 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0); 177 /* ... and move it to LR */ 178 PPC_MTLR(b2p[TMP_REG_1]); 179 /* 180 * Load TOC from function descriptor at offset 8. 181 * We can clobber r2 since we get called through a 182 * function pointer (so caller will save/restore r2) 183 * and since we don't use a TOC ourself. 184 */ 185 PPC_BPF_LL(2, b2p[TMP_REG_2], 8); 186 #else 187 /* We can clobber r12 */ 188 PPC_FUNC_ADDR(12, func); 189 PPC_MTLR(12); 190 #endif 191 PPC_BLRL(); 192 } 193 194 static void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, 195 u64 func) 196 { 197 unsigned int i, ctx_idx = ctx->idx; 198 199 /* Load function address into r12 */ 200 PPC_LI64(12, func); 201 202 /* For bpf-to-bpf function calls, the callee's address is unknown 203 * until the last extra pass. As seen above, we use PPC_LI64() to 204 * load the callee's address, but this may optimize the number of 205 * instructions required based on the nature of the address. 206 * 207 * Since we don't want the number of instructions emitted to change, 208 * we pad the optimized PPC_LI64() call with NOPs to guarantee that 209 * we always have a five-instruction sequence, which is the maximum 210 * that PPC_LI64() can emit. 211 */ 212 for (i = ctx->idx - ctx_idx; i < 5; i++) 213 PPC_NOP(); 214 215 #ifdef PPC64_ELF_ABI_v1 216 /* 217 * Load TOC from function descriptor at offset 8. 218 * We can clobber r2 since we get called through a 219 * function pointer (so caller will save/restore r2) 220 * and since we don't use a TOC ourself. 221 */ 222 PPC_BPF_LL(2, 12, 8); 223 /* Load actual entry point from function descriptor */ 224 PPC_BPF_LL(12, 12, 0); 225 #endif 226 227 PPC_MTLR(12); 228 PPC_BLRL(); 229 } 230 231 static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 232 { 233 /* 234 * By now, the eBPF program has already setup parameters in r3, r4 and r5 235 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 236 * r4/BPF_REG_2 - pointer to bpf_array 237 * r5/BPF_REG_3 - index in bpf_array 238 */ 239 int b2p_bpf_array = b2p[BPF_REG_2]; 240 int b2p_index = b2p[BPF_REG_3]; 241 242 /* 243 * if (index >= array->map.max_entries) 244 * goto out; 245 */ 246 PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)); 247 PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31); 248 PPC_CMPLW(b2p_index, b2p[TMP_REG_1]); 249 PPC_BCC(COND_GE, out); 250 251 /* 252 * if (tail_call_cnt > MAX_TAIL_CALL_CNT) 253 * goto out; 254 */ 255 PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 256 PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT); 257 PPC_BCC(COND_GT, out); 258 259 /* 260 * tail_call_cnt++; 261 */ 262 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1); 263 PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 264 265 /* prog = array->ptrs[index]; */ 266 PPC_MULI(b2p[TMP_REG_1], b2p_index, 8); 267 PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array); 268 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); 269 270 /* 271 * if (prog == NULL) 272 * goto out; 273 */ 274 PPC_CMPLDI(b2p[TMP_REG_1], 0); 275 PPC_BCC(COND_EQ, out); 276 277 /* goto *(prog->bpf_func + prologue_size); */ 278 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); 279 #ifdef PPC64_ELF_ABI_v1 280 /* skip past the function descriptor */ 281 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 282 FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE); 283 #else 284 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE); 285 #endif 286 PPC_MTCTR(b2p[TMP_REG_1]); 287 288 /* tear down stack, restore NVRs, ... */ 289 bpf_jit_emit_common_epilogue(image, ctx); 290 291 PPC_BCTR(); 292 /* out: */ 293 } 294 295 /* Assemble the body code between the prologue & epilogue */ 296 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 297 struct codegen_context *ctx, 298 u32 *addrs, bool extra_pass) 299 { 300 const struct bpf_insn *insn = fp->insnsi; 301 int flen = fp->len; 302 int i, ret; 303 304 /* Start of epilogue code - will only be valid 2nd pass onwards */ 305 u32 exit_addr = addrs[flen]; 306 307 for (i = 0; i < flen; i++) { 308 u32 code = insn[i].code; 309 u32 dst_reg = b2p[insn[i].dst_reg]; 310 u32 src_reg = b2p[insn[i].src_reg]; 311 s16 off = insn[i].off; 312 s32 imm = insn[i].imm; 313 bool func_addr_fixed; 314 u64 func_addr; 315 u64 imm64; 316 u32 true_cond; 317 u32 tmp_idx; 318 319 /* 320 * addrs[] maps a BPF bytecode address into a real offset from 321 * the start of the body code. 322 */ 323 addrs[i] = ctx->idx * 4; 324 325 /* 326 * As an optimization, we note down which non-volatile registers 327 * are used so that we can only save/restore those in our 328 * prologue and epilogue. We do this here regardless of whether 329 * the actual BPF instruction uses src/dst registers or not 330 * (for instance, BPF_CALL does not use them). The expectation 331 * is that those instructions will have src_reg/dst_reg set to 332 * 0. Even otherwise, we just lose some prologue/epilogue 333 * optimization but everything else should work without 334 * any issues. 335 */ 336 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32) 337 bpf_set_seen_register(ctx, insn[i].dst_reg); 338 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32) 339 bpf_set_seen_register(ctx, insn[i].src_reg); 340 341 switch (code) { 342 /* 343 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 344 */ 345 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 346 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 347 PPC_ADD(dst_reg, dst_reg, src_reg); 348 goto bpf_alu32_trunc; 349 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 350 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 351 PPC_SUB(dst_reg, dst_reg, src_reg); 352 goto bpf_alu32_trunc; 353 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 354 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 355 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 356 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 357 if (BPF_OP(code) == BPF_SUB) 358 imm = -imm; 359 if (imm) { 360 if (imm >= -32768 && imm < 32768) 361 PPC_ADDI(dst_reg, dst_reg, IMM_L(imm)); 362 else { 363 PPC_LI32(b2p[TMP_REG_1], imm); 364 PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]); 365 } 366 } 367 goto bpf_alu32_trunc; 368 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 369 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 370 if (BPF_CLASS(code) == BPF_ALU) 371 PPC_MULW(dst_reg, dst_reg, src_reg); 372 else 373 PPC_MULD(dst_reg, dst_reg, src_reg); 374 goto bpf_alu32_trunc; 375 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 376 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 377 if (imm >= -32768 && imm < 32768) 378 PPC_MULI(dst_reg, dst_reg, IMM_L(imm)); 379 else { 380 PPC_LI32(b2p[TMP_REG_1], imm); 381 if (BPF_CLASS(code) == BPF_ALU) 382 PPC_MULW(dst_reg, dst_reg, 383 b2p[TMP_REG_1]); 384 else 385 PPC_MULD(dst_reg, dst_reg, 386 b2p[TMP_REG_1]); 387 } 388 goto bpf_alu32_trunc; 389 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 390 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 391 if (BPF_OP(code) == BPF_MOD) { 392 PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg); 393 PPC_MULW(b2p[TMP_REG_1], src_reg, 394 b2p[TMP_REG_1]); 395 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 396 } else 397 PPC_DIVWU(dst_reg, dst_reg, src_reg); 398 goto bpf_alu32_trunc; 399 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 400 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 401 if (BPF_OP(code) == BPF_MOD) { 402 PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg); 403 PPC_MULD(b2p[TMP_REG_1], src_reg, 404 b2p[TMP_REG_1]); 405 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 406 } else 407 PPC_DIVD(dst_reg, dst_reg, src_reg); 408 break; 409 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 410 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 411 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 412 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 413 if (imm == 0) 414 return -EINVAL; 415 else if (imm == 1) 416 goto bpf_alu32_trunc; 417 418 PPC_LI32(b2p[TMP_REG_1], imm); 419 switch (BPF_CLASS(code)) { 420 case BPF_ALU: 421 if (BPF_OP(code) == BPF_MOD) { 422 PPC_DIVWU(b2p[TMP_REG_2], dst_reg, 423 b2p[TMP_REG_1]); 424 PPC_MULW(b2p[TMP_REG_1], 425 b2p[TMP_REG_1], 426 b2p[TMP_REG_2]); 427 PPC_SUB(dst_reg, dst_reg, 428 b2p[TMP_REG_1]); 429 } else 430 PPC_DIVWU(dst_reg, dst_reg, 431 b2p[TMP_REG_1]); 432 break; 433 case BPF_ALU64: 434 if (BPF_OP(code) == BPF_MOD) { 435 PPC_DIVD(b2p[TMP_REG_2], dst_reg, 436 b2p[TMP_REG_1]); 437 PPC_MULD(b2p[TMP_REG_1], 438 b2p[TMP_REG_1], 439 b2p[TMP_REG_2]); 440 PPC_SUB(dst_reg, dst_reg, 441 b2p[TMP_REG_1]); 442 } else 443 PPC_DIVD(dst_reg, dst_reg, 444 b2p[TMP_REG_1]); 445 break; 446 } 447 goto bpf_alu32_trunc; 448 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 449 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 450 PPC_NEG(dst_reg, dst_reg); 451 goto bpf_alu32_trunc; 452 453 /* 454 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 455 */ 456 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 457 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 458 PPC_AND(dst_reg, dst_reg, src_reg); 459 goto bpf_alu32_trunc; 460 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 461 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 462 if (!IMM_H(imm)) 463 PPC_ANDI(dst_reg, dst_reg, IMM_L(imm)); 464 else { 465 /* Sign-extended */ 466 PPC_LI32(b2p[TMP_REG_1], imm); 467 PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]); 468 } 469 goto bpf_alu32_trunc; 470 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 471 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 472 PPC_OR(dst_reg, dst_reg, src_reg); 473 goto bpf_alu32_trunc; 474 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 475 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 476 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 477 /* Sign-extended */ 478 PPC_LI32(b2p[TMP_REG_1], imm); 479 PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]); 480 } else { 481 if (IMM_L(imm)) 482 PPC_ORI(dst_reg, dst_reg, IMM_L(imm)); 483 if (IMM_H(imm)) 484 PPC_ORIS(dst_reg, dst_reg, IMM_H(imm)); 485 } 486 goto bpf_alu32_trunc; 487 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 488 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 489 PPC_XOR(dst_reg, dst_reg, src_reg); 490 goto bpf_alu32_trunc; 491 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 492 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 493 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 494 /* Sign-extended */ 495 PPC_LI32(b2p[TMP_REG_1], imm); 496 PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]); 497 } else { 498 if (IMM_L(imm)) 499 PPC_XORI(dst_reg, dst_reg, IMM_L(imm)); 500 if (IMM_H(imm)) 501 PPC_XORIS(dst_reg, dst_reg, IMM_H(imm)); 502 } 503 goto bpf_alu32_trunc; 504 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 505 /* slw clears top 32 bits */ 506 PPC_SLW(dst_reg, dst_reg, src_reg); 507 /* skip zero extension move, but set address map. */ 508 if (insn_is_zext(&insn[i + 1])) 509 addrs[++i] = ctx->idx * 4; 510 break; 511 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 512 PPC_SLD(dst_reg, dst_reg, src_reg); 513 break; 514 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */ 515 /* with imm 0, we still need to clear top 32 bits */ 516 PPC_SLWI(dst_reg, dst_reg, imm); 517 if (insn_is_zext(&insn[i + 1])) 518 addrs[++i] = ctx->idx * 4; 519 break; 520 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */ 521 if (imm != 0) 522 PPC_SLDI(dst_reg, dst_reg, imm); 523 break; 524 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 525 PPC_SRW(dst_reg, dst_reg, src_reg); 526 if (insn_is_zext(&insn[i + 1])) 527 addrs[++i] = ctx->idx * 4; 528 break; 529 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 530 PPC_SRD(dst_reg, dst_reg, src_reg); 531 break; 532 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 533 PPC_SRWI(dst_reg, dst_reg, imm); 534 if (insn_is_zext(&insn[i + 1])) 535 addrs[++i] = ctx->idx * 4; 536 break; 537 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 538 if (imm != 0) 539 PPC_SRDI(dst_reg, dst_reg, imm); 540 break; 541 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ 542 PPC_SRAW(dst_reg, dst_reg, src_reg); 543 goto bpf_alu32_trunc; 544 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 545 PPC_SRAD(dst_reg, dst_reg, src_reg); 546 break; 547 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ 548 PPC_SRAWI(dst_reg, dst_reg, imm); 549 goto bpf_alu32_trunc; 550 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 551 if (imm != 0) 552 PPC_SRADI(dst_reg, dst_reg, imm); 553 break; 554 555 /* 556 * MOV 557 */ 558 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 559 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 560 if (imm == 1) { 561 /* special mov32 for zext */ 562 PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31); 563 break; 564 } 565 PPC_MR(dst_reg, src_reg); 566 goto bpf_alu32_trunc; 567 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 568 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 569 PPC_LI32(dst_reg, imm); 570 if (imm < 0) 571 goto bpf_alu32_trunc; 572 else if (insn_is_zext(&insn[i + 1])) 573 addrs[++i] = ctx->idx * 4; 574 break; 575 576 bpf_alu32_trunc: 577 /* Truncate to 32-bits */ 578 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext) 579 PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31); 580 break; 581 582 /* 583 * BPF_FROM_BE/LE 584 */ 585 case BPF_ALU | BPF_END | BPF_FROM_LE: 586 case BPF_ALU | BPF_END | BPF_FROM_BE: 587 #ifdef __BIG_ENDIAN__ 588 if (BPF_SRC(code) == BPF_FROM_BE) 589 goto emit_clear; 590 #else /* !__BIG_ENDIAN__ */ 591 if (BPF_SRC(code) == BPF_FROM_LE) 592 goto emit_clear; 593 #endif 594 switch (imm) { 595 case 16: 596 /* Rotate 8 bits left & mask with 0x0000ff00 */ 597 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23); 598 /* Rotate 8 bits right & insert LSB to reg */ 599 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31); 600 /* Move result back to dst_reg */ 601 PPC_MR(dst_reg, b2p[TMP_REG_1]); 602 break; 603 case 32: 604 /* 605 * Rotate word left by 8 bits: 606 * 2 bytes are already in their final position 607 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 608 */ 609 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31); 610 /* Rotate 24 bits and insert byte 1 */ 611 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7); 612 /* Rotate 24 bits and insert byte 3 */ 613 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23); 614 PPC_MR(dst_reg, b2p[TMP_REG_1]); 615 break; 616 case 64: 617 /* 618 * Way easier and faster(?) to store the value 619 * into stack and then use ldbrx 620 * 621 * ctx->seen will be reliable in pass2, but 622 * the instructions generated will remain the 623 * same across all passes 624 */ 625 PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx)); 626 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)); 627 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]); 628 break; 629 } 630 break; 631 632 emit_clear: 633 switch (imm) { 634 case 16: 635 /* zero-extend 16 bits into 64 bits */ 636 PPC_RLDICL(dst_reg, dst_reg, 0, 48); 637 if (insn_is_zext(&insn[i + 1])) 638 addrs[++i] = ctx->idx * 4; 639 break; 640 case 32: 641 if (!fp->aux->verifier_zext) 642 /* zero-extend 32 bits into 64 bits */ 643 PPC_RLDICL(dst_reg, dst_reg, 0, 32); 644 break; 645 case 64: 646 /* nop */ 647 break; 648 } 649 break; 650 651 /* 652 * BPF_ST(X) 653 */ 654 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 655 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 656 if (BPF_CLASS(code) == BPF_ST) { 657 PPC_LI(b2p[TMP_REG_1], imm); 658 src_reg = b2p[TMP_REG_1]; 659 } 660 PPC_STB(src_reg, dst_reg, off); 661 break; 662 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 663 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 664 if (BPF_CLASS(code) == BPF_ST) { 665 PPC_LI(b2p[TMP_REG_1], imm); 666 src_reg = b2p[TMP_REG_1]; 667 } 668 PPC_STH(src_reg, dst_reg, off); 669 break; 670 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 671 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 672 if (BPF_CLASS(code) == BPF_ST) { 673 PPC_LI32(b2p[TMP_REG_1], imm); 674 src_reg = b2p[TMP_REG_1]; 675 } 676 PPC_STW(src_reg, dst_reg, off); 677 break; 678 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 679 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 680 if (BPF_CLASS(code) == BPF_ST) { 681 PPC_LI32(b2p[TMP_REG_1], imm); 682 src_reg = b2p[TMP_REG_1]; 683 } 684 PPC_BPF_STL(src_reg, dst_reg, off); 685 break; 686 687 /* 688 * BPF_STX XADD (atomic_add) 689 */ 690 /* *(u32 *)(dst + off) += src */ 691 case BPF_STX | BPF_XADD | BPF_W: 692 /* Get EA into TMP_REG_1 */ 693 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 694 tmp_idx = ctx->idx * 4; 695 /* load value from memory into TMP_REG_2 */ 696 PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 697 /* add value from src_reg into this */ 698 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 699 /* store result back */ 700 PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 701 /* we're done if this succeeded */ 702 PPC_BCC_SHORT(COND_NE, tmp_idx); 703 break; 704 /* *(u64 *)(dst + off) += src */ 705 case BPF_STX | BPF_XADD | BPF_DW: 706 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 707 tmp_idx = ctx->idx * 4; 708 PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 709 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 710 PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 711 PPC_BCC_SHORT(COND_NE, tmp_idx); 712 break; 713 714 /* 715 * BPF_LDX 716 */ 717 /* dst = *(u8 *)(ul) (src + off) */ 718 case BPF_LDX | BPF_MEM | BPF_B: 719 PPC_LBZ(dst_reg, src_reg, off); 720 if (insn_is_zext(&insn[i + 1])) 721 addrs[++i] = ctx->idx * 4; 722 break; 723 /* dst = *(u16 *)(ul) (src + off) */ 724 case BPF_LDX | BPF_MEM | BPF_H: 725 PPC_LHZ(dst_reg, src_reg, off); 726 if (insn_is_zext(&insn[i + 1])) 727 addrs[++i] = ctx->idx * 4; 728 break; 729 /* dst = *(u32 *)(ul) (src + off) */ 730 case BPF_LDX | BPF_MEM | BPF_W: 731 PPC_LWZ(dst_reg, src_reg, off); 732 if (insn_is_zext(&insn[i + 1])) 733 addrs[++i] = ctx->idx * 4; 734 break; 735 /* dst = *(u64 *)(ul) (src + off) */ 736 case BPF_LDX | BPF_MEM | BPF_DW: 737 PPC_BPF_LL(dst_reg, src_reg, off); 738 break; 739 740 /* 741 * Doubleword load 742 * 16 byte instruction that uses two 'struct bpf_insn' 743 */ 744 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 745 imm64 = ((u64)(u32) insn[i].imm) | 746 (((u64)(u32) insn[i+1].imm) << 32); 747 /* Adjust for two bpf instructions */ 748 addrs[++i] = ctx->idx * 4; 749 PPC_LI64(dst_reg, imm64); 750 break; 751 752 /* 753 * Return/Exit 754 */ 755 case BPF_JMP | BPF_EXIT: 756 /* 757 * If this isn't the very last instruction, branch to 758 * the epilogue. If we _are_ the last instruction, 759 * we'll just fall through to the epilogue. 760 */ 761 if (i != flen - 1) 762 PPC_JMP(exit_addr); 763 /* else fall through to the epilogue */ 764 break; 765 766 /* 767 * Call kernel helper or bpf function 768 */ 769 case BPF_JMP | BPF_CALL: 770 ctx->seen |= SEEN_FUNC; 771 772 ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, 773 &func_addr, &func_addr_fixed); 774 if (ret < 0) 775 return ret; 776 777 if (func_addr_fixed) 778 bpf_jit_emit_func_call_hlp(image, ctx, func_addr); 779 else 780 bpf_jit_emit_func_call_rel(image, ctx, func_addr); 781 /* move return value from r3 to BPF_REG_0 */ 782 PPC_MR(b2p[BPF_REG_0], 3); 783 break; 784 785 /* 786 * Jumps and branches 787 */ 788 case BPF_JMP | BPF_JA: 789 PPC_JMP(addrs[i + 1 + off]); 790 break; 791 792 case BPF_JMP | BPF_JGT | BPF_K: 793 case BPF_JMP | BPF_JGT | BPF_X: 794 case BPF_JMP | BPF_JSGT | BPF_K: 795 case BPF_JMP | BPF_JSGT | BPF_X: 796 case BPF_JMP32 | BPF_JGT | BPF_K: 797 case BPF_JMP32 | BPF_JGT | BPF_X: 798 case BPF_JMP32 | BPF_JSGT | BPF_K: 799 case BPF_JMP32 | BPF_JSGT | BPF_X: 800 true_cond = COND_GT; 801 goto cond_branch; 802 case BPF_JMP | BPF_JLT | BPF_K: 803 case BPF_JMP | BPF_JLT | BPF_X: 804 case BPF_JMP | BPF_JSLT | BPF_K: 805 case BPF_JMP | BPF_JSLT | BPF_X: 806 case BPF_JMP32 | BPF_JLT | BPF_K: 807 case BPF_JMP32 | BPF_JLT | BPF_X: 808 case BPF_JMP32 | BPF_JSLT | BPF_K: 809 case BPF_JMP32 | BPF_JSLT | BPF_X: 810 true_cond = COND_LT; 811 goto cond_branch; 812 case BPF_JMP | BPF_JGE | BPF_K: 813 case BPF_JMP | BPF_JGE | BPF_X: 814 case BPF_JMP | BPF_JSGE | BPF_K: 815 case BPF_JMP | BPF_JSGE | BPF_X: 816 case BPF_JMP32 | BPF_JGE | BPF_K: 817 case BPF_JMP32 | BPF_JGE | BPF_X: 818 case BPF_JMP32 | BPF_JSGE | BPF_K: 819 case BPF_JMP32 | BPF_JSGE | BPF_X: 820 true_cond = COND_GE; 821 goto cond_branch; 822 case BPF_JMP | BPF_JLE | BPF_K: 823 case BPF_JMP | BPF_JLE | BPF_X: 824 case BPF_JMP | BPF_JSLE | BPF_K: 825 case BPF_JMP | BPF_JSLE | BPF_X: 826 case BPF_JMP32 | BPF_JLE | BPF_K: 827 case BPF_JMP32 | BPF_JLE | BPF_X: 828 case BPF_JMP32 | BPF_JSLE | BPF_K: 829 case BPF_JMP32 | BPF_JSLE | BPF_X: 830 true_cond = COND_LE; 831 goto cond_branch; 832 case BPF_JMP | BPF_JEQ | BPF_K: 833 case BPF_JMP | BPF_JEQ | BPF_X: 834 case BPF_JMP32 | BPF_JEQ | BPF_K: 835 case BPF_JMP32 | BPF_JEQ | BPF_X: 836 true_cond = COND_EQ; 837 goto cond_branch; 838 case BPF_JMP | BPF_JNE | BPF_K: 839 case BPF_JMP | BPF_JNE | BPF_X: 840 case BPF_JMP32 | BPF_JNE | BPF_K: 841 case BPF_JMP32 | BPF_JNE | BPF_X: 842 true_cond = COND_NE; 843 goto cond_branch; 844 case BPF_JMP | BPF_JSET | BPF_K: 845 case BPF_JMP | BPF_JSET | BPF_X: 846 case BPF_JMP32 | BPF_JSET | BPF_K: 847 case BPF_JMP32 | BPF_JSET | BPF_X: 848 true_cond = COND_NE; 849 /* Fall through */ 850 851 cond_branch: 852 switch (code) { 853 case BPF_JMP | BPF_JGT | BPF_X: 854 case BPF_JMP | BPF_JLT | BPF_X: 855 case BPF_JMP | BPF_JGE | BPF_X: 856 case BPF_JMP | BPF_JLE | BPF_X: 857 case BPF_JMP | BPF_JEQ | BPF_X: 858 case BPF_JMP | BPF_JNE | BPF_X: 859 case BPF_JMP32 | BPF_JGT | BPF_X: 860 case BPF_JMP32 | BPF_JLT | BPF_X: 861 case BPF_JMP32 | BPF_JGE | BPF_X: 862 case BPF_JMP32 | BPF_JLE | BPF_X: 863 case BPF_JMP32 | BPF_JEQ | BPF_X: 864 case BPF_JMP32 | BPF_JNE | BPF_X: 865 /* unsigned comparison */ 866 if (BPF_CLASS(code) == BPF_JMP32) 867 PPC_CMPLW(dst_reg, src_reg); 868 else 869 PPC_CMPLD(dst_reg, src_reg); 870 break; 871 case BPF_JMP | BPF_JSGT | BPF_X: 872 case BPF_JMP | BPF_JSLT | BPF_X: 873 case BPF_JMP | BPF_JSGE | BPF_X: 874 case BPF_JMP | BPF_JSLE | BPF_X: 875 case BPF_JMP32 | BPF_JSGT | BPF_X: 876 case BPF_JMP32 | BPF_JSLT | BPF_X: 877 case BPF_JMP32 | BPF_JSGE | BPF_X: 878 case BPF_JMP32 | BPF_JSLE | BPF_X: 879 /* signed comparison */ 880 if (BPF_CLASS(code) == BPF_JMP32) 881 PPC_CMPW(dst_reg, src_reg); 882 else 883 PPC_CMPD(dst_reg, src_reg); 884 break; 885 case BPF_JMP | BPF_JSET | BPF_X: 886 case BPF_JMP32 | BPF_JSET | BPF_X: 887 if (BPF_CLASS(code) == BPF_JMP) { 888 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, 889 src_reg); 890 } else { 891 int tmp_reg = b2p[TMP_REG_1]; 892 893 PPC_AND(tmp_reg, dst_reg, src_reg); 894 PPC_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0, 895 31); 896 } 897 break; 898 case BPF_JMP | BPF_JNE | BPF_K: 899 case BPF_JMP | BPF_JEQ | BPF_K: 900 case BPF_JMP | BPF_JGT | BPF_K: 901 case BPF_JMP | BPF_JLT | BPF_K: 902 case BPF_JMP | BPF_JGE | BPF_K: 903 case BPF_JMP | BPF_JLE | BPF_K: 904 case BPF_JMP32 | BPF_JNE | BPF_K: 905 case BPF_JMP32 | BPF_JEQ | BPF_K: 906 case BPF_JMP32 | BPF_JGT | BPF_K: 907 case BPF_JMP32 | BPF_JLT | BPF_K: 908 case BPF_JMP32 | BPF_JGE | BPF_K: 909 case BPF_JMP32 | BPF_JLE | BPF_K: 910 { 911 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32; 912 913 /* 914 * Need sign-extended load, so only positive 915 * values can be used as imm in cmpldi 916 */ 917 if (imm >= 0 && imm < 32768) { 918 if (is_jmp32) 919 PPC_CMPLWI(dst_reg, imm); 920 else 921 PPC_CMPLDI(dst_reg, imm); 922 } else { 923 /* sign-extending load */ 924 PPC_LI32(b2p[TMP_REG_1], imm); 925 /* ... but unsigned comparison */ 926 if (is_jmp32) 927 PPC_CMPLW(dst_reg, 928 b2p[TMP_REG_1]); 929 else 930 PPC_CMPLD(dst_reg, 931 b2p[TMP_REG_1]); 932 } 933 break; 934 } 935 case BPF_JMP | BPF_JSGT | BPF_K: 936 case BPF_JMP | BPF_JSLT | BPF_K: 937 case BPF_JMP | BPF_JSGE | BPF_K: 938 case BPF_JMP | BPF_JSLE | BPF_K: 939 case BPF_JMP32 | BPF_JSGT | BPF_K: 940 case BPF_JMP32 | BPF_JSLT | BPF_K: 941 case BPF_JMP32 | BPF_JSGE | BPF_K: 942 case BPF_JMP32 | BPF_JSLE | BPF_K: 943 { 944 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32; 945 946 /* 947 * signed comparison, so any 16-bit value 948 * can be used in cmpdi 949 */ 950 if (imm >= -32768 && imm < 32768) { 951 if (is_jmp32) 952 PPC_CMPWI(dst_reg, imm); 953 else 954 PPC_CMPDI(dst_reg, imm); 955 } else { 956 PPC_LI32(b2p[TMP_REG_1], imm); 957 if (is_jmp32) 958 PPC_CMPW(dst_reg, 959 b2p[TMP_REG_1]); 960 else 961 PPC_CMPD(dst_reg, 962 b2p[TMP_REG_1]); 963 } 964 break; 965 } 966 case BPF_JMP | BPF_JSET | BPF_K: 967 case BPF_JMP32 | BPF_JSET | BPF_K: 968 /* andi does not sign-extend the immediate */ 969 if (imm >= 0 && imm < 32768) 970 /* PPC_ANDI is _only/always_ dot-form */ 971 PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm); 972 else { 973 int tmp_reg = b2p[TMP_REG_1]; 974 975 PPC_LI32(tmp_reg, imm); 976 if (BPF_CLASS(code) == BPF_JMP) { 977 PPC_AND_DOT(tmp_reg, dst_reg, 978 tmp_reg); 979 } else { 980 PPC_AND(tmp_reg, dst_reg, 981 tmp_reg); 982 PPC_RLWINM_DOT(tmp_reg, tmp_reg, 983 0, 0, 31); 984 } 985 } 986 break; 987 } 988 PPC_BCC(true_cond, addrs[i + 1 + off]); 989 break; 990 991 /* 992 * Tail call 993 */ 994 case BPF_JMP | BPF_TAIL_CALL: 995 ctx->seen |= SEEN_TAILCALL; 996 bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 997 break; 998 999 default: 1000 /* 1001 * The filter contains something cruel & unusual. 1002 * We don't handle it, but also there shouldn't be 1003 * anything missing from our list. 1004 */ 1005 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", 1006 code, i); 1007 return -ENOTSUPP; 1008 } 1009 } 1010 1011 /* Set end-of-body-code address for exit. */ 1012 addrs[i] = ctx->idx * 4; 1013 1014 return 0; 1015 } 1016 1017 /* Fix the branch target addresses for subprog calls */ 1018 static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image, 1019 struct codegen_context *ctx, u32 *addrs) 1020 { 1021 const struct bpf_insn *insn = fp->insnsi; 1022 bool func_addr_fixed; 1023 u64 func_addr; 1024 u32 tmp_idx; 1025 int i, ret; 1026 1027 for (i = 0; i < fp->len; i++) { 1028 /* 1029 * During the extra pass, only the branch target addresses for 1030 * the subprog calls need to be fixed. All other instructions 1031 * can left untouched. 1032 * 1033 * The JITed image length does not change because we already 1034 * ensure that the JITed instruction sequence for these calls 1035 * are of fixed length by padding them with NOPs. 1036 */ 1037 if (insn[i].code == (BPF_JMP | BPF_CALL) && 1038 insn[i].src_reg == BPF_PSEUDO_CALL) { 1039 ret = bpf_jit_get_func_addr(fp, &insn[i], true, 1040 &func_addr, 1041 &func_addr_fixed); 1042 if (ret < 0) 1043 return ret; 1044 1045 /* 1046 * Save ctx->idx as this would currently point to the 1047 * end of the JITed image and set it to the offset of 1048 * the instruction sequence corresponding to the 1049 * subprog call temporarily. 1050 */ 1051 tmp_idx = ctx->idx; 1052 ctx->idx = addrs[i] / 4; 1053 bpf_jit_emit_func_call_rel(image, ctx, func_addr); 1054 1055 /* 1056 * Restore ctx->idx here. This is safe as the length 1057 * of the JITed sequence remains unchanged. 1058 */ 1059 ctx->idx = tmp_idx; 1060 } 1061 } 1062 1063 return 0; 1064 } 1065 1066 struct powerpc64_jit_data { 1067 struct bpf_binary_header *header; 1068 u32 *addrs; 1069 u8 *image; 1070 u32 proglen; 1071 struct codegen_context ctx; 1072 }; 1073 1074 bool bpf_jit_needs_zext(void) 1075 { 1076 return true; 1077 } 1078 1079 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 1080 { 1081 u32 proglen; 1082 u32 alloclen; 1083 u8 *image = NULL; 1084 u32 *code_base; 1085 u32 *addrs; 1086 struct powerpc64_jit_data *jit_data; 1087 struct codegen_context cgctx; 1088 int pass; 1089 int flen; 1090 struct bpf_binary_header *bpf_hdr; 1091 struct bpf_prog *org_fp = fp; 1092 struct bpf_prog *tmp_fp; 1093 bool bpf_blinded = false; 1094 bool extra_pass = false; 1095 1096 if (!fp->jit_requested) 1097 return org_fp; 1098 1099 tmp_fp = bpf_jit_blind_constants(org_fp); 1100 if (IS_ERR(tmp_fp)) 1101 return org_fp; 1102 1103 if (tmp_fp != org_fp) { 1104 bpf_blinded = true; 1105 fp = tmp_fp; 1106 } 1107 1108 jit_data = fp->aux->jit_data; 1109 if (!jit_data) { 1110 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1111 if (!jit_data) { 1112 fp = org_fp; 1113 goto out; 1114 } 1115 fp->aux->jit_data = jit_data; 1116 } 1117 1118 flen = fp->len; 1119 addrs = jit_data->addrs; 1120 if (addrs) { 1121 cgctx = jit_data->ctx; 1122 image = jit_data->image; 1123 bpf_hdr = jit_data->header; 1124 proglen = jit_data->proglen; 1125 alloclen = proglen + FUNCTION_DESCR_SIZE; 1126 extra_pass = true; 1127 goto skip_init_ctx; 1128 } 1129 1130 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 1131 if (addrs == NULL) { 1132 fp = org_fp; 1133 goto out_addrs; 1134 } 1135 1136 memset(&cgctx, 0, sizeof(struct codegen_context)); 1137 1138 /* Make sure that the stack is quadword aligned. */ 1139 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 1140 1141 /* Scouting faux-generate pass 0 */ 1142 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 1143 /* We hit something illegal or unsupported. */ 1144 fp = org_fp; 1145 goto out_addrs; 1146 } 1147 1148 /* 1149 * Pretend to build prologue, given the features we've seen. This will 1150 * update ctgtx.idx as it pretends to output instructions, then we can 1151 * calculate total size from idx. 1152 */ 1153 bpf_jit_build_prologue(0, &cgctx); 1154 bpf_jit_build_epilogue(0, &cgctx); 1155 1156 proglen = cgctx.idx * 4; 1157 alloclen = proglen + FUNCTION_DESCR_SIZE; 1158 1159 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, 1160 bpf_jit_fill_ill_insns); 1161 if (!bpf_hdr) { 1162 fp = org_fp; 1163 goto out_addrs; 1164 } 1165 1166 skip_init_ctx: 1167 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 1168 1169 if (extra_pass) { 1170 /* 1171 * Do not touch the prologue and epilogue as they will remain 1172 * unchanged. Only fix the branch target address for subprog 1173 * calls in the body. 1174 * 1175 * This does not change the offsets and lengths of the subprog 1176 * call instruction sequences and hence, the size of the JITed 1177 * image as well. 1178 */ 1179 bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs); 1180 1181 /* There is no need to perform the usual passes. */ 1182 goto skip_codegen_passes; 1183 } 1184 1185 /* Code generation passes 1-2 */ 1186 for (pass = 1; pass < 3; pass++) { 1187 /* Now build the prologue, body code & epilogue for real. */ 1188 cgctx.idx = 0; 1189 bpf_jit_build_prologue(code_base, &cgctx); 1190 bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass); 1191 bpf_jit_build_epilogue(code_base, &cgctx); 1192 1193 if (bpf_jit_enable > 1) 1194 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 1195 proglen - (cgctx.idx * 4), cgctx.seen); 1196 } 1197 1198 skip_codegen_passes: 1199 if (bpf_jit_enable > 1) 1200 /* 1201 * Note that we output the base address of the code_base 1202 * rather than image, since opcodes are in code_base. 1203 */ 1204 bpf_jit_dump(flen, proglen, pass, code_base); 1205 1206 #ifdef PPC64_ELF_ABI_v1 1207 /* Function descriptor nastiness: Address + TOC */ 1208 ((u64 *)image)[0] = (u64)code_base; 1209 ((u64 *)image)[1] = local_paca->kernel_toc; 1210 #endif 1211 1212 fp->bpf_func = (void *)image; 1213 fp->jited = 1; 1214 fp->jited_len = alloclen; 1215 1216 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); 1217 if (!fp->is_func || extra_pass) { 1218 bpf_prog_fill_jited_linfo(fp, addrs); 1219 out_addrs: 1220 kfree(addrs); 1221 kfree(jit_data); 1222 fp->aux->jit_data = NULL; 1223 } else { 1224 jit_data->addrs = addrs; 1225 jit_data->ctx = cgctx; 1226 jit_data->proglen = proglen; 1227 jit_data->image = image; 1228 jit_data->header = bpf_hdr; 1229 } 1230 1231 out: 1232 if (bpf_blinded) 1233 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 1234 1235 return fp; 1236 } 1237 1238 /* Overriding bpf_jit_free() as we don't set images read-only. */ 1239 void bpf_jit_free(struct bpf_prog *fp) 1240 { 1241 unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 1242 struct bpf_binary_header *bpf_hdr = (void *)addr; 1243 1244 if (fp->jited) 1245 bpf_jit_binary_free(bpf_hdr); 1246 1247 bpf_prog_unlock_free(fp); 1248 } 1249