1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include <linux/memory.h> 12 #include <asm/text-patching.h> 13 #include <asm/cfi.h> 14 #include "bpf_jit.h" 15 16 /* Number of iterations to try until offsets converge. */ 17 #define NR_JIT_ITERATIONS 32 18 19 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) 20 { 21 const struct bpf_prog *prog = ctx->prog; 22 int i; 23 24 for (i = 0; i < prog->len; i++) { 25 const struct bpf_insn *insn = &prog->insnsi[i]; 26 int ret; 27 28 ret = bpf_jit_emit_insn(insn, ctx, extra_pass); 29 if (ret > 0) 30 i++; /* skip the next instruction */ 31 if (offset) 32 offset[i] = ctx->ninsns; 33 if (ret < 0) 34 return ret; 35 } 36 return 0; 37 } 38 39 bool bpf_jit_needs_zext(void) 40 { 41 return true; 42 } 43 44 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 45 { 46 unsigned int prog_size = 0, extable_size = 0; 47 bool extra_pass = false; 48 int pass = 0, prev_ninsns = 0, i; 49 struct rv_jit_data *jit_data; 50 struct rv_jit_context *ctx; 51 52 if (!prog->jit_requested) 53 return prog; 54 55 jit_data = prog->aux->jit_data; 56 if (!jit_data) { 57 jit_data = kzalloc_obj(*jit_data); 58 if (!jit_data) { 59 return prog; 60 } 61 prog->aux->jit_data = jit_data; 62 } 63 64 ctx = &jit_data->ctx; 65 66 if (ctx->offset) { 67 extra_pass = true; 68 prog_size = sizeof(*ctx->insns) * ctx->ninsns; 69 goto skip_init_ctx; 70 } 71 72 ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); 73 ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); 74 ctx->prog = prog; 75 ctx->offset = kzalloc_objs(int, prog->len); 76 if (!ctx->offset) 77 goto out_offset; 78 79 if (build_body(ctx, extra_pass, NULL)) 80 goto out_offset; 81 82 for (i = 0; i < prog->len; i++) { 83 prev_ninsns += 32; 84 ctx->offset[i] = prev_ninsns; 85 } 86 87 for (i = 0; i < NR_JIT_ITERATIONS; i++) { 88 pass++; 89 ctx->ninsns = 0; 90 91 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); 92 ctx->prologue_len = ctx->ninsns; 93 94 if (build_body(ctx, extra_pass, ctx->offset)) 95 goto out_offset; 96 97 ctx->epilogue_offset = ctx->ninsns; 98 bpf_jit_build_epilogue(ctx); 99 100 if (ctx->ninsns == prev_ninsns) { 101 if (jit_data->header) 102 break; 103 /* obtain the actual image size */ 104 extable_size = prog->aux->num_exentries * 105 sizeof(struct exception_table_entry); 106 prog_size = sizeof(*ctx->insns) * ctx->ninsns; 107 108 jit_data->ro_header = 109 bpf_jit_binary_pack_alloc(prog_size + extable_size, 110 &jit_data->ro_image, sizeof(u32), 111 &jit_data->header, &jit_data->image, 112 bpf_fill_ill_insns, 113 bpf_prog_was_classic(prog)); 114 if (!jit_data->ro_header) 115 goto out_offset; 116 117 /* 118 * Use the image(RW) for writing the JITed instructions. But also save 119 * the ro_image(RX) for calculating the offsets in the image. The RW 120 * image will be later copied to the RX image from where the program 121 * will run. The bpf_jit_binary_pack_finalize() will do this copy in the 122 * final step. 123 */ 124 ctx->ro_insns = (u16 *)jit_data->ro_image; 125 ctx->insns = (u16 *)jit_data->image; 126 /* 127 * Now, when the image is allocated, the image can 128 * potentially shrink more (auipc/jalr -> jal). 129 */ 130 } 131 prev_ninsns = ctx->ninsns; 132 } 133 134 if (i == NR_JIT_ITERATIONS) { 135 pr_err("bpf-jit: image did not converge in <%d passes!\n", i); 136 goto out_free_hdr; 137 } 138 139 if (extable_size) 140 prog->aux->extable = (void *)ctx->ro_insns + prog_size; 141 142 skip_init_ctx: 143 pass++; 144 ctx->ninsns = 0; 145 ctx->nexentries = 0; 146 147 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); 148 if (build_body(ctx, extra_pass, NULL)) 149 goto out_free_hdr; 150 bpf_jit_build_epilogue(ctx); 151 152 if (bpf_jit_enable > 1) 153 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); 154 155 if (!prog->is_func || extra_pass) { 156 if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) { 157 /* ro_header has been freed */ 158 jit_data->ro_header = NULL; 159 jit_data->header = NULL; 160 goto out_free_hdr; 161 } 162 } 163 164 prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); 165 prog->jited = 1; 166 prog->jited_len = prog_size - cfi_get_offset(); 167 168 if (!prog->is_func || extra_pass) { 169 for (i = 0; i < prog->len; i++) 170 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); 171 bpf_prog_fill_jited_linfo(prog, ctx->offset); 172 out_offset: 173 kfree(ctx->offset); 174 kfree(jit_data); 175 prog->aux->jit_data = NULL; 176 } 177 178 return prog; 179 180 out_free_hdr: 181 if (extra_pass) { 182 prog->bpf_func = NULL; 183 prog->jited = 0; 184 prog->jited_len = 0; 185 } 186 if (jit_data->header) { 187 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size, 188 sizeof(jit_data->header->size)); 189 bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header); 190 } 191 goto out_offset; 192 } 193 194 u64 bpf_jit_alloc_exec_limit(void) 195 { 196 return BPF_JIT_REGION_SIZE; 197 } 198 199 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 200 { 201 int ret; 202 203 mutex_lock(&text_mutex); 204 ret = patch_text_nosync(dst, src, len); 205 mutex_unlock(&text_mutex); 206 207 if (ret) 208 return ERR_PTR(-EINVAL); 209 210 return dst; 211 } 212 213 int bpf_arch_text_invalidate(void *dst, size_t len) 214 { 215 int ret; 216 217 mutex_lock(&text_mutex); 218 ret = patch_text_set_nosync(dst, 0, len); 219 mutex_unlock(&text_mutex); 220 221 return ret; 222 } 223 224 void bpf_jit_free(struct bpf_prog *prog) 225 { 226 if (prog->jited) { 227 struct rv_jit_data *jit_data = prog->aux->jit_data; 228 struct bpf_binary_header *hdr; 229 230 /* 231 * If we fail the final pass of JIT (from jit_subprogs), 232 * the program may not be finalized yet. Call finalize here 233 * before freeing it. 234 */ 235 if (jit_data) { 236 bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); 237 kfree(jit_data); 238 } 239 hdr = bpf_jit_binary_pack_hdr(prog); 240 bpf_jit_binary_pack_free(hdr, NULL); 241 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); 242 } 243 244 bpf_prog_unlock_free(prog); 245 } 246