1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include <linux/memory.h> 12 #include <asm/patch.h> 13 #include <asm/cfi.h> 14 #include "bpf_jit.h" 15 16 /* Number of iterations to try until offsets converge. */ 17 #define NR_JIT_ITERATIONS 32 18 19 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) 20 { 21 const struct bpf_prog *prog = ctx->prog; 22 int i; 23 24 for (i = 0; i < prog->len; i++) { 25 const struct bpf_insn *insn = &prog->insnsi[i]; 26 int ret; 27 28 ret = bpf_jit_emit_insn(insn, ctx, extra_pass); 29 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */ 30 if (ret > 0) 31 i++; 32 if (offset) 33 offset[i] = ctx->ninsns; 34 if (ret < 0) 35 return ret; 36 } 37 return 0; 38 } 39 40 bool bpf_jit_needs_zext(void) 41 { 42 return true; 43 } 44 45 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 46 { 47 unsigned int prog_size = 0, extable_size = 0; 48 bool tmp_blinded = false, extra_pass = false; 49 struct bpf_prog *tmp, *orig_prog = prog; 50 int pass = 0, prev_ninsns = 0, i; 51 struct rv_jit_data *jit_data; 52 struct rv_jit_context *ctx; 53 54 if (!prog->jit_requested) 55 return orig_prog; 56 57 tmp = bpf_jit_blind_constants(prog); 58 if (IS_ERR(tmp)) 59 return orig_prog; 60 if (tmp != prog) { 61 tmp_blinded = true; 62 prog = tmp; 63 } 64 65 jit_data = prog->aux->jit_data; 66 if (!jit_data) { 67 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 68 if (!jit_data) { 69 prog = orig_prog; 70 goto out; 71 } 72 prog->aux->jit_data = jit_data; 73 } 74 75 ctx = &jit_data->ctx; 76 77 if (ctx->offset) { 78 extra_pass = true; 79 prog_size = sizeof(*ctx->insns) * ctx->ninsns; 80 goto skip_init_ctx; 81 } 82 83 ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); 84 ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); 85 ctx->prog = prog; 86 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); 87 if (!ctx->offset) { 88 prog = orig_prog; 89 goto out_offset; 90 } 91 92 if (build_body(ctx, extra_pass, NULL)) { 93 prog = orig_prog; 94 goto out_offset; 95 } 96 97 for (i = 0; i < prog->len; i++) { 98 prev_ninsns += 32; 99 ctx->offset[i] = prev_ninsns; 100 } 101 102 for (i = 0; i < NR_JIT_ITERATIONS; i++) { 103 pass++; 104 ctx->ninsns = 0; 105 106 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); 107 ctx->prologue_len = ctx->ninsns; 108 109 if (build_body(ctx, extra_pass, ctx->offset)) { 110 prog = orig_prog; 111 goto out_offset; 112 } 113 114 ctx->epilogue_offset = ctx->ninsns; 115 bpf_jit_build_epilogue(ctx); 116 117 if (ctx->ninsns == prev_ninsns) { 118 if (jit_data->header) 119 break; 120 /* obtain the actual image size */ 121 extable_size = prog->aux->num_exentries * 122 sizeof(struct exception_table_entry); 123 prog_size = sizeof(*ctx->insns) * ctx->ninsns; 124 125 jit_data->ro_header = 126 bpf_jit_binary_pack_alloc(prog_size + extable_size, 127 &jit_data->ro_image, sizeof(u32), 128 &jit_data->header, &jit_data->image, 129 bpf_fill_ill_insns); 130 if (!jit_data->ro_header) { 131 prog = orig_prog; 132 goto out_offset; 133 } 134 135 /* 136 * Use the image(RW) for writing the JITed instructions. But also save 137 * the ro_image(RX) for calculating the offsets in the image. The RW 138 * image will be later copied to the RX image from where the program 139 * will run. The bpf_jit_binary_pack_finalize() will do this copy in the 140 * final step. 141 */ 142 ctx->ro_insns = (u16 *)jit_data->ro_image; 143 ctx->insns = (u16 *)jit_data->image; 144 /* 145 * Now, when the image is allocated, the image can 146 * potentially shrink more (auipc/jalr -> jal). 147 */ 148 } 149 prev_ninsns = ctx->ninsns; 150 } 151 152 if (i == NR_JIT_ITERATIONS) { 153 pr_err("bpf-jit: image did not converge in <%d passes!\n", i); 154 prog = orig_prog; 155 goto out_free_hdr; 156 } 157 158 if (extable_size) 159 prog->aux->extable = (void *)ctx->ro_insns + prog_size; 160 161 skip_init_ctx: 162 pass++; 163 ctx->ninsns = 0; 164 ctx->nexentries = 0; 165 166 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); 167 if (build_body(ctx, extra_pass, NULL)) { 168 prog = orig_prog; 169 goto out_free_hdr; 170 } 171 bpf_jit_build_epilogue(ctx); 172 173 if (bpf_jit_enable > 1) 174 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); 175 176 prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); 177 prog->jited = 1; 178 prog->jited_len = prog_size - cfi_get_offset(); 179 180 if (!prog->is_func || extra_pass) { 181 if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, 182 jit_data->header))) { 183 /* ro_header has been freed */ 184 jit_data->ro_header = NULL; 185 prog = orig_prog; 186 goto out_offset; 187 } 188 /* 189 * The instructions have now been copied to the ROX region from 190 * where they will execute. 191 * Write any modified data cache blocks out to memory and 192 * invalidate the corresponding blocks in the instruction cache. 193 */ 194 bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns); 195 for (i = 0; i < prog->len; i++) 196 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); 197 bpf_prog_fill_jited_linfo(prog, ctx->offset); 198 out_offset: 199 kfree(ctx->offset); 200 kfree(jit_data); 201 prog->aux->jit_data = NULL; 202 } 203 out: 204 205 if (tmp_blinded) 206 bpf_jit_prog_release_other(prog, prog == orig_prog ? 207 tmp : orig_prog); 208 return prog; 209 210 out_free_hdr: 211 if (jit_data->header) { 212 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size, 213 sizeof(jit_data->header->size)); 214 bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header); 215 } 216 goto out_offset; 217 } 218 219 u64 bpf_jit_alloc_exec_limit(void) 220 { 221 return BPF_JIT_REGION_SIZE; 222 } 223 224 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 225 { 226 int ret; 227 228 mutex_lock(&text_mutex); 229 ret = patch_text_nosync(dst, src, len); 230 mutex_unlock(&text_mutex); 231 232 if (ret) 233 return ERR_PTR(-EINVAL); 234 235 return dst; 236 } 237 238 int bpf_arch_text_invalidate(void *dst, size_t len) 239 { 240 int ret; 241 242 mutex_lock(&text_mutex); 243 ret = patch_text_set_nosync(dst, 0, len); 244 mutex_unlock(&text_mutex); 245 246 return ret; 247 } 248 249 void bpf_jit_free(struct bpf_prog *prog) 250 { 251 if (prog->jited) { 252 struct rv_jit_data *jit_data = prog->aux->jit_data; 253 struct bpf_binary_header *hdr; 254 255 /* 256 * If we fail the final pass of JIT (from jit_subprogs), 257 * the program may not be finalized yet. Call finalize here 258 * before freeing it. 259 */ 260 if (jit_data) { 261 bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header); 262 kfree(jit_data); 263 } 264 hdr = bpf_jit_binary_pack_hdr(prog); 265 bpf_jit_binary_pack_free(hdr, NULL); 266 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); 267 } 268 269 bpf_prog_unlock_free(prog); 270 } 271