1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2019 Facebook */ 3 #include <linux/hash.h> 4 #include <linux/bpf.h> 5 #include <linux/filter.h> 6 #include <linux/ftrace.h> 7 8 /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */ 9 #define TRAMPOLINE_HASH_BITS 10 10 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) 11 12 static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; 13 14 /* serializes access to trampoline_table */ 15 static DEFINE_MUTEX(trampoline_mutex); 16 17 void *bpf_jit_alloc_exec_page(void) 18 { 19 void *image; 20 21 image = bpf_jit_alloc_exec(PAGE_SIZE); 22 if (!image) 23 return NULL; 24 25 set_vm_flush_reset_perms(image); 26 /* Keep image as writeable. The alternative is to keep flipping ro/rw 27 * everytime new program is attached or detached. 28 */ 29 set_memory_x((long)image, 1); 30 return image; 31 } 32 33 struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 34 { 35 struct bpf_trampoline *tr; 36 struct hlist_head *head; 37 void *image; 38 int i; 39 40 mutex_lock(&trampoline_mutex); 41 head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; 42 hlist_for_each_entry(tr, head, hlist) { 43 if (tr->key == key) { 44 refcount_inc(&tr->refcnt); 45 goto out; 46 } 47 } 48 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 49 if (!tr) 50 goto out; 51 52 /* is_root was checked earlier. No need for bpf_jit_charge_modmem() */ 53 image = bpf_jit_alloc_exec_page(); 54 if (!image) { 55 kfree(tr); 56 tr = NULL; 57 goto out; 58 } 59 60 tr->key = key; 61 INIT_HLIST_NODE(&tr->hlist); 62 hlist_add_head(&tr->hlist, head); 63 refcount_set(&tr->refcnt, 1); 64 mutex_init(&tr->mutex); 65 for (i = 0; i < BPF_TRAMP_MAX; i++) 66 INIT_HLIST_HEAD(&tr->progs_hlist[i]); 67 tr->image = image; 68 out: 69 mutex_unlock(&trampoline_mutex); 70 return tr; 71 } 72 73 static int is_ftrace_location(void *ip) 74 { 75 long addr; 76 77 addr = ftrace_location((long)ip); 78 if (!addr) 79 return 0; 80 if (WARN_ON_ONCE(addr != (long)ip)) 81 return -EFAULT; 82 return 1; 83 } 84 85 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) 86 { 87 void *ip = tr->func.addr; 88 int ret; 89 90 if (tr->func.ftrace_managed) 91 ret = unregister_ftrace_direct((long)ip, (long)old_addr); 92 else 93 ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); 94 return ret; 95 } 96 97 static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr) 98 { 99 void *ip = tr->func.addr; 100 int ret; 101 102 if (tr->func.ftrace_managed) 103 ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr); 104 else 105 ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); 106 return ret; 107 } 108 109 /* first time registering */ 110 static int register_fentry(struct bpf_trampoline *tr, void *new_addr) 111 { 112 void *ip = tr->func.addr; 113 int ret; 114 115 ret = is_ftrace_location(ip); 116 if (ret < 0) 117 return ret; 118 tr->func.ftrace_managed = ret; 119 120 if (tr->func.ftrace_managed) 121 ret = register_ftrace_direct((long)ip, (long)new_addr); 122 else 123 ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); 124 return ret; 125 } 126 127 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 128 * bytes on x86. Pick a number to fit into PAGE_SIZE / 2 129 */ 130 #define BPF_MAX_TRAMP_PROGS 40 131 132 static int bpf_trampoline_update(struct bpf_trampoline *tr) 133 { 134 void *old_image = tr->image + ((tr->selector + 1) & 1) * PAGE_SIZE/2; 135 void *new_image = tr->image + (tr->selector & 1) * PAGE_SIZE/2; 136 struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS]; 137 int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY]; 138 int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT]; 139 struct bpf_prog **progs, **fentry, **fexit; 140 u32 flags = BPF_TRAMP_F_RESTORE_REGS; 141 struct bpf_prog_aux *aux; 142 int err; 143 144 if (fentry_cnt + fexit_cnt == 0) { 145 err = unregister_fentry(tr, old_image); 146 tr->selector = 0; 147 goto out; 148 } 149 150 /* populate fentry progs */ 151 fentry = progs = progs_to_run; 152 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist) 153 *progs++ = aux->prog; 154 155 /* populate fexit progs */ 156 fexit = progs; 157 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist) 158 *progs++ = aux->prog; 159 160 if (fexit_cnt) 161 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; 162 163 err = arch_prepare_bpf_trampoline(new_image, new_image + PAGE_SIZE / 2, 164 &tr->func.model, flags, 165 fentry, fentry_cnt, 166 fexit, fexit_cnt, 167 tr->func.addr); 168 if (err < 0) 169 goto out; 170 171 if (tr->selector) 172 /* progs already running at this address */ 173 err = modify_fentry(tr, old_image, new_image); 174 else 175 /* first time registering */ 176 err = register_fentry(tr, new_image); 177 if (err) 178 goto out; 179 tr->selector++; 180 out: 181 return err; 182 } 183 184 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(enum bpf_attach_type t) 185 { 186 switch (t) { 187 case BPF_TRACE_FENTRY: 188 return BPF_TRAMP_FENTRY; 189 default: 190 return BPF_TRAMP_FEXIT; 191 } 192 } 193 194 int bpf_trampoline_link_prog(struct bpf_prog *prog) 195 { 196 enum bpf_tramp_prog_type kind; 197 struct bpf_trampoline *tr; 198 int err = 0; 199 200 tr = prog->aux->trampoline; 201 kind = bpf_attach_type_to_tramp(prog->expected_attach_type); 202 mutex_lock(&tr->mutex); 203 if (tr->progs_cnt[BPF_TRAMP_FENTRY] + tr->progs_cnt[BPF_TRAMP_FEXIT] 204 >= BPF_MAX_TRAMP_PROGS) { 205 err = -E2BIG; 206 goto out; 207 } 208 if (!hlist_unhashed(&prog->aux->tramp_hlist)) { 209 /* prog already linked */ 210 err = -EBUSY; 211 goto out; 212 } 213 hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]); 214 tr->progs_cnt[kind]++; 215 err = bpf_trampoline_update(prog->aux->trampoline); 216 if (err) { 217 hlist_del(&prog->aux->tramp_hlist); 218 tr->progs_cnt[kind]--; 219 } 220 out: 221 mutex_unlock(&tr->mutex); 222 return err; 223 } 224 225 /* bpf_trampoline_unlink_prog() should never fail. */ 226 int bpf_trampoline_unlink_prog(struct bpf_prog *prog) 227 { 228 enum bpf_tramp_prog_type kind; 229 struct bpf_trampoline *tr; 230 int err; 231 232 tr = prog->aux->trampoline; 233 kind = bpf_attach_type_to_tramp(prog->expected_attach_type); 234 mutex_lock(&tr->mutex); 235 hlist_del(&prog->aux->tramp_hlist); 236 tr->progs_cnt[kind]--; 237 err = bpf_trampoline_update(prog->aux->trampoline); 238 mutex_unlock(&tr->mutex); 239 return err; 240 } 241 242 void bpf_trampoline_put(struct bpf_trampoline *tr) 243 { 244 if (!tr) 245 return; 246 mutex_lock(&trampoline_mutex); 247 if (!refcount_dec_and_test(&tr->refcnt)) 248 goto out; 249 WARN_ON_ONCE(mutex_is_locked(&tr->mutex)); 250 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FENTRY]))) 251 goto out; 252 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT]))) 253 goto out; 254 bpf_jit_free_exec(tr->image); 255 hlist_del(&tr->hlist); 256 kfree(tr); 257 out: 258 mutex_unlock(&trampoline_mutex); 259 } 260 261 /* The logic is similar to BPF_PROG_RUN, but with explicit rcu and preempt that 262 * are needed for trampoline. The macro is split into 263 * call _bpf_prog_enter 264 * call prog->bpf_func 265 * call __bpf_prog_exit 266 */ 267 u64 notrace __bpf_prog_enter(void) 268 { 269 u64 start = 0; 270 271 rcu_read_lock(); 272 preempt_disable(); 273 if (static_branch_unlikely(&bpf_stats_enabled_key)) 274 start = sched_clock(); 275 return start; 276 } 277 278 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) 279 { 280 struct bpf_prog_stats *stats; 281 282 if (static_branch_unlikely(&bpf_stats_enabled_key) && 283 /* static_key could be enabled in __bpf_prog_enter 284 * and disabled in __bpf_prog_exit. 285 * And vice versa. 286 * Hence check that 'start' is not zero. 287 */ 288 start) { 289 stats = this_cpu_ptr(prog->aux->stats); 290 u64_stats_update_begin(&stats->syncp); 291 stats->cnt++; 292 stats->nsecs += sched_clock() - start; 293 u64_stats_update_end(&stats->syncp); 294 } 295 preempt_enable(); 296 rcu_read_unlock(); 297 } 298 299 int __weak 300 arch_prepare_bpf_trampoline(void *image, void *image_end, 301 const struct btf_func_model *m, u32 flags, 302 struct bpf_prog **fentry_progs, int fentry_cnt, 303 struct bpf_prog **fexit_progs, int fexit_cnt, 304 void *orig_call) 305 { 306 return -ENOTSUPP; 307 } 308 309 static int __init init_trampolines(void) 310 { 311 int i; 312 313 for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++) 314 INIT_HLIST_HEAD(&trampoline_table[i]); 315 return 0; 316 } 317 late_initcall(init_trampolines); 318