1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2019 Facebook */ 3 #include <linux/hash.h> 4 #include <linux/bpf.h> 5 #include <linux/filter.h> 6 7 /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */ 8 #define TRAMPOLINE_HASH_BITS 10 9 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) 10 11 static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; 12 13 /* serializes access to trampoline_table */ 14 static DEFINE_MUTEX(trampoline_mutex); 15 16 void *bpf_jit_alloc_exec_page(void) 17 { 18 void *image; 19 20 image = bpf_jit_alloc_exec(PAGE_SIZE); 21 if (!image) 22 return NULL; 23 24 set_vm_flush_reset_perms(image); 25 /* Keep image as writeable. The alternative is to keep flipping ro/rw 26 * everytime new program is attached or detached. 27 */ 28 set_memory_x((long)image, 1); 29 return image; 30 } 31 32 struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 33 { 34 struct bpf_trampoline *tr; 35 struct hlist_head *head; 36 void *image; 37 int i; 38 39 mutex_lock(&trampoline_mutex); 40 head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; 41 hlist_for_each_entry(tr, head, hlist) { 42 if (tr->key == key) { 43 refcount_inc(&tr->refcnt); 44 goto out; 45 } 46 } 47 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 48 if (!tr) 49 goto out; 50 51 /* is_root was checked earlier. No need for bpf_jit_charge_modmem() */ 52 image = bpf_jit_alloc_exec_page(); 53 if (!image) { 54 kfree(tr); 55 tr = NULL; 56 goto out; 57 } 58 59 tr->key = key; 60 INIT_HLIST_NODE(&tr->hlist); 61 hlist_add_head(&tr->hlist, head); 62 refcount_set(&tr->refcnt, 1); 63 mutex_init(&tr->mutex); 64 for (i = 0; i < BPF_TRAMP_MAX; i++) 65 INIT_HLIST_HEAD(&tr->progs_hlist[i]); 66 tr->image = image; 67 out: 68 mutex_unlock(&trampoline_mutex); 69 return tr; 70 } 71 72 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 73 * bytes on x86. Pick a number to fit into PAGE_SIZE / 2 74 */ 75 #define BPF_MAX_TRAMP_PROGS 40 76 77 static int bpf_trampoline_update(struct bpf_trampoline *tr) 78 { 79 void *old_image = tr->image + ((tr->selector + 1) & 1) * PAGE_SIZE/2; 80 void *new_image = tr->image + (tr->selector & 1) * PAGE_SIZE/2; 81 struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS]; 82 int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY]; 83 int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT]; 84 struct bpf_prog **progs, **fentry, **fexit; 85 u32 flags = BPF_TRAMP_F_RESTORE_REGS; 86 struct bpf_prog_aux *aux; 87 int err; 88 89 if (fentry_cnt + fexit_cnt == 0) { 90 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, 91 old_image, NULL); 92 tr->selector = 0; 93 goto out; 94 } 95 96 /* populate fentry progs */ 97 fentry = progs = progs_to_run; 98 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist) 99 *progs++ = aux->prog; 100 101 /* populate fexit progs */ 102 fexit = progs; 103 hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist) 104 *progs++ = aux->prog; 105 106 if (fexit_cnt) 107 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; 108 109 err = arch_prepare_bpf_trampoline(new_image, &tr->func.model, flags, 110 fentry, fentry_cnt, 111 fexit, fexit_cnt, 112 tr->func.addr); 113 if (err) 114 goto out; 115 116 if (tr->selector) 117 /* progs already running at this address */ 118 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, 119 old_image, new_image); 120 else 121 /* first time registering */ 122 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_CALL, NULL, 123 new_image); 124 if (err) 125 goto out; 126 tr->selector++; 127 out: 128 return err; 129 } 130 131 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(enum bpf_attach_type t) 132 { 133 switch (t) { 134 case BPF_TRACE_FENTRY: 135 return BPF_TRAMP_FENTRY; 136 default: 137 return BPF_TRAMP_FEXIT; 138 } 139 } 140 141 int bpf_trampoline_link_prog(struct bpf_prog *prog) 142 { 143 enum bpf_tramp_prog_type kind; 144 struct bpf_trampoline *tr; 145 int err = 0; 146 147 tr = prog->aux->trampoline; 148 kind = bpf_attach_type_to_tramp(prog->expected_attach_type); 149 mutex_lock(&tr->mutex); 150 if (tr->progs_cnt[BPF_TRAMP_FENTRY] + tr->progs_cnt[BPF_TRAMP_FEXIT] 151 >= BPF_MAX_TRAMP_PROGS) { 152 err = -E2BIG; 153 goto out; 154 } 155 if (!hlist_unhashed(&prog->aux->tramp_hlist)) { 156 /* prog already linked */ 157 err = -EBUSY; 158 goto out; 159 } 160 hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]); 161 tr->progs_cnt[kind]++; 162 err = bpf_trampoline_update(prog->aux->trampoline); 163 if (err) { 164 hlist_del(&prog->aux->tramp_hlist); 165 tr->progs_cnt[kind]--; 166 } 167 out: 168 mutex_unlock(&tr->mutex); 169 return err; 170 } 171 172 /* bpf_trampoline_unlink_prog() should never fail. */ 173 int bpf_trampoline_unlink_prog(struct bpf_prog *prog) 174 { 175 enum bpf_tramp_prog_type kind; 176 struct bpf_trampoline *tr; 177 int err; 178 179 tr = prog->aux->trampoline; 180 kind = bpf_attach_type_to_tramp(prog->expected_attach_type); 181 mutex_lock(&tr->mutex); 182 hlist_del(&prog->aux->tramp_hlist); 183 tr->progs_cnt[kind]--; 184 err = bpf_trampoline_update(prog->aux->trampoline); 185 mutex_unlock(&tr->mutex); 186 return err; 187 } 188 189 void bpf_trampoline_put(struct bpf_trampoline *tr) 190 { 191 if (!tr) 192 return; 193 mutex_lock(&trampoline_mutex); 194 if (!refcount_dec_and_test(&tr->refcnt)) 195 goto out; 196 WARN_ON_ONCE(mutex_is_locked(&tr->mutex)); 197 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FENTRY]))) 198 goto out; 199 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT]))) 200 goto out; 201 bpf_jit_free_exec(tr->image); 202 hlist_del(&tr->hlist); 203 kfree(tr); 204 out: 205 mutex_unlock(&trampoline_mutex); 206 } 207 208 /* The logic is similar to BPF_PROG_RUN, but with explicit rcu and preempt that 209 * are needed for trampoline. The macro is split into 210 * call _bpf_prog_enter 211 * call prog->bpf_func 212 * call __bpf_prog_exit 213 */ 214 u64 notrace __bpf_prog_enter(void) 215 { 216 u64 start = 0; 217 218 rcu_read_lock(); 219 preempt_disable(); 220 if (static_branch_unlikely(&bpf_stats_enabled_key)) 221 start = sched_clock(); 222 return start; 223 } 224 225 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) 226 { 227 struct bpf_prog_stats *stats; 228 229 if (static_branch_unlikely(&bpf_stats_enabled_key) && 230 /* static_key could be enabled in __bpf_prog_enter 231 * and disabled in __bpf_prog_exit. 232 * And vice versa. 233 * Hence check that 'start' is not zero. 234 */ 235 start) { 236 stats = this_cpu_ptr(prog->aux->stats); 237 u64_stats_update_begin(&stats->syncp); 238 stats->cnt++; 239 stats->nsecs += sched_clock() - start; 240 u64_stats_update_end(&stats->syncp); 241 } 242 preempt_enable(); 243 rcu_read_unlock(); 244 } 245 246 int __weak 247 arch_prepare_bpf_trampoline(void *image, struct btf_func_model *m, u32 flags, 248 struct bpf_prog **fentry_progs, int fentry_cnt, 249 struct bpf_prog **fexit_progs, int fexit_cnt, 250 void *orig_call) 251 { 252 return -ENOTSUPP; 253 } 254 255 static int __init init_trampolines(void) 256 { 257 int i; 258 259 for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++) 260 INIT_HLIST_HEAD(&trampoline_table[i]); 261 return 0; 262 } 263 late_initcall(init_trampolines); 264