Lines Matching defs:tr
33 static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
37 struct bpf_trampoline *tr = ops->private;
42 * tr->mutex is already locked.
44 lockdep_assert_held_once(&tr->mutex);
51 if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
52 !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) {
53 if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY))
56 tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
64 * tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
72 * mutex_trylock(&tr->mutex) to avoid deadlock in race condition
75 if (!mutex_trylock(&tr->mutex)) {
76 /* sleep 1 ms to make sure whatever holding tr->mutex makes
85 tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
87 if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
88 !(tr->flags & BPF_TRAMP_F_ORIG_STACK))
89 ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
92 tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;
94 if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
95 ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
102 mutex_unlock(&tr->mutex);
140 struct bpf_trampoline *tr;
146 hlist_for_each_entry(tr, head, hlist) {
147 if (tr->key == key) {
148 refcount_inc(&tr->refcnt);
152 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
153 if (!tr)
156 tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
157 if (!tr->fops) {
158 kfree(tr);
159 tr = NULL;
162 tr->fops->private = tr;
163 tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
166 tr->key = key;
167 INIT_HLIST_NODE(&tr->hlist);
168 hlist_add_head(&tr->hlist, head);
169 refcount_set(&tr->refcnt, 1);
170 mutex_init(&tr->mutex);
172 INIT_HLIST_HEAD(&tr->progs_hlist[i]);
175 return tr;
178 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
180 void *ip = tr->func.addr;
183 if (tr->func.ftrace_managed)
184 ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);
191 static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr,
194 void *ip = tr->func.addr;
197 if (tr->func.ftrace_managed) {
199 ret = modify_ftrace_direct(tr->fops, (long)new_addr);
201 ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr);
209 static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
211 void *ip = tr->func.addr;
217 if (!tr->fops)
219 tr->func.ftrace_managed = true;
222 if (tr->func.ftrace_managed) {
223 ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1);
224 ret = register_ftrace_direct(tr->fops, (long)new_addr);
233 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
246 tlinks[kind].nr_links = tr->progs_cnt[kind];
247 *total += tr->progs_cnt[kind];
250 hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
398 static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex)
402 u32 orig_flags = tr->flags;
406 tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg);
411 err = unregister_fentry(tr, tr->cur_image->image);
412 bpf_tramp_image_put(tr->cur_image);
413 tr->cur_image = NULL;
418 tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX);
425 tr->flags |= BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
427 tr->flags |= BPF_TRAMP_F_RESTORE_REGS;
431 tr->flags |= BPF_TRAMP_F_IP_ARG;
435 if ((tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) &&
436 (tr->flags & BPF_TRAMP_F_CALL_ORIG))
437 tr->flags |= BPF_TRAMP_F_ORIG_STACK;
440 size = arch_bpf_trampoline_size(&tr->func.model, tr->flags,
441 tlinks, tr->func.addr);
452 im = bpf_tramp_image_alloc(tr->key, size);
459 &tr->func.model, tr->flags, tlinks,
460 tr->func.addr);
468 WARN_ON(tr->cur_image && total == 0);
469 if (tr->cur_image)
471 err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex);
474 err = register_fentry(tr, im->image);
483 tr->fops->func = NULL;
484 tr->fops->trampoline = 0;
494 if (tr->cur_image)
495 bpf_tramp_image_put(tr->cur_image);
496 tr->cur_image = im;
500 tr->flags = orig_flags;
550 struct bpf_trampoline *tr,
559 if (tr->extension_prog)
566 cnt += tr->progs_cnt[i];
575 tr->extension_prog = link->link.prog;
576 return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,
584 hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {
591 hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
592 tr->progs_cnt[kind]++;
593 err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
596 tr->progs_cnt[kind]--;
602 struct bpf_trampoline *tr,
607 mutex_lock(&tr->mutex);
608 err = __bpf_trampoline_link_prog(link, tr, tgt_prog);
609 mutex_unlock(&tr->mutex);
614 struct bpf_trampoline *tr,
622 WARN_ON_ONCE(!tr->extension_prog);
623 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
624 tr->extension_prog->bpf_func, NULL);
625 tr->extension_prog = NULL;
631 tr->progs_cnt[kind]--;
632 return bpf_trampoline_update(tr, true /* lock_direct_mutex */);
637 struct bpf_trampoline *tr,
642 mutex_lock(&tr->mutex);
643 err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);
644 mutex_unlock(&tr->mutex);
654 /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */
711 static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr,
718 hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
735 struct bpf_trampoline *tr;
750 tr = bpf_trampoline_get(key, &tgt_info);
751 if (!tr)
754 mutex_lock(&tr->mutex);
756 shim_link = cgroup_shim_find(tr, bpf_func);
761 mutex_unlock(&tr->mutex);
762 bpf_trampoline_put(tr); /* bpf_trampoline_get above */
774 err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL);
778 shim_link->trampoline = tr;
779 /* note, we're still holding tr refcnt from above */
781 mutex_unlock(&tr->mutex);
785 mutex_unlock(&tr->mutex);
790 /* have to release tr while _not_ holding its mutex */
791 bpf_trampoline_put(tr); /* bpf_trampoline_get above */
799 struct bpf_trampoline *tr;
807 tr = bpf_trampoline_lookup(key);
808 if (WARN_ON_ONCE(!tr))
811 mutex_lock(&tr->mutex);
812 shim_link = cgroup_shim_find(tr, bpf_func);
813 mutex_unlock(&tr->mutex);
818 bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */
825 struct bpf_trampoline *tr;
827 tr = bpf_trampoline_lookup(key);
828 if (!tr)
831 mutex_lock(&tr->mutex);
832 if (tr->func.addr)
835 memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));
836 tr->func.addr = (void *)tgt_info->tgt_addr;
838 mutex_unlock(&tr->mutex);
839 return tr;
842 void bpf_trampoline_put(struct bpf_trampoline *tr)
846 if (!tr)
849 if (!refcount_dec_and_test(&tr->refcnt))
851 WARN_ON_ONCE(mutex_is_locked(&tr->mutex));
854 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i])))
863 hlist_del(&tr->hlist);
864 if (tr->fops) {
865 ftrace_free_filter(tr->fops);
866 kfree(tr->fops);
868 kfree(tr);
1049 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)
1051 percpu_ref_get(&tr->pcref);
1054 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr)
1056 percpu_ref_put(&tr->pcref);