1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2023 Isovalent */ 3 #ifndef __NET_TCX_H 4 #define __NET_TCX_H 5 6 #include <linux/bpf.h> 7 #include <linux/bpf_mprog.h> 8 9 #include <net/sch_generic.h> 10 11 struct mini_Qdisc; 12 13 struct tcx_entry { 14 struct mini_Qdisc __rcu *miniq; 15 struct bpf_mprog_bundle bundle; 16 u32 miniq_active; 17 struct rcu_head rcu; 18 }; 19 20 struct tcx_link { 21 struct bpf_link link; 22 struct net_device *dev; 23 }; 24 25 static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress) 26 { 27 #ifdef CONFIG_NET_XGRESS 28 skb->tc_at_ingress = ingress; 29 #endif 30 } 31 32 #ifdef CONFIG_NET_XGRESS 33 static inline struct tcx_entry *tcx_entry(struct bpf_mprog_entry *entry) 34 { 35 struct bpf_mprog_bundle *bundle = entry->parent; 36 37 return container_of(bundle, struct tcx_entry, bundle); 38 } 39 40 static inline struct tcx_link *tcx_link(const struct bpf_link *link) 41 { 42 return container_of(link, struct tcx_link, link); 43 } 44 45 void tcx_inc(void); 46 void tcx_dec(void); 47 48 static inline void tcx_entry_sync(void) 49 { 50 /* bpf_mprog_entry got a/b swapped, therefore ensure that 51 * there are no inflight users on the old one anymore. 52 */ 53 synchronize_rcu(); 54 } 55 56 static inline void 57 tcx_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry, 58 bool ingress) 59 { 60 ASSERT_RTNL(); 61 if (ingress) 62 rcu_assign_pointer(dev->tcx_ingress, entry); 63 else 64 rcu_assign_pointer(dev->tcx_egress, entry); 65 } 66 67 static inline struct bpf_mprog_entry * 68 tcx_entry_fetch(struct net_device *dev, bool ingress) 69 { 70 ASSERT_RTNL(); 71 if (ingress) 72 return rcu_dereference_rtnl(dev->tcx_ingress); 73 else 74 return rcu_dereference_rtnl(dev->tcx_egress); 75 } 76 77 static inline struct bpf_mprog_entry *tcx_entry_create_noprof(void) 78 { 79 struct tcx_entry *tcx = kzalloc_noprof(sizeof(*tcx), GFP_KERNEL); 80 81 if (tcx) { 82 bpf_mprog_bundle_init(&tcx->bundle); 83 return &tcx->bundle.a; 84 } 85 return NULL; 86 } 87 #define tcx_entry_create(...) alloc_hooks(tcx_entry_create_noprof(__VA_ARGS__)) 88 89 static inline void tcx_entry_free(struct bpf_mprog_entry *entry) 90 { 91 kfree_rcu(tcx_entry(entry), rcu); 92 } 93 94 static inline struct bpf_mprog_entry * 95 tcx_entry_fetch_or_create(struct net_device *dev, bool ingress, bool *created) 96 { 97 struct bpf_mprog_entry *entry = tcx_entry_fetch(dev, ingress); 98 99 *created = false; 100 if (!entry) { 101 entry = tcx_entry_create(); 102 if (!entry) 103 return NULL; 104 *created = true; 105 } 106 return entry; 107 } 108 109 static inline void tcx_skeys_inc(bool ingress) 110 { 111 tcx_inc(); 112 if (ingress) 113 net_inc_ingress_queue(); 114 else 115 net_inc_egress_queue(); 116 } 117 118 static inline void tcx_skeys_dec(bool ingress) 119 { 120 if (ingress) 121 net_dec_ingress_queue(); 122 else 123 net_dec_egress_queue(); 124 tcx_dec(); 125 } 126 127 static inline void tcx_miniq_inc(struct bpf_mprog_entry *entry) 128 { 129 ASSERT_RTNL(); 130 tcx_entry(entry)->miniq_active++; 131 } 132 133 static inline void tcx_miniq_dec(struct bpf_mprog_entry *entry) 134 { 135 ASSERT_RTNL(); 136 tcx_entry(entry)->miniq_active--; 137 } 138 139 static inline bool tcx_entry_is_active(struct bpf_mprog_entry *entry) 140 { 141 ASSERT_RTNL(); 142 return bpf_mprog_total(entry) || tcx_entry(entry)->miniq_active; 143 } 144 145 static inline enum tcx_action_base tcx_action_code(struct sk_buff *skb, 146 int code) 147 { 148 switch (code) { 149 case TCX_PASS: 150 skb->tc_index = qdisc_skb_cb(skb)->tc_classid; 151 fallthrough; 152 case TCX_DROP: 153 case TCX_REDIRECT: 154 return code; 155 case TCX_NEXT: 156 default: 157 return TCX_NEXT; 158 } 159 } 160 #endif /* CONFIG_NET_XGRESS */ 161 162 #if defined(CONFIG_NET_XGRESS) && defined(CONFIG_BPF_SYSCALL) 163 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog); 164 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 165 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog); 166 void tcx_uninstall(struct net_device *dev, bool ingress); 167 168 int tcx_prog_query(const union bpf_attr *attr, 169 union bpf_attr __user *uattr); 170 171 static inline void dev_tcx_uninstall(struct net_device *dev) 172 { 173 ASSERT_RTNL(); 174 tcx_uninstall(dev, true); 175 tcx_uninstall(dev, false); 176 } 177 #else 178 static inline int tcx_prog_attach(const union bpf_attr *attr, 179 struct bpf_prog *prog) 180 { 181 return -EINVAL; 182 } 183 184 static inline int tcx_link_attach(const union bpf_attr *attr, 185 struct bpf_prog *prog) 186 { 187 return -EINVAL; 188 } 189 190 static inline int tcx_prog_detach(const union bpf_attr *attr, 191 struct bpf_prog *prog) 192 { 193 return -EINVAL; 194 } 195 196 static inline int tcx_prog_query(const union bpf_attr *attr, 197 union bpf_attr __user *uattr) 198 { 199 return -EINVAL; 200 } 201 202 static inline void dev_tcx_uninstall(struct net_device *dev) 203 { 204 } 205 #endif /* CONFIG_NET_XGRESS && CONFIG_BPF_SYSCALL */ 206 #endif /* __NET_TCX_H */ 207