1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2023 Isovalent */ 3 #ifndef __NET_TCX_H 4 #define __NET_TCX_H 5 6 #include <linux/bpf.h> 7 #include <linux/bpf_mprog.h> 8 9 #include <net/sch_generic.h> 10 11 struct mini_Qdisc; 12 13 struct tcx_entry { 14 struct mini_Qdisc __rcu *miniq; 15 struct bpf_mprog_bundle bundle; 16 u32 miniq_active; 17 struct rcu_head rcu; 18 }; 19 20 struct tcx_link { 21 struct bpf_link link; 22 struct net_device *dev; 23 u32 location; 24 }; 25 26 static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress) 27 { 28 #ifdef CONFIG_NET_XGRESS 29 skb->tc_at_ingress = ingress; 30 #endif 31 } 32 33 #ifdef CONFIG_NET_XGRESS 34 static inline struct tcx_entry *tcx_entry(struct bpf_mprog_entry *entry) 35 { 36 struct bpf_mprog_bundle *bundle = entry->parent; 37 38 return container_of(bundle, struct tcx_entry, bundle); 39 } 40 41 static inline struct tcx_link *tcx_link(const struct bpf_link *link) 42 { 43 return container_of(link, struct tcx_link, link); 44 } 45 46 void tcx_inc(void); 47 void tcx_dec(void); 48 49 static inline void tcx_entry_sync(void) 50 { 51 /* bpf_mprog_entry got a/b swapped, therefore ensure that 52 * there are no inflight users on the old one anymore. 53 */ 54 synchronize_rcu(); 55 } 56 57 static inline void 58 tcx_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry, 59 bool ingress) 60 { 61 ASSERT_RTNL(); 62 if (ingress) 63 rcu_assign_pointer(dev->tcx_ingress, entry); 64 else 65 rcu_assign_pointer(dev->tcx_egress, entry); 66 } 67 68 static inline struct bpf_mprog_entry * 69 tcx_entry_fetch(struct net_device *dev, bool ingress) 70 { 71 ASSERT_RTNL(); 72 if (ingress) 73 return rcu_dereference_rtnl(dev->tcx_ingress); 74 else 75 return rcu_dereference_rtnl(dev->tcx_egress); 76 } 77 78 static inline struct bpf_mprog_entry *tcx_entry_create_noprof(void) 79 { 80 struct tcx_entry *tcx = kzalloc_noprof(sizeof(*tcx), GFP_KERNEL); 81 82 if (tcx) { 83 bpf_mprog_bundle_init(&tcx->bundle); 84 return &tcx->bundle.a; 85 } 86 return NULL; 87 } 88 #define tcx_entry_create(...) alloc_hooks(tcx_entry_create_noprof(__VA_ARGS__)) 89 90 static inline void tcx_entry_free(struct bpf_mprog_entry *entry) 91 { 92 kfree_rcu(tcx_entry(entry), rcu); 93 } 94 95 static inline struct bpf_mprog_entry * 96 tcx_entry_fetch_or_create(struct net_device *dev, bool ingress, bool *created) 97 { 98 struct bpf_mprog_entry *entry = tcx_entry_fetch(dev, ingress); 99 100 *created = false; 101 if (!entry) { 102 entry = tcx_entry_create(); 103 if (!entry) 104 return NULL; 105 *created = true; 106 } 107 return entry; 108 } 109 110 static inline void tcx_skeys_inc(bool ingress) 111 { 112 tcx_inc(); 113 if (ingress) 114 net_inc_ingress_queue(); 115 else 116 net_inc_egress_queue(); 117 } 118 119 static inline void tcx_skeys_dec(bool ingress) 120 { 121 if (ingress) 122 net_dec_ingress_queue(); 123 else 124 net_dec_egress_queue(); 125 tcx_dec(); 126 } 127 128 static inline void tcx_miniq_inc(struct bpf_mprog_entry *entry) 129 { 130 ASSERT_RTNL(); 131 tcx_entry(entry)->miniq_active++; 132 } 133 134 static inline void tcx_miniq_dec(struct bpf_mprog_entry *entry) 135 { 136 ASSERT_RTNL(); 137 tcx_entry(entry)->miniq_active--; 138 } 139 140 static inline bool tcx_entry_is_active(struct bpf_mprog_entry *entry) 141 { 142 ASSERT_RTNL(); 143 return bpf_mprog_total(entry) || tcx_entry(entry)->miniq_active; 144 } 145 146 static inline enum tcx_action_base tcx_action_code(struct sk_buff *skb, 147 int code) 148 { 149 switch (code) { 150 case TCX_PASS: 151 skb->tc_index = qdisc_skb_cb(skb)->tc_classid; 152 fallthrough; 153 case TCX_DROP: 154 case TCX_REDIRECT: 155 return code; 156 case TCX_NEXT: 157 default: 158 return TCX_NEXT; 159 } 160 } 161 #endif /* CONFIG_NET_XGRESS */ 162 163 #if defined(CONFIG_NET_XGRESS) && defined(CONFIG_BPF_SYSCALL) 164 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog); 165 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 166 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog); 167 void tcx_uninstall(struct net_device *dev, bool ingress); 168 169 int tcx_prog_query(const union bpf_attr *attr, 170 union bpf_attr __user *uattr); 171 172 static inline void dev_tcx_uninstall(struct net_device *dev) 173 { 174 ASSERT_RTNL(); 175 tcx_uninstall(dev, true); 176 tcx_uninstall(dev, false); 177 } 178 #else 179 static inline int tcx_prog_attach(const union bpf_attr *attr, 180 struct bpf_prog *prog) 181 { 182 return -EINVAL; 183 } 184 185 static inline int tcx_link_attach(const union bpf_attr *attr, 186 struct bpf_prog *prog) 187 { 188 return -EINVAL; 189 } 190 191 static inline int tcx_prog_detach(const union bpf_attr *attr, 192 struct bpf_prog *prog) 193 { 194 return -EINVAL; 195 } 196 197 static inline int tcx_prog_query(const union bpf_attr *attr, 198 union bpf_attr __user *uattr) 199 { 200 return -EINVAL; 201 } 202 203 static inline void dev_tcx_uninstall(struct net_device *dev) 204 { 205 } 206 #endif /* CONFIG_NET_XGRESS && CONFIG_BPF_SYSCALL */ 207 #endif /* __NET_TCX_H */ 208