1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_TC_WRAPPER_H 3 #define __NET_TC_WRAPPER_H 4 5 #include <net/pkt_cls.h> 6 7 #if IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) 8 9 #include <linux/cpufeature.h> 10 #include <linux/static_key.h> 11 #include <linux/indirect_call_wrapper.h> 12 13 #define TC_INDIRECT_SCOPE 14 15 extern struct static_key_false tc_skip_wrapper_act; 16 extern struct static_key_false tc_skip_wrapper_cls; 17 18 /* TC Actions */ 19 #ifdef CONFIG_NET_CLS_ACT 20 21 #define TC_INDIRECT_ACTION_DECLARE(fname) \ 22 INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \ 23 const struct tc_action *a, \ 24 struct tcf_result *res)) 25 26 TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act); 27 TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act); 28 TC_INDIRECT_ACTION_DECLARE(tcf_csum_act); 29 TC_INDIRECT_ACTION_DECLARE(tcf_ct_act); 30 TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act); 31 TC_INDIRECT_ACTION_DECLARE(tcf_gact_act); 32 TC_INDIRECT_ACTION_DECLARE(tcf_gate_act); 33 TC_INDIRECT_ACTION_DECLARE(tcf_ife_act); 34 TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act); 35 TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act); 36 TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act); 37 TC_INDIRECT_ACTION_DECLARE(tcf_nat_act); 38 TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act); 39 TC_INDIRECT_ACTION_DECLARE(tcf_police_act); 40 TC_INDIRECT_ACTION_DECLARE(tcf_sample_act); 41 TC_INDIRECT_ACTION_DECLARE(tcf_simp_act); 42 TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act); 43 TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act); 44 TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act); 45 TC_INDIRECT_ACTION_DECLARE(tunnel_key_act); 46 47 static inline int tc_act(struct sk_buff *skb, const struct tc_action *a, 48 struct tcf_result *res) 49 { 50 if (static_branch_likely(&tc_skip_wrapper_act)) 51 goto skip; 52 53 #if IS_BUILTIN(CONFIG_NET_ACT_GACT) 54 if (a->ops->act == tcf_gact_act) 55 return tcf_gact_act(skb, a, res); 56 #endif 57 #if IS_BUILTIN(CONFIG_NET_ACT_MIRRED) 58 if (a->ops->act == tcf_mirred_act) 59 return tcf_mirred_act(skb, a, res); 60 #endif 61 #if IS_BUILTIN(CONFIG_NET_ACT_PEDIT) 62 if (a->ops->act == tcf_pedit_act) 63 return tcf_pedit_act(skb, a, res); 64 #endif 65 #if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) 66 if (a->ops->act == tcf_skbedit_act) 67 return tcf_skbedit_act(skb, a, res); 68 #endif 69 #if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) 70 if (a->ops->act == tcf_skbmod_act) 71 return tcf_skbmod_act(skb, a, res); 72 #endif 73 #if IS_BUILTIN(CONFIG_NET_ACT_POLICE) 74 if (a->ops->act == tcf_police_act) 75 return tcf_police_act(skb, a, res); 76 #endif 77 #if IS_BUILTIN(CONFIG_NET_ACT_BPF) 78 if (a->ops->act == tcf_bpf_act) 79 return tcf_bpf_act(skb, a, res); 80 #endif 81 #if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) 82 if (a->ops->act == tcf_connmark_act) 83 return tcf_connmark_act(skb, a, res); 84 #endif 85 #if IS_BUILTIN(CONFIG_NET_ACT_CSUM) 86 if (a->ops->act == tcf_csum_act) 87 return tcf_csum_act(skb, a, res); 88 #endif 89 #if IS_BUILTIN(CONFIG_NET_ACT_CT) 90 if (a->ops->act == tcf_ct_act) 91 return tcf_ct_act(skb, a, res); 92 #endif 93 #if IS_BUILTIN(CONFIG_NET_ACT_CTINFO) 94 if (a->ops->act == tcf_ctinfo_act) 95 return tcf_ctinfo_act(skb, a, res); 96 #endif 97 #if IS_BUILTIN(CONFIG_NET_ACT_GATE) 98 if (a->ops->act == tcf_gate_act) 99 return tcf_gate_act(skb, a, res); 100 #endif 101 #if IS_BUILTIN(CONFIG_NET_ACT_MPLS) 102 if (a->ops->act == tcf_mpls_act) 103 return tcf_mpls_act(skb, a, res); 104 #endif 105 #if IS_BUILTIN(CONFIG_NET_ACT_NAT) 106 if (a->ops->act == tcf_nat_act) 107 return tcf_nat_act(skb, a, res); 108 #endif 109 #if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) 110 if (a->ops->act == tunnel_key_act) 111 return tunnel_key_act(skb, a, res); 112 #endif 113 #if IS_BUILTIN(CONFIG_NET_ACT_VLAN) 114 if (a->ops->act == tcf_vlan_act) 115 return tcf_vlan_act(skb, a, res); 116 #endif 117 #if IS_BUILTIN(CONFIG_NET_ACT_IFE) 118 if (a->ops->act == tcf_ife_act) 119 return tcf_ife_act(skb, a, res); 120 #endif 121 #if IS_BUILTIN(CONFIG_NET_ACT_SIMP) 122 if (a->ops->act == tcf_simp_act) 123 return tcf_simp_act(skb, a, res); 124 #endif 125 #if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE) 126 if (a->ops->act == tcf_sample_act) 127 return tcf_sample_act(skb, a, res); 128 #endif 129 130 skip: 131 return a->ops->act(skb, a, res); 132 } 133 134 #endif /* CONFIG_NET_CLS_ACT */ 135 136 /* TC Filters */ 137 #ifdef CONFIG_NET_CLS 138 139 #define TC_INDIRECT_FILTER_DECLARE(fname) \ 140 INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \ 141 const struct tcf_proto *tp, \ 142 struct tcf_result *res)) 143 144 TC_INDIRECT_FILTER_DECLARE(basic_classify); 145 TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify); 146 TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify); 147 TC_INDIRECT_FILTER_DECLARE(fl_classify); 148 TC_INDIRECT_FILTER_DECLARE(flow_classify); 149 TC_INDIRECT_FILTER_DECLARE(fw_classify); 150 TC_INDIRECT_FILTER_DECLARE(mall_classify); 151 TC_INDIRECT_FILTER_DECLARE(route4_classify); 152 TC_INDIRECT_FILTER_DECLARE(u32_classify); 153 154 static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp, 155 struct tcf_result *res) 156 { 157 if (static_branch_likely(&tc_skip_wrapper_cls)) 158 goto skip; 159 160 #if IS_BUILTIN(CONFIG_NET_CLS_BPF) 161 if (tp->classify == cls_bpf_classify) 162 return cls_bpf_classify(skb, tp, res); 163 #endif 164 #if IS_BUILTIN(CONFIG_NET_CLS_U32) 165 if (tp->classify == u32_classify) 166 return u32_classify(skb, tp, res); 167 #endif 168 #if IS_BUILTIN(CONFIG_NET_CLS_FLOWER) 169 if (tp->classify == fl_classify) 170 return fl_classify(skb, tp, res); 171 #endif 172 #if IS_BUILTIN(CONFIG_NET_CLS_FW) 173 if (tp->classify == fw_classify) 174 return fw_classify(skb, tp, res); 175 #endif 176 #if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) 177 if (tp->classify == mall_classify) 178 return mall_classify(skb, tp, res); 179 #endif 180 #if IS_BUILTIN(CONFIG_NET_CLS_BASIC) 181 if (tp->classify == basic_classify) 182 return basic_classify(skb, tp, res); 183 #endif 184 #if IS_BUILTIN(CONFIG_NET_CLS_CGROUP) 185 if (tp->classify == cls_cgroup_classify) 186 return cls_cgroup_classify(skb, tp, res); 187 #endif 188 #if IS_BUILTIN(CONFIG_NET_CLS_FLOW) 189 if (tp->classify == flow_classify) 190 return flow_classify(skb, tp, res); 191 #endif 192 #if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4) 193 if (tp->classify == route4_classify) 194 return route4_classify(skb, tp, res); 195 #endif 196 197 skip: 198 return tp->classify(skb, tp, res); 199 } 200 201 #endif /* CONFIG_NET_CLS */ 202 203 static inline void tc_wrapper_init(void) 204 { 205 #ifdef CONFIG_X86 206 int cnt_cls = IS_BUILTIN(CONFIG_NET_CLS_BPF) + 207 IS_BUILTIN(CONFIG_NET_CLS_U32) + 208 IS_BUILTIN(CONFIG_NET_CLS_FLOWER) + 209 IS_BUILTIN(CONFIG_NET_CLS_FW) + 210 IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) + 211 IS_BUILTIN(CONFIG_NET_CLS_BASIC) + 212 IS_BUILTIN(CONFIG_NET_CLS_CGROUP) + 213 IS_BUILTIN(CONFIG_NET_CLS_FLOW) + 214 IS_BUILTIN(CONFIG_NET_CLS_ROUTE4); 215 216 int cnt_act = IS_BUILTIN(CONFIG_NET_ACT_GACT) + 217 IS_BUILTIN(CONFIG_NET_ACT_MIRRED) + 218 IS_BUILTIN(CONFIG_NET_ACT_PEDIT) + 219 IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) + 220 IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) + 221 IS_BUILTIN(CONFIG_NET_ACT_POLICE) + 222 IS_BUILTIN(CONFIG_NET_ACT_BPF) + 223 IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) + 224 IS_BUILTIN(CONFIG_NET_ACT_CSUM) + 225 IS_BUILTIN(CONFIG_NET_ACT_CT) + 226 IS_BUILTIN(CONFIG_NET_ACT_CTINFO) + 227 IS_BUILTIN(CONFIG_NET_ACT_GATE) + 228 IS_BUILTIN(CONFIG_NET_ACT_MPLS) + 229 IS_BUILTIN(CONFIG_NET_ACT_NAT) + 230 IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) + 231 IS_BUILTIN(CONFIG_NET_ACT_VLAN) + 232 IS_BUILTIN(CONFIG_NET_ACT_IFE) + 233 IS_BUILTIN(CONFIG_NET_ACT_SIMP) + 234 IS_BUILTIN(CONFIG_NET_ACT_SAMPLE); 235 236 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) 237 return; 238 239 if (cnt_cls > 1) 240 static_branch_enable(&tc_skip_wrapper_cls); 241 242 if (cnt_act > 1) 243 static_branch_enable(&tc_skip_wrapper_act); 244 #endif 245 } 246 247 #else 248 249 #define TC_INDIRECT_SCOPE static 250 251 static inline int tc_act(struct sk_buff *skb, const struct tc_action *a, 252 struct tcf_result *res) 253 { 254 return a->ops->act(skb, a, res); 255 } 256 257 static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp, 258 struct tcf_result *res) 259 { 260 return tp->classify(skb, tp, res); 261 } 262 263 static inline void tc_wrapper_init(void) 264 { 265 } 266 267 #endif 268 269 #endif /* __NET_TC_WRAPPER_H */ 270