1 #ifndef _NF_FLOW_TABLE_H 2 #define _NF_FLOW_TABLE_H 3 4 #include <linux/in.h> 5 #include <linux/in6.h> 6 #include <linux/netdevice.h> 7 #include <linux/rhashtable-types.h> 8 #include <linux/rcupdate.h> 9 #include <linux/netfilter.h> 10 #include <linux/netfilter/nf_conntrack_tuple_common.h> 11 #include <net/flow_offload.h> 12 #include <net/dst.h> 13 #include <linux/if_pppox.h> 14 #include <linux/ppp_defs.h> 15 16 struct nf_flowtable; 17 struct nf_flow_rule; 18 struct flow_offload; 19 enum flow_offload_tuple_dir; 20 21 struct nf_flow_key { 22 struct flow_dissector_key_meta meta; 23 struct flow_dissector_key_control control; 24 struct flow_dissector_key_control enc_control; 25 struct flow_dissector_key_basic basic; 26 struct flow_dissector_key_vlan vlan; 27 struct flow_dissector_key_vlan cvlan; 28 union { 29 struct flow_dissector_key_ipv4_addrs ipv4; 30 struct flow_dissector_key_ipv6_addrs ipv6; 31 }; 32 struct flow_dissector_key_keyid enc_key_id; 33 union { 34 struct flow_dissector_key_ipv4_addrs enc_ipv4; 35 struct flow_dissector_key_ipv6_addrs enc_ipv6; 36 }; 37 struct flow_dissector_key_tcp tcp; 38 struct flow_dissector_key_ports tp; 39 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 40 41 struct nf_flow_match { 42 struct flow_dissector dissector; 43 struct nf_flow_key key; 44 struct nf_flow_key mask; 45 }; 46 47 struct nf_flow_rule { 48 struct nf_flow_match match; 49 struct flow_rule *rule; 50 }; 51 52 struct nf_flowtable_type { 53 struct list_head list; 54 int family; 55 int (*init)(struct nf_flowtable *ft); 56 bool (*gc)(const struct flow_offload *flow); 57 int (*setup)(struct nf_flowtable *ft, 58 struct net_device *dev, 59 enum flow_block_command cmd); 60 int (*action)(struct net *net, 61 struct flow_offload *flow, 62 enum flow_offload_tuple_dir dir, 63 struct nf_flow_rule *flow_rule); 64 void (*free)(struct nf_flowtable *ft); 65 void (*get)(struct nf_flowtable *ft); 66 void (*put)(struct nf_flowtable *ft); 67 nf_hookfn *hook; 68 struct module *owner; 69 }; 70 71 enum nf_flowtable_flags { 72 NF_FLOWTABLE_HW_OFFLOAD = 0x1, /* NFT_FLOWTABLE_HW_OFFLOAD */ 73 NF_FLOWTABLE_COUNTER = 0x2, /* NFT_FLOWTABLE_COUNTER */ 74 }; 75 76 struct nf_flowtable { 77 unsigned int flags; /* readonly in datapath */ 78 int priority; /* control path (padding hole) */ 79 struct rhashtable rhashtable; /* datapath, read-mostly members come first */ 80 81 struct list_head list; /* slowpath parts */ 82 const struct nf_flowtable_type *type; 83 struct delayed_work gc_work; 84 struct flow_block flow_block; 85 struct rw_semaphore flow_block_lock; /* Guards flow_block */ 86 possible_net_t net; 87 }; 88 89 static inline bool nf_flowtable_hw_offload(struct nf_flowtable *flowtable) 90 { 91 return flowtable->flags & NF_FLOWTABLE_HW_OFFLOAD; 92 } 93 94 enum flow_offload_tuple_dir { 95 FLOW_OFFLOAD_DIR_ORIGINAL = IP_CT_DIR_ORIGINAL, 96 FLOW_OFFLOAD_DIR_REPLY = IP_CT_DIR_REPLY, 97 }; 98 #define FLOW_OFFLOAD_DIR_MAX IP_CT_DIR_MAX 99 100 enum flow_offload_xmit_type { 101 FLOW_OFFLOAD_XMIT_UNSPEC = 0, 102 FLOW_OFFLOAD_XMIT_NEIGH, 103 FLOW_OFFLOAD_XMIT_XFRM, 104 FLOW_OFFLOAD_XMIT_DIRECT, 105 FLOW_OFFLOAD_XMIT_TC, 106 }; 107 108 #define NF_FLOW_TABLE_ENCAP_MAX 2 109 110 struct flow_offload_tunnel { 111 union { 112 struct in_addr src_v4; 113 struct in6_addr src_v6; 114 }; 115 union { 116 struct in_addr dst_v4; 117 struct in6_addr dst_v6; 118 }; 119 120 u8 l3_proto; 121 }; 122 123 struct flow_offload_tuple { 124 union { 125 struct in_addr src_v4; 126 struct in6_addr src_v6; 127 }; 128 union { 129 struct in_addr dst_v4; 130 struct in6_addr dst_v6; 131 }; 132 struct { 133 __be16 src_port; 134 __be16 dst_port; 135 }; 136 137 int iifidx; 138 139 u8 l3proto; 140 u8 l4proto; 141 struct { 142 u16 id; 143 __be16 proto; 144 } encap[NF_FLOW_TABLE_ENCAP_MAX]; 145 146 struct flow_offload_tunnel tun; 147 148 /* All members above are keys for lookups, see flow_offload_hash(). */ 149 struct { } __hash; 150 151 u8 dir:2, 152 xmit_type:3, 153 encap_num:2, 154 tun_num:2, 155 in_vlan_ingress:2; 156 u16 mtu; 157 union { 158 struct { 159 struct dst_entry *dst_cache; 160 u32 ifidx; 161 u32 dst_cookie; 162 }; 163 struct { 164 u32 ifidx; 165 u8 h_source[ETH_ALEN]; 166 u8 h_dest[ETH_ALEN]; 167 } out; 168 struct { 169 u32 iifidx; 170 } tc; 171 }; 172 }; 173 174 struct flow_offload_tuple_rhash { 175 struct rhash_head node; 176 struct flow_offload_tuple tuple; 177 }; 178 179 enum nf_flow_flags { 180 NF_FLOW_SNAT, 181 NF_FLOW_DNAT, 182 NF_FLOW_CLOSING, 183 NF_FLOW_TEARDOWN, 184 NF_FLOW_HW, 185 NF_FLOW_HW_DYING, 186 NF_FLOW_HW_DEAD, 187 NF_FLOW_HW_PENDING, 188 NF_FLOW_HW_BIDIRECTIONAL, 189 NF_FLOW_HW_ESTABLISHED, 190 }; 191 192 enum flow_offload_type { 193 NF_FLOW_OFFLOAD_UNSPEC = 0, 194 NF_FLOW_OFFLOAD_ROUTE, 195 }; 196 197 struct flow_offload { 198 struct flow_offload_tuple_rhash tuplehash[FLOW_OFFLOAD_DIR_MAX]; 199 struct nf_conn *ct; 200 unsigned long flags; 201 u16 type; 202 u32 timeout; 203 struct rcu_head rcu_head; 204 }; 205 206 #define NF_FLOW_TIMEOUT (30 * HZ) 207 #define nf_flowtable_time_stamp (u32)jiffies 208 209 unsigned long flow_offload_get_timeout(struct flow_offload *flow); 210 211 static inline __s32 nf_flow_timeout_delta(unsigned int timeout) 212 { 213 return (__s32)(timeout - nf_flowtable_time_stamp); 214 } 215 216 struct nf_flow_route { 217 struct { 218 struct dst_entry *dst; 219 struct { 220 u32 ifindex; 221 struct { 222 u16 id; 223 __be16 proto; 224 } encap[NF_FLOW_TABLE_ENCAP_MAX]; 225 struct flow_offload_tunnel tun; 226 u8 num_encaps:2, 227 num_tuns:2, 228 ingress_vlans:2; 229 } in; 230 struct { 231 u32 ifindex; 232 u32 hw_ifindex; 233 u8 h_source[ETH_ALEN]; 234 u8 h_dest[ETH_ALEN]; 235 } out; 236 enum flow_offload_xmit_type xmit_type; 237 } tuple[FLOW_OFFLOAD_DIR_MAX]; 238 }; 239 240 struct flow_offload *flow_offload_alloc(struct nf_conn *ct); 241 void flow_offload_free(struct flow_offload *flow); 242 243 struct nft_flowtable; 244 struct nft_pktinfo; 245 int nft_flow_route(const struct nft_pktinfo *pkt, const struct nf_conn *ct, 246 struct nf_flow_route *route, enum ip_conntrack_dir dir, 247 struct nft_flowtable *ft); 248 249 static inline int 250 nf_flow_table_offload_add_cb(struct nf_flowtable *flow_table, 251 flow_setup_cb_t *cb, void *cb_priv) 252 { 253 struct flow_block *block = &flow_table->flow_block; 254 struct flow_block_cb *block_cb; 255 int err = 0; 256 257 down_write(&flow_table->flow_block_lock); 258 block_cb = flow_block_cb_lookup(block, cb, cb_priv); 259 if (block_cb) { 260 err = -EEXIST; 261 goto unlock; 262 } 263 264 block_cb = flow_block_cb_alloc(cb, cb_priv, cb_priv, NULL); 265 if (IS_ERR(block_cb)) { 266 err = PTR_ERR(block_cb); 267 goto unlock; 268 } 269 270 list_add_tail(&block_cb->list, &block->cb_list); 271 up_write(&flow_table->flow_block_lock); 272 273 if (flow_table->type->get) 274 flow_table->type->get(flow_table); 275 return 0; 276 277 unlock: 278 up_write(&flow_table->flow_block_lock); 279 return err; 280 } 281 282 static inline void 283 nf_flow_table_offload_del_cb(struct nf_flowtable *flow_table, 284 flow_setup_cb_t *cb, void *cb_priv) 285 { 286 struct flow_block *block = &flow_table->flow_block; 287 struct flow_block_cb *block_cb; 288 289 down_write(&flow_table->flow_block_lock); 290 block_cb = flow_block_cb_lookup(block, cb, cb_priv); 291 if (block_cb) { 292 list_del(&block_cb->list); 293 flow_block_cb_free(block_cb); 294 } else { 295 WARN_ON(true); 296 } 297 up_write(&flow_table->flow_block_lock); 298 299 if (flow_table->type->put) 300 flow_table->type->put(flow_table); 301 } 302 303 void flow_offload_route_init(struct flow_offload *flow, 304 struct nf_flow_route *route); 305 306 int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow); 307 void flow_offload_refresh(struct nf_flowtable *flow_table, 308 struct flow_offload *flow, bool force); 309 310 struct flow_offload_tuple_rhash *flow_offload_lookup(struct nf_flowtable *flow_table, 311 struct flow_offload_tuple *tuple); 312 void nf_flow_table_gc_run(struct nf_flowtable *flow_table); 313 void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable, 314 struct net_device *dev); 315 void nf_flow_table_cleanup(struct net_device *dev); 316 317 int nf_flow_table_init(struct nf_flowtable *flow_table); 318 void nf_flow_table_free(struct nf_flowtable *flow_table); 319 320 void flow_offload_teardown(struct flow_offload *flow); 321 322 void nf_flow_snat_port(const struct flow_offload *flow, 323 struct sk_buff *skb, unsigned int thoff, 324 u8 protocol, enum flow_offload_tuple_dir dir); 325 void nf_flow_dnat_port(const struct flow_offload *flow, 326 struct sk_buff *skb, unsigned int thoff, 327 u8 protocol, enum flow_offload_tuple_dir dir); 328 329 struct flow_ports { 330 __be16 source, dest; 331 }; 332 333 struct nf_flowtable *nf_flowtable_by_dev(const struct net_device *dev); 334 int nf_flow_offload_xdp_setup(struct nf_flowtable *flowtable, 335 struct net_device *dev, 336 enum flow_block_command cmd); 337 338 unsigned int nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, 339 const struct nf_hook_state *state); 340 unsigned int nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, 341 const struct nf_hook_state *state); 342 343 #if (IS_BUILTIN(CONFIG_NF_FLOW_TABLE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ 344 (IS_MODULE(CONFIG_NF_FLOW_TABLE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)) 345 extern int nf_flow_register_bpf(void); 346 #else 347 static inline int nf_flow_register_bpf(void) 348 { 349 return 0; 350 } 351 #endif 352 353 #define MODULE_ALIAS_NF_FLOWTABLE(family) \ 354 MODULE_ALIAS("nf-flowtable-" __stringify(family)) 355 356 void nf_flow_offload_add(struct nf_flowtable *flowtable, 357 struct flow_offload *flow); 358 void nf_flow_offload_del(struct nf_flowtable *flowtable, 359 struct flow_offload *flow); 360 void nf_flow_offload_stats(struct nf_flowtable *flowtable, 361 struct flow_offload *flow); 362 363 void nf_flow_table_offload_flush(struct nf_flowtable *flowtable); 364 void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable); 365 366 int nf_flow_table_offload_setup(struct nf_flowtable *flowtable, 367 struct net_device *dev, 368 enum flow_block_command cmd); 369 int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow, 370 enum flow_offload_tuple_dir dir, 371 struct nf_flow_rule *flow_rule); 372 int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow, 373 enum flow_offload_tuple_dir dir, 374 struct nf_flow_rule *flow_rule); 375 376 int nf_flow_table_offload_init(void); 377 void nf_flow_table_offload_exit(void); 378 379 static inline __be16 __nf_flow_pppoe_proto(const struct sk_buff *skb) 380 { 381 __be16 proto; 382 383 proto = *((__be16 *)(skb_mac_header(skb) + ETH_HLEN + 384 sizeof(struct pppoe_hdr))); 385 switch (proto) { 386 case htons(PPP_IP): 387 return htons(ETH_P_IP); 388 case htons(PPP_IPV6): 389 return htons(ETH_P_IPV6); 390 } 391 392 return 0; 393 } 394 395 static inline bool nf_flow_pppoe_proto(struct sk_buff *skb, __be16 *inner_proto) 396 { 397 if (!pskb_may_pull(skb, ETH_HLEN + PPPOE_SES_HLEN)) 398 return false; 399 400 *inner_proto = __nf_flow_pppoe_proto(skb); 401 402 return true; 403 } 404 405 #define NF_FLOW_TABLE_STAT_INC(net, count) __this_cpu_inc((net)->ft.stat->count) 406 #define NF_FLOW_TABLE_STAT_DEC(net, count) __this_cpu_dec((net)->ft.stat->count) 407 #define NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count) \ 408 this_cpu_inc((net)->ft.stat->count) 409 #define NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count) \ 410 this_cpu_dec((net)->ft.stat->count) 411 412 #ifdef CONFIG_NF_FLOW_TABLE_PROCFS 413 int nf_flow_table_init_proc(struct net *net); 414 void nf_flow_table_fini_proc(struct net *net); 415 #else 416 static inline int nf_flow_table_init_proc(struct net *net) 417 { 418 return 0; 419 } 420 421 static inline void nf_flow_table_fini_proc(struct net *net) 422 { 423 } 424 #endif /* CONFIG_NF_FLOW_TABLE_PROCFS */ 425 426 #endif /* _NF_FLOW_TABLE_H */ 427