1 #ifndef _NF_FLOW_TABLE_H 2 #define _NF_FLOW_TABLE_H 3 4 #include <linux/in.h> 5 #include <linux/in6.h> 6 #include <linux/netdevice.h> 7 #include <linux/rhashtable-types.h> 8 #include <linux/rcupdate.h> 9 #include <linux/netfilter.h> 10 #include <linux/netfilter/nf_conntrack_tuple_common.h> 11 #include <net/flow_offload.h> 12 #include <net/dst.h> 13 #include <linux/if_pppox.h> 14 #include <linux/ppp_defs.h> 15 16 struct nf_flowtable; 17 struct nf_flow_rule; 18 struct flow_offload; 19 enum flow_offload_tuple_dir; 20 21 struct nf_flow_key { 22 struct flow_dissector_key_meta meta; 23 struct flow_dissector_key_control control; 24 struct flow_dissector_key_control enc_control; 25 struct flow_dissector_key_basic basic; 26 struct flow_dissector_key_vlan vlan; 27 struct flow_dissector_key_vlan cvlan; 28 union { 29 struct flow_dissector_key_ipv4_addrs ipv4; 30 struct flow_dissector_key_ipv6_addrs ipv6; 31 }; 32 struct flow_dissector_key_keyid enc_key_id; 33 union { 34 struct flow_dissector_key_ipv4_addrs enc_ipv4; 35 struct flow_dissector_key_ipv6_addrs enc_ipv6; 36 }; 37 struct flow_dissector_key_tcp tcp; 38 struct flow_dissector_key_ports tp; 39 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 40 41 struct nf_flow_match { 42 struct flow_dissector dissector; 43 struct nf_flow_key key; 44 struct nf_flow_key mask; 45 }; 46 47 struct nf_flow_rule { 48 struct nf_flow_match match; 49 struct flow_rule *rule; 50 }; 51 52 struct nf_flowtable_type { 53 struct list_head list; 54 int family; 55 int (*init)(struct nf_flowtable *ft); 56 bool (*gc)(const struct flow_offload *flow); 57 int (*setup)(struct nf_flowtable *ft, 58 struct net_device *dev, 59 enum flow_block_command cmd); 60 int (*action)(struct net *net, 61 struct flow_offload *flow, 62 enum flow_offload_tuple_dir dir, 63 struct nf_flow_rule *flow_rule); 64 void (*free)(struct nf_flowtable *ft); 65 void (*get)(struct nf_flowtable *ft); 66 void (*put)(struct nf_flowtable *ft); 67 nf_hookfn *hook; 68 struct module *owner; 69 }; 70 71 enum nf_flowtable_flags { 72 NF_FLOWTABLE_HW_OFFLOAD = 0x1, /* NFT_FLOWTABLE_HW_OFFLOAD */ 73 NF_FLOWTABLE_COUNTER = 0x2, /* NFT_FLOWTABLE_COUNTER */ 74 }; 75 76 struct nf_flowtable { 77 unsigned int flags; /* readonly in datapath */ 78 int priority; /* control path (padding hole) */ 79 struct rhashtable rhashtable; /* datapath, read-mostly members come first */ 80 81 struct list_head list; /* slowpath parts */ 82 const struct nf_flowtable_type *type; 83 struct delayed_work gc_work; 84 struct flow_block flow_block; 85 struct rw_semaphore flow_block_lock; /* Guards flow_block */ 86 possible_net_t net; 87 }; 88 89 static inline bool nf_flowtable_hw_offload(struct nf_flowtable *flowtable) 90 { 91 return flowtable->flags & NF_FLOWTABLE_HW_OFFLOAD; 92 } 93 94 enum flow_offload_tuple_dir { 95 FLOW_OFFLOAD_DIR_ORIGINAL = IP_CT_DIR_ORIGINAL, 96 FLOW_OFFLOAD_DIR_REPLY = IP_CT_DIR_REPLY, 97 }; 98 #define FLOW_OFFLOAD_DIR_MAX IP_CT_DIR_MAX 99 100 enum flow_offload_xmit_type { 101 FLOW_OFFLOAD_XMIT_UNSPEC = 0, 102 FLOW_OFFLOAD_XMIT_NEIGH, 103 FLOW_OFFLOAD_XMIT_XFRM, 104 FLOW_OFFLOAD_XMIT_DIRECT, 105 FLOW_OFFLOAD_XMIT_TC, 106 }; 107 108 #define NF_FLOW_TABLE_ENCAP_MAX 2 109 110 struct flow_offload_tunnel { 111 union { 112 struct in_addr src_v4; 113 struct in6_addr src_v6; 114 }; 115 union { 116 struct in_addr dst_v4; 117 struct in6_addr dst_v6; 118 }; 119 120 u8 inner_proto; 121 }; 122 123 struct flow_offload_tuple { 124 union { 125 struct in_addr src_v4; 126 struct in6_addr src_v6; 127 }; 128 union { 129 struct in_addr dst_v4; 130 struct in6_addr dst_v6; 131 }; 132 struct { 133 __be16 src_port; 134 __be16 dst_port; 135 }; 136 137 int iifidx; 138 139 u8 l3proto; 140 u8 l4proto; 141 struct { 142 u16 id; 143 __be16 proto; 144 } encap[NF_FLOW_TABLE_ENCAP_MAX]; 145 146 struct flow_offload_tunnel tun; 147 148 /* All members above are keys for lookups, see flow_offload_hash(). */ 149 struct { } __hash; 150 151 u16 dir:2, 152 xmit_type:3, 153 encap_num:2, 154 needs_gso_segment:1, 155 tun_num:2, 156 in_vlan_ingress:2; 157 u16 mtu; 158 u32 dst_cookie; 159 struct dst_entry *dst_cache; 160 161 union { 162 struct { 163 u32 ifidx; 164 }; 165 struct { 166 u32 ifidx; 167 u8 h_source[ETH_ALEN]; 168 u8 h_dest[ETH_ALEN]; 169 } out; 170 struct { 171 u32 iifidx; 172 } tc; 173 }; 174 }; 175 176 struct flow_offload_tuple_rhash { 177 struct rhash_head node; 178 struct flow_offload_tuple tuple; 179 }; 180 181 enum nf_flow_flags { 182 NF_FLOW_SNAT, 183 NF_FLOW_DNAT, 184 NF_FLOW_CLOSING, 185 NF_FLOW_TEARDOWN, 186 NF_FLOW_PENDING, 187 NF_FLOW_HW, 188 NF_FLOW_HW_DYING, 189 NF_FLOW_HW_DEAD, 190 NF_FLOW_HW_BIDIRECTIONAL, 191 NF_FLOW_HW_ESTABLISHED, 192 }; 193 194 enum flow_offload_type { 195 NF_FLOW_OFFLOAD_UNSPEC = 0, 196 NF_FLOW_OFFLOAD_ROUTE, 197 }; 198 199 struct flow_offload { 200 struct flow_offload_tuple_rhash tuplehash[FLOW_OFFLOAD_DIR_MAX]; 201 struct nf_conn *ct; 202 unsigned long flags; 203 u16 type; 204 u32 timeout; 205 struct rcu_head rcu_head; 206 }; 207 208 #define NF_FLOW_TIMEOUT (30 * HZ) 209 #define nf_flowtable_time_stamp (u32)jiffies 210 211 unsigned long flow_offload_get_timeout(struct flow_offload *flow); 212 213 static inline __s32 nf_flow_timeout_delta(unsigned int timeout) 214 { 215 return (__s32)(timeout - nf_flowtable_time_stamp); 216 } 217 218 struct nf_flow_route { 219 struct { 220 struct dst_entry *dst; 221 struct { 222 u32 ifindex; 223 struct { 224 u16 id; 225 __be16 proto; 226 } encap[NF_FLOW_TABLE_ENCAP_MAX]; 227 struct flow_offload_tunnel tun; 228 u8 num_encaps:2, 229 num_tuns:2, 230 ingress_vlans:2; 231 } in; 232 struct { 233 u32 ifindex; 234 u32 hw_ifindex; 235 u8 h_source[ETH_ALEN]; 236 u8 h_dest[ETH_ALEN]; 237 u8 needs_gso_segment:1; 238 } out; 239 enum flow_offload_xmit_type xmit_type; 240 } tuple[FLOW_OFFLOAD_DIR_MAX]; 241 }; 242 243 struct flow_offload *flow_offload_alloc(struct nf_conn *ct); 244 void flow_offload_free(struct flow_offload *flow); 245 246 struct nft_flowtable; 247 struct nft_pktinfo; 248 int nft_flow_route(const struct nft_pktinfo *pkt, const struct nf_conn *ct, 249 struct nf_flow_route *route, enum ip_conntrack_dir dir, 250 struct nft_flowtable *ft); 251 252 static inline int 253 nf_flow_table_offload_add_cb(struct nf_flowtable *flow_table, 254 flow_setup_cb_t *cb, void *cb_priv) 255 { 256 struct flow_block *block = &flow_table->flow_block; 257 struct flow_block_cb *block_cb; 258 int err = 0; 259 260 down_write(&flow_table->flow_block_lock); 261 block_cb = flow_block_cb_lookup(block, cb, cb_priv); 262 if (block_cb) { 263 err = -EEXIST; 264 goto unlock; 265 } 266 267 block_cb = flow_block_cb_alloc(cb, cb_priv, cb_priv, NULL); 268 if (IS_ERR(block_cb)) { 269 err = PTR_ERR(block_cb); 270 goto unlock; 271 } 272 273 list_add_tail(&block_cb->list, &block->cb_list); 274 up_write(&flow_table->flow_block_lock); 275 276 if (flow_table->type->get) 277 flow_table->type->get(flow_table); 278 return 0; 279 280 unlock: 281 up_write(&flow_table->flow_block_lock); 282 return err; 283 } 284 285 static inline void 286 nf_flow_table_offload_del_cb(struct nf_flowtable *flow_table, 287 flow_setup_cb_t *cb, void *cb_priv) 288 { 289 struct flow_block *block = &flow_table->flow_block; 290 struct flow_block_cb *block_cb; 291 292 down_write(&flow_table->flow_block_lock); 293 block_cb = flow_block_cb_lookup(block, cb, cb_priv); 294 if (block_cb) { 295 list_del(&block_cb->list); 296 flow_block_cb_free(block_cb); 297 } else { 298 WARN_ON(true); 299 } 300 up_write(&flow_table->flow_block_lock); 301 302 if (flow_table->type->put) 303 flow_table->type->put(flow_table); 304 } 305 306 void flow_offload_route_init(struct flow_offload *flow, 307 struct nf_flow_route *route); 308 309 int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow); 310 void flow_offload_refresh(struct nf_flowtable *flow_table, 311 struct flow_offload *flow, bool force); 312 313 static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple) 314 { 315 if (!tuple->dst_cache) 316 return true; 317 318 return dst_check(tuple->dst_cache, tuple->dst_cookie); 319 } 320 321 struct flow_offload_tuple_rhash *flow_offload_lookup(struct nf_flowtable *flow_table, 322 struct flow_offload_tuple *tuple); 323 void nf_flow_table_gc_run(struct nf_flowtable *flow_table); 324 void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable, 325 struct net_device *dev); 326 void nf_flow_table_cleanup(struct net_device *dev); 327 328 int nf_flow_table_init(struct nf_flowtable *flow_table); 329 void nf_flow_table_free(struct nf_flowtable *flow_table); 330 331 void flow_offload_teardown(struct flow_offload *flow); 332 333 void nf_flow_snat_port(const struct flow_offload *flow, 334 struct sk_buff *skb, unsigned int thoff, 335 u8 protocol, enum flow_offload_tuple_dir dir); 336 void nf_flow_dnat_port(const struct flow_offload *flow, 337 struct sk_buff *skb, unsigned int thoff, 338 u8 protocol, enum flow_offload_tuple_dir dir); 339 340 struct flow_ports { 341 __be16 source, dest; 342 }; 343 344 struct nf_flowtable *nf_flowtable_by_dev(const struct net_device *dev); 345 int nf_flow_offload_xdp_setup(struct nf_flowtable *flowtable, 346 struct net_device *dev, 347 enum flow_block_command cmd); 348 349 unsigned int nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, 350 const struct nf_hook_state *state); 351 unsigned int nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, 352 const struct nf_hook_state *state); 353 354 #if (IS_BUILTIN(CONFIG_NF_FLOW_TABLE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ 355 (IS_MODULE(CONFIG_NF_FLOW_TABLE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)) 356 extern int nf_flow_register_bpf(void); 357 #else 358 static inline int nf_flow_register_bpf(void) 359 { 360 return 0; 361 } 362 #endif 363 364 #define MODULE_ALIAS_NF_FLOWTABLE(family) \ 365 MODULE_ALIAS("nf-flowtable-" __stringify(family)) 366 367 void nf_flow_offload_add(struct nf_flowtable *flowtable, 368 struct flow_offload *flow); 369 void nf_flow_offload_refresh(struct nf_flowtable *flowtable, 370 struct flow_offload *flow); 371 void nf_flow_offload_del(struct nf_flowtable *flowtable, 372 struct flow_offload *flow); 373 void nf_flow_offload_stats(struct nf_flowtable *flowtable, 374 struct flow_offload *flow); 375 376 void nf_flow_table_offload_flush(struct nf_flowtable *flowtable); 377 void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable); 378 379 int nf_flow_table_offload_setup(struct nf_flowtable *flowtable, 380 struct net_device *dev, 381 enum flow_block_command cmd); 382 int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow, 383 enum flow_offload_tuple_dir dir, 384 struct nf_flow_rule *flow_rule); 385 int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow, 386 enum flow_offload_tuple_dir dir, 387 struct nf_flow_rule *flow_rule); 388 389 int nf_flow_table_offload_init(void); 390 void nf_flow_table_offload_exit(void); 391 392 static inline __be16 __nf_flow_pppoe_proto(const struct sk_buff *skb) 393 { 394 __be16 proto; 395 396 proto = *((__be16 *)(skb_mac_header(skb) + ETH_HLEN + 397 sizeof(struct pppoe_hdr))); 398 switch (proto) { 399 case htons(PPP_IP): 400 return htons(ETH_P_IP); 401 case htons(PPP_IPV6): 402 return htons(ETH_P_IPV6); 403 } 404 405 return 0; 406 } 407 408 static inline bool nf_flow_pppoe_proto(struct sk_buff *skb, __be16 *inner_proto) 409 { 410 if (!pskb_may_pull(skb, ETH_HLEN + PPPOE_SES_HLEN)) 411 return false; 412 413 *inner_proto = __nf_flow_pppoe_proto(skb); 414 415 return true; 416 } 417 418 #define NF_FLOW_TABLE_STAT_INC(net, count) __this_cpu_inc((net)->ft.stat->count) 419 #define NF_FLOW_TABLE_STAT_DEC(net, count) __this_cpu_dec((net)->ft.stat->count) 420 #define NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count) \ 421 this_cpu_inc((net)->ft.stat->count) 422 #define NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count) \ 423 this_cpu_dec((net)->ft.stat->count) 424 425 #ifdef CONFIG_NF_FLOW_TABLE_PROCFS 426 int nf_flow_table_init_proc(struct net *net); 427 void nf_flow_table_fini_proc(struct net *net); 428 #else 429 static inline int nf_flow_table_init_proc(struct net *net) 430 { 431 return 0; 432 } 433 434 static inline void nf_flow_table_fini_proc(struct net *net) 435 { 436 } 437 #endif /* CONFIG_NF_FLOW_TABLE_PROCFS */ 438 439 #endif /* _NF_FLOW_TABLE_H */ 440