1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/init.h>
4 #include <linux/module.h>
5 #include <linux/netfilter.h>
6 #include <linux/rhashtable.h>
7 #include <linux/netdevice.h>
8 #include <net/ip.h>
9 #include <net/ip6_route.h>
10 #include <net/netfilter/nf_tables.h>
11 #include <net/netfilter/nf_flow_table.h>
12 #include <net/netfilter/nf_conntrack.h>
13 #include <net/netfilter/nf_conntrack_core.h>
14 #include <net/netfilter/nf_conntrack_l4proto.h>
15 #include <net/netfilter/nf_conntrack_tuple.h>
16
17 static DEFINE_MUTEX(flowtable_lock);
18 static LIST_HEAD(flowtables);
19 static __read_mostly struct kmem_cache *flow_offload_cachep;
20
21 static void
flow_offload_fill_dir(struct flow_offload * flow,enum flow_offload_tuple_dir dir)22 flow_offload_fill_dir(struct flow_offload *flow,
23 enum flow_offload_tuple_dir dir)
24 {
25 struct flow_offload_tuple *ft = &flow->tuplehash[dir].tuple;
26 struct nf_conntrack_tuple *ctt = &flow->ct->tuplehash[dir].tuple;
27
28 ft->dir = dir;
29
30 switch (ctt->src.l3num) {
31 case NFPROTO_IPV4:
32 ft->src_v4 = ctt->src.u3.in;
33 ft->dst_v4 = ctt->dst.u3.in;
34 break;
35 case NFPROTO_IPV6:
36 ft->src_v6 = ctt->src.u3.in6;
37 ft->dst_v6 = ctt->dst.u3.in6;
38 break;
39 }
40
41 ft->l3proto = ctt->src.l3num;
42 ft->l4proto = ctt->dst.protonum;
43
44 switch (ctt->dst.protonum) {
45 case IPPROTO_TCP:
46 case IPPROTO_UDP:
47 ft->src_port = ctt->src.u.tcp.port;
48 ft->dst_port = ctt->dst.u.tcp.port;
49 break;
50 }
51 }
52
flow_offload_alloc(struct nf_conn * ct)53 struct flow_offload *flow_offload_alloc(struct nf_conn *ct)
54 {
55 struct flow_offload *flow;
56
57 if (unlikely(nf_ct_is_dying(ct)))
58 return NULL;
59
60 flow = kmem_cache_zalloc(flow_offload_cachep, GFP_ATOMIC);
61 if (!flow)
62 return NULL;
63
64 refcount_inc(&ct->ct_general.use);
65 flow->ct = ct;
66
67 flow_offload_fill_dir(flow, FLOW_OFFLOAD_DIR_ORIGINAL);
68 flow_offload_fill_dir(flow, FLOW_OFFLOAD_DIR_REPLY);
69
70 if (ct->status & IPS_SRC_NAT)
71 __set_bit(NF_FLOW_SNAT, &flow->flags);
72 if (ct->status & IPS_DST_NAT)
73 __set_bit(NF_FLOW_DNAT, &flow->flags);
74
75 return flow;
76 }
77 EXPORT_SYMBOL_GPL(flow_offload_alloc);
78
flow_offload_dst_cookie(struct flow_offload_tuple * flow_tuple)79 static u32 flow_offload_dst_cookie(struct flow_offload_tuple *flow_tuple)
80 {
81 if (flow_tuple->l3proto == NFPROTO_IPV6)
82 return rt6_get_cookie(dst_rt6_info(flow_tuple->dst_cache));
83
84 return 0;
85 }
86
nft_route_dst_fetch(struct nf_flow_route * route,enum flow_offload_tuple_dir dir)87 static struct dst_entry *nft_route_dst_fetch(struct nf_flow_route *route,
88 enum flow_offload_tuple_dir dir)
89 {
90 struct dst_entry *dst = route->tuple[dir].dst;
91
92 route->tuple[dir].dst = NULL;
93
94 return dst;
95 }
96
flow_offload_fill_route(struct flow_offload * flow,struct nf_flow_route * route,enum flow_offload_tuple_dir dir)97 static int flow_offload_fill_route(struct flow_offload *flow,
98 struct nf_flow_route *route,
99 enum flow_offload_tuple_dir dir)
100 {
101 struct flow_offload_tuple *flow_tuple = &flow->tuplehash[dir].tuple;
102 struct dst_entry *dst = nft_route_dst_fetch(route, dir);
103 int i, j = 0;
104
105 switch (flow_tuple->l3proto) {
106 case NFPROTO_IPV4:
107 flow_tuple->mtu = ip_dst_mtu_maybe_forward(dst, true);
108 break;
109 case NFPROTO_IPV6:
110 flow_tuple->mtu = ip6_dst_mtu_maybe_forward(dst, true);
111 break;
112 }
113
114 flow_tuple->iifidx = route->tuple[dir].in.ifindex;
115 for (i = route->tuple[dir].in.num_encaps - 1; i >= 0; i--) {
116 flow_tuple->encap[j].id = route->tuple[dir].in.encap[i].id;
117 flow_tuple->encap[j].proto = route->tuple[dir].in.encap[i].proto;
118 if (route->tuple[dir].in.ingress_vlans & BIT(i))
119 flow_tuple->in_vlan_ingress |= BIT(j);
120 j++;
121 }
122
123 flow_tuple->tun = route->tuple[dir].in.tun;
124 flow_tuple->encap_num = route->tuple[dir].in.num_encaps;
125 flow_tuple->needs_gso_segment = route->tuple[dir].out.needs_gso_segment;
126 flow_tuple->tun_num = route->tuple[dir].in.num_tuns;
127
128 switch (route->tuple[dir].xmit_type) {
129 case FLOW_OFFLOAD_XMIT_DIRECT:
130 if (route->tuple[!dir].in.num_tuns) {
131 flow_tuple->dst_cache = dst;
132 flow_tuple->dst_cookie =
133 flow_offload_dst_cookie(flow_tuple);
134 } else {
135 dst_release(dst);
136 }
137 memcpy(flow_tuple->out.h_dest, route->tuple[dir].out.h_dest,
138 ETH_ALEN);
139 memcpy(flow_tuple->out.h_source, route->tuple[dir].out.h_source,
140 ETH_ALEN);
141 flow_tuple->out.ifidx = route->tuple[dir].out.ifindex;
142 break;
143 case FLOW_OFFLOAD_XMIT_XFRM:
144 case FLOW_OFFLOAD_XMIT_NEIGH:
145 flow_tuple->ifidx = route->tuple[dir].out.ifindex;
146 flow_tuple->dst_cache = dst;
147 flow_tuple->dst_cookie = flow_offload_dst_cookie(flow_tuple);
148 break;
149 default:
150 WARN_ON_ONCE(1);
151 break;
152 }
153 flow_tuple->xmit_type = route->tuple[dir].xmit_type;
154
155 return 0;
156 }
157
nft_flow_dst_release(struct flow_offload * flow,enum flow_offload_tuple_dir dir)158 static void nft_flow_dst_release(struct flow_offload *flow,
159 enum flow_offload_tuple_dir dir)
160 {
161 dst_release(flow->tuplehash[dir].tuple.dst_cache);
162 }
163
flow_offload_route_init(struct flow_offload * flow,struct nf_flow_route * route)164 void flow_offload_route_init(struct flow_offload *flow,
165 struct nf_flow_route *route)
166 {
167 flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_ORIGINAL);
168 flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_REPLY);
169 flow->type = NF_FLOW_OFFLOAD_ROUTE;
170 }
171 EXPORT_SYMBOL_GPL(flow_offload_route_init);
172
nf_flow_has_expired(const struct flow_offload * flow)173 static inline bool nf_flow_has_expired(const struct flow_offload *flow)
174 {
175 return nf_flow_timeout_delta(flow->timeout) <= 0;
176 }
177
flow_offload_fixup_tcp(struct nf_conn * ct,u8 tcp_state)178 static void flow_offload_fixup_tcp(struct nf_conn *ct, u8 tcp_state)
179 {
180 struct ip_ct_tcp *tcp = &ct->proto.tcp;
181
182 spin_lock_bh(&ct->lock);
183 if (tcp->state != tcp_state)
184 tcp->state = tcp_state;
185
186 /* syn packet triggers the TCP reopen case from conntrack. */
187 if (tcp->state == TCP_CONNTRACK_CLOSE)
188 ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_CLOSE_INIT;
189
190 /* Conntrack state is outdated due to offload bypass.
191 * Clear IP_CT_TCP_FLAG_MAXACK_SET, otherwise conntracks
192 * TCP reset validation will fail.
193 */
194 tcp->seen[0].td_maxwin = 0;
195 tcp->seen[0].flags &= ~IP_CT_TCP_FLAG_MAXACK_SET;
196 tcp->seen[1].td_maxwin = 0;
197 tcp->seen[1].flags &= ~IP_CT_TCP_FLAG_MAXACK_SET;
198 spin_unlock_bh(&ct->lock);
199 }
200
flow_offload_fixup_ct(struct flow_offload * flow)201 static void flow_offload_fixup_ct(struct flow_offload *flow)
202 {
203 struct nf_conn *ct = flow->ct;
204 struct net *net = nf_ct_net(ct);
205 int l4num = nf_ct_protonum(ct);
206 bool expired, closing = false;
207 u32 offload_timeout = 0;
208 s32 timeout;
209
210 if (l4num == IPPROTO_TCP) {
211 const struct nf_tcp_net *tn = nf_tcp_pernet(net);
212 u8 tcp_state;
213
214 /* Enter CLOSE state if fin/rst packet has been seen, this
215 * allows TCP reopen from conntrack. Otherwise, pick up from
216 * the last seen TCP state.
217 */
218 closing = test_bit(NF_FLOW_CLOSING, &flow->flags);
219 if (closing) {
220 flow_offload_fixup_tcp(ct, TCP_CONNTRACK_CLOSE);
221 timeout = READ_ONCE(tn->timeouts[TCP_CONNTRACK_CLOSE]);
222 expired = false;
223 } else {
224 tcp_state = READ_ONCE(ct->proto.tcp.state);
225 flow_offload_fixup_tcp(ct, tcp_state);
226 timeout = READ_ONCE(tn->timeouts[tcp_state]);
227 expired = nf_flow_has_expired(flow);
228 }
229 offload_timeout = READ_ONCE(tn->offload_timeout);
230
231 } else if (l4num == IPPROTO_UDP) {
232 const struct nf_udp_net *tn = nf_udp_pernet(net);
233 enum udp_conntrack state =
234 test_bit(IPS_SEEN_REPLY_BIT, &ct->status) ?
235 UDP_CT_REPLIED : UDP_CT_UNREPLIED;
236
237 timeout = READ_ONCE(tn->timeouts[state]);
238 expired = nf_flow_has_expired(flow);
239 offload_timeout = READ_ONCE(tn->offload_timeout);
240 } else {
241 return;
242 }
243
244 if (expired)
245 timeout -= offload_timeout;
246
247 if (timeout < 0)
248 timeout = 0;
249
250 if (closing ||
251 nf_flow_timeout_delta(READ_ONCE(ct->timeout)) > (__s32)timeout)
252 nf_ct_refresh(ct, timeout);
253 }
254
flow_offload_route_release(struct flow_offload * flow)255 static void flow_offload_route_release(struct flow_offload *flow)
256 {
257 nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_ORIGINAL);
258 nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_REPLY);
259 }
260
flow_offload_free(struct flow_offload * flow)261 void flow_offload_free(struct flow_offload *flow)
262 {
263 switch (flow->type) {
264 case NF_FLOW_OFFLOAD_ROUTE:
265 flow_offload_route_release(flow);
266 break;
267 default:
268 break;
269 }
270 nf_ct_put(flow->ct);
271 kfree_rcu(flow, rcu_head);
272 }
273 EXPORT_SYMBOL_GPL(flow_offload_free);
274
flow_offload_hash(const void * data,u32 len,u32 seed)275 static u32 flow_offload_hash(const void *data, u32 len, u32 seed)
276 {
277 const struct flow_offload_tuple *tuple = data;
278
279 return jhash(tuple, offsetof(struct flow_offload_tuple, __hash), seed);
280 }
281
flow_offload_hash_obj(const void * data,u32 len,u32 seed)282 static u32 flow_offload_hash_obj(const void *data, u32 len, u32 seed)
283 {
284 const struct flow_offload_tuple_rhash *tuplehash = data;
285
286 return jhash(&tuplehash->tuple, offsetof(struct flow_offload_tuple, __hash), seed);
287 }
288
flow_offload_hash_cmp(struct rhashtable_compare_arg * arg,const void * ptr)289 static int flow_offload_hash_cmp(struct rhashtable_compare_arg *arg,
290 const void *ptr)
291 {
292 const struct flow_offload_tuple *tuple = arg->key;
293 const struct flow_offload_tuple_rhash *x = ptr;
294
295 if (memcmp(&x->tuple, tuple, offsetof(struct flow_offload_tuple, __hash)))
296 return 1;
297
298 return 0;
299 }
300
301 static const struct rhashtable_params nf_flow_offload_rhash_params = {
302 .head_offset = offsetof(struct flow_offload_tuple_rhash, node),
303 .hashfn = flow_offload_hash,
304 .obj_hashfn = flow_offload_hash_obj,
305 .obj_cmpfn = flow_offload_hash_cmp,
306 .automatic_shrinking = true,
307 };
308
flow_offload_get_timeout(struct flow_offload * flow)309 unsigned long flow_offload_get_timeout(struct flow_offload *flow)
310 {
311 unsigned long timeout = NF_FLOW_TIMEOUT;
312 struct net *net = nf_ct_net(flow->ct);
313 int l4num = nf_ct_protonum(flow->ct);
314
315 if (l4num == IPPROTO_TCP) {
316 struct nf_tcp_net *tn = nf_tcp_pernet(net);
317
318 timeout = tn->offload_timeout;
319 } else if (l4num == IPPROTO_UDP) {
320 struct nf_udp_net *tn = nf_udp_pernet(net);
321
322 timeout = tn->offload_timeout;
323 }
324
325 return timeout;
326 }
327
flow_offload_add(struct nf_flowtable * flow_table,struct flow_offload * flow)328 int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
329 {
330 int err;
331
332 flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
333
334 err = rhashtable_insert_fast(&flow_table->rhashtable,
335 &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
336 nf_flow_offload_rhash_params);
337 if (err < 0)
338 return err;
339
340 /* GC only iterates original-direction entries; publish original last. */
341 err = rhashtable_insert_fast(&flow_table->rhashtable,
342 &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
343 nf_flow_offload_rhash_params);
344 if (err < 0) {
345 rhashtable_remove_fast(&flow_table->rhashtable,
346 &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
347 nf_flow_offload_rhash_params);
348 return err;
349 }
350
351 nf_ct_refresh(flow->ct, NF_CT_DAY);
352
353 if (nf_flowtable_hw_offload(flow_table))
354 nf_flow_offload_add(flow_table, flow);
355
356 return 0;
357 }
358 EXPORT_SYMBOL_GPL(flow_offload_add);
359
flow_offload_refresh(struct nf_flowtable * flow_table,struct flow_offload * flow,bool force)360 void flow_offload_refresh(struct nf_flowtable *flow_table,
361 struct flow_offload *flow, bool force)
362 {
363 u32 timeout;
364
365 timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
366 if (force || timeout - READ_ONCE(flow->timeout) > HZ)
367 WRITE_ONCE(flow->timeout, timeout);
368 else
369 return;
370
371 if (likely(!nf_flowtable_hw_offload(flow_table)) ||
372 test_bit(NF_FLOW_CLOSING, &flow->flags))
373 return;
374
375 if (test_bit(NF_FLOW_HW, &flow->flags))
376 nf_flow_offload_refresh(flow_table, flow);
377 }
378 EXPORT_SYMBOL_GPL(flow_offload_refresh);
379
flow_offload_del(struct nf_flowtable * flow_table,struct flow_offload * flow)380 static void flow_offload_del(struct nf_flowtable *flow_table,
381 struct flow_offload *flow)
382 {
383 rhashtable_remove_fast(&flow_table->rhashtable,
384 &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
385 nf_flow_offload_rhash_params);
386 rhashtable_remove_fast(&flow_table->rhashtable,
387 &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
388 nf_flow_offload_rhash_params);
389 flow_offload_free(flow);
390 }
391
flow_offload_teardown(struct flow_offload * flow)392 void flow_offload_teardown(struct flow_offload *flow)
393 {
394 clear_bit(IPS_OFFLOAD_BIT, &flow->ct->status);
395 if (!test_and_set_bit(NF_FLOW_TEARDOWN, &flow->flags))
396 flow_offload_fixup_ct(flow);
397 }
398 EXPORT_SYMBOL_GPL(flow_offload_teardown);
399
400 struct flow_offload_tuple_rhash *
flow_offload_lookup(struct nf_flowtable * flow_table,struct flow_offload_tuple * tuple)401 flow_offload_lookup(struct nf_flowtable *flow_table,
402 struct flow_offload_tuple *tuple)
403 {
404 struct flow_offload_tuple_rhash *tuplehash;
405 struct flow_offload *flow;
406 int dir;
407
408 tuplehash = rhashtable_lookup(&flow_table->rhashtable, tuple,
409 nf_flow_offload_rhash_params);
410 if (!tuplehash)
411 return NULL;
412
413 dir = tuplehash->tuple.dir;
414 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
415 if (test_bit(NF_FLOW_TEARDOWN, &flow->flags))
416 return NULL;
417
418 if (unlikely(nf_ct_is_dying(flow->ct)))
419 return NULL;
420
421 return tuplehash;
422 }
423 EXPORT_SYMBOL_GPL(flow_offload_lookup);
424
425 static int
nf_flow_table_iterate(struct nf_flowtable * flow_table,void (* iter)(struct nf_flowtable * flowtable,struct flow_offload * flow,void * data),void * data)426 nf_flow_table_iterate(struct nf_flowtable *flow_table,
427 void (*iter)(struct nf_flowtable *flowtable,
428 struct flow_offload *flow, void *data),
429 void *data)
430 {
431 struct flow_offload_tuple_rhash *tuplehash;
432 struct rhashtable_iter hti;
433 struct flow_offload *flow;
434 int err = 0;
435
436 rhashtable_walk_enter(&flow_table->rhashtable, &hti);
437 rhashtable_walk_start(&hti);
438
439 while ((tuplehash = rhashtable_walk_next(&hti))) {
440 if (IS_ERR(tuplehash)) {
441 if (PTR_ERR(tuplehash) != -EAGAIN) {
442 err = PTR_ERR(tuplehash);
443 break;
444 }
445 continue;
446 }
447 if (tuplehash->tuple.dir)
448 continue;
449
450 flow = container_of(tuplehash, struct flow_offload, tuplehash[0]);
451
452 iter(flow_table, flow, data);
453 }
454 rhashtable_walk_stop(&hti);
455 rhashtable_walk_exit(&hti);
456
457 return err;
458 }
459
nf_flow_custom_gc(struct nf_flowtable * flow_table,const struct flow_offload * flow)460 static bool nf_flow_custom_gc(struct nf_flowtable *flow_table,
461 const struct flow_offload *flow)
462 {
463 return flow_table->type->gc && flow_table->type->gc(flow);
464 }
465
466 /**
467 * nf_flow_table_tcp_timeout() - new timeout of offloaded tcp entry
468 * @ct: Flowtable offloaded tcp ct
469 *
470 * Return: number of seconds when ct entry should expire.
471 */
nf_flow_table_tcp_timeout(const struct nf_conn * ct)472 static u32 nf_flow_table_tcp_timeout(const struct nf_conn *ct)
473 {
474 u8 state = READ_ONCE(ct->proto.tcp.state);
475
476 switch (state) {
477 case TCP_CONNTRACK_SYN_SENT:
478 case TCP_CONNTRACK_SYN_RECV:
479 return 0;
480 case TCP_CONNTRACK_ESTABLISHED:
481 return NF_CT_DAY;
482 case TCP_CONNTRACK_FIN_WAIT:
483 case TCP_CONNTRACK_CLOSE_WAIT:
484 case TCP_CONNTRACK_LAST_ACK:
485 case TCP_CONNTRACK_TIME_WAIT:
486 return 5 * 60 * HZ;
487 case TCP_CONNTRACK_CLOSE:
488 return 0;
489 }
490
491 return 0;
492 }
493
494 /**
495 * nf_flow_table_extend_ct_timeout() - Extend ct timeout of offloaded conntrack entry
496 * @ct: Flowtable offloaded ct
497 *
498 * Datapath lookups in the conntrack table will evict nf_conn entries
499 * if they have expired.
500 *
501 * Once nf_conn entries have been offloaded, nf_conntrack might not see any
502 * packets anymore. Thus ct->timeout is no longer refreshed and ct can
503 * be evicted.
504 *
505 * To avoid the need for an additional check on the offload bit for every
506 * packet processed via nf_conntrack_in(), set an arbitrary timeout large
507 * enough not to ever expire, this save us a check for the IPS_OFFLOAD_BIT
508 * from the packet path via nf_ct_is_expired().
509 */
nf_flow_table_extend_ct_timeout(struct nf_conn * ct)510 static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct)
511 {
512 static const s32 min_timeout = 5 * 60 * HZ;
513 u32 ct_timeout = READ_ONCE(ct->timeout);
514 s32 expires;
515
516 expires = ct_timeout - nfct_time_stamp;
517 if (expires <= 0) /* already expired */
518 return;
519
520 /* normal case: large enough timeout, nothing to do. */
521 if (likely(expires >= min_timeout))
522 return;
523
524 /* must check offload bit after this, we do not hold any locks.
525 * flowtable and ct entries could have been removed on another CPU.
526 */
527 if (!refcount_inc_not_zero(&ct->ct_general.use))
528 return;
529
530 /* load ct->status after refcount increase */
531 smp_acquire__after_ctrl_dep();
532
533 if (nf_ct_is_confirmed(ct) &&
534 test_bit(IPS_OFFLOAD_BIT, &ct->status)) {
535 u8 l4proto = nf_ct_protonum(ct);
536 u32 new_timeout = 1;
537
538 switch (l4proto) {
539 case IPPROTO_UDP:
540 new_timeout = NF_CT_DAY;
541 break;
542 case IPPROTO_TCP:
543 new_timeout = nf_flow_table_tcp_timeout(ct);
544 break;
545 default:
546 WARN_ON_ONCE(1);
547 break;
548 }
549
550 /* Update to ct->timeout from nf_conntrack happens
551 * without holding ct->lock.
552 *
553 * Use cmpxchg to ensure timeout extension doesn't
554 * happen when we race with conntrack datapath.
555 *
556 * The inverse -- datapath updating ->timeout right
557 * after this -- is fine, datapath is authoritative.
558 */
559 if (new_timeout) {
560 new_timeout += nfct_time_stamp;
561 cmpxchg(&ct->timeout, ct_timeout, new_timeout);
562 }
563 }
564
565 nf_ct_put(ct);
566 }
567
nf_flow_offload_gc_step(struct nf_flowtable * flow_table,struct flow_offload * flow,void * data)568 static void nf_flow_offload_gc_step(struct nf_flowtable *flow_table,
569 struct flow_offload *flow, void *data)
570 {
571 bool teardown = test_bit(NF_FLOW_TEARDOWN, &flow->flags);
572
573 if (nf_flow_has_expired(flow) ||
574 nf_ct_is_dying(flow->ct) ||
575 !nf_flow_dst_check(&flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple) ||
576 !nf_flow_dst_check(&flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple) ||
577 nf_flow_custom_gc(flow_table, flow)) {
578 flow_offload_teardown(flow);
579 teardown = true;
580 } else if (!teardown) {
581 nf_flow_table_extend_ct_timeout(flow->ct);
582 }
583
584 if (teardown) {
585 if (test_bit(NF_FLOW_HW, &flow->flags)) {
586 if (!test_bit(NF_FLOW_HW_DYING, &flow->flags))
587 nf_flow_offload_del(flow_table, flow);
588 else if (test_bit(NF_FLOW_HW_DEAD, &flow->flags))
589 flow_offload_del(flow_table, flow);
590 } else {
591 flow_offload_del(flow_table, flow);
592 }
593 } else if (test_bit(NF_FLOW_CLOSING, &flow->flags) &&
594 test_bit(NF_FLOW_HW, &flow->flags) &&
595 !test_bit(NF_FLOW_HW_DYING, &flow->flags)) {
596 nf_flow_offload_del(flow_table, flow);
597 } else if (test_bit(NF_FLOW_HW, &flow->flags)) {
598 nf_flow_offload_stats(flow_table, flow);
599 }
600 }
601
nf_flow_table_gc_run(struct nf_flowtable * flow_table)602 void nf_flow_table_gc_run(struct nf_flowtable *flow_table)
603 {
604 nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL);
605 }
606
nf_flow_offload_work_gc(struct work_struct * work)607 static void nf_flow_offload_work_gc(struct work_struct *work)
608 {
609 struct nf_flowtable *flow_table;
610
611 flow_table = container_of(work, struct nf_flowtable, gc_work.work);
612 nf_flow_table_gc_run(flow_table);
613 queue_delayed_work(system_power_efficient_wq, &flow_table->gc_work, HZ);
614 }
615
nf_flow_nat_port_tcp(struct sk_buff * skb,unsigned int thoff,__be16 port,__be16 new_port)616 static void nf_flow_nat_port_tcp(struct sk_buff *skb, unsigned int thoff,
617 __be16 port, __be16 new_port)
618 {
619 struct tcphdr *tcph;
620
621 tcph = (void *)(skb_network_header(skb) + thoff);
622 inet_proto_csum_replace2(&tcph->check, skb, port, new_port, false);
623 }
624
nf_flow_nat_port_udp(struct sk_buff * skb,unsigned int thoff,__be16 port,__be16 new_port)625 static void nf_flow_nat_port_udp(struct sk_buff *skb, unsigned int thoff,
626 __be16 port, __be16 new_port)
627 {
628 struct udphdr *udph;
629
630 udph = (void *)(skb_network_header(skb) + thoff);
631 if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
632 inet_proto_csum_replace2(&udph->check, skb, port,
633 new_port, false);
634 if (!udph->check)
635 udph->check = CSUM_MANGLED_0;
636 }
637 }
638
nf_flow_nat_port(struct sk_buff * skb,unsigned int thoff,u8 protocol,__be16 port,__be16 new_port)639 static void nf_flow_nat_port(struct sk_buff *skb, unsigned int thoff,
640 u8 protocol, __be16 port, __be16 new_port)
641 {
642 switch (protocol) {
643 case IPPROTO_TCP:
644 nf_flow_nat_port_tcp(skb, thoff, port, new_port);
645 break;
646 case IPPROTO_UDP:
647 nf_flow_nat_port_udp(skb, thoff, port, new_port);
648 break;
649 }
650 }
651
nf_flow_snat_port(const struct flow_offload * flow,struct sk_buff * skb,unsigned int thoff,u8 protocol,enum flow_offload_tuple_dir dir)652 void nf_flow_snat_port(const struct flow_offload *flow,
653 struct sk_buff *skb, unsigned int thoff,
654 u8 protocol, enum flow_offload_tuple_dir dir)
655 {
656 struct flow_ports *hdr;
657 __be16 port, new_port;
658
659 hdr = (void *)(skb_network_header(skb) + thoff);
660
661 switch (dir) {
662 case FLOW_OFFLOAD_DIR_ORIGINAL:
663 port = hdr->source;
664 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port;
665 hdr->source = new_port;
666 break;
667 case FLOW_OFFLOAD_DIR_REPLY:
668 port = hdr->dest;
669 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port;
670 hdr->dest = new_port;
671 break;
672 }
673
674 nf_flow_nat_port(skb, thoff, protocol, port, new_port);
675 }
676 EXPORT_SYMBOL_GPL(nf_flow_snat_port);
677
nf_flow_dnat_port(const struct flow_offload * flow,struct sk_buff * skb,unsigned int thoff,u8 protocol,enum flow_offload_tuple_dir dir)678 void nf_flow_dnat_port(const struct flow_offload *flow, struct sk_buff *skb,
679 unsigned int thoff, u8 protocol,
680 enum flow_offload_tuple_dir dir)
681 {
682 struct flow_ports *hdr;
683 __be16 port, new_port;
684
685 hdr = (void *)(skb_network_header(skb) + thoff);
686
687 switch (dir) {
688 case FLOW_OFFLOAD_DIR_ORIGINAL:
689 port = hdr->dest;
690 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port;
691 hdr->dest = new_port;
692 break;
693 case FLOW_OFFLOAD_DIR_REPLY:
694 port = hdr->source;
695 new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port;
696 hdr->source = new_port;
697 break;
698 }
699
700 nf_flow_nat_port(skb, thoff, protocol, port, new_port);
701 }
702 EXPORT_SYMBOL_GPL(nf_flow_dnat_port);
703
nf_flow_table_init(struct nf_flowtable * flowtable)704 int nf_flow_table_init(struct nf_flowtable *flowtable)
705 {
706 int err;
707
708 INIT_DELAYED_WORK(&flowtable->gc_work, nf_flow_offload_work_gc);
709 flow_block_init(&flowtable->flow_block);
710 init_rwsem(&flowtable->flow_block_lock);
711
712 err = rhashtable_init(&flowtable->rhashtable,
713 &nf_flow_offload_rhash_params);
714 if (err < 0)
715 return err;
716
717 queue_delayed_work(system_power_efficient_wq,
718 &flowtable->gc_work, HZ);
719
720 mutex_lock(&flowtable_lock);
721 list_add(&flowtable->list, &flowtables);
722 mutex_unlock(&flowtable_lock);
723
724 return 0;
725 }
726 EXPORT_SYMBOL_GPL(nf_flow_table_init);
727
nf_flow_table_do_cleanup(struct nf_flowtable * flow_table,struct flow_offload * flow,void * data)728 static void nf_flow_table_do_cleanup(struct nf_flowtable *flow_table,
729 struct flow_offload *flow, void *data)
730 {
731 struct net_device *dev = data;
732
733 if (!dev) {
734 flow_offload_teardown(flow);
735 return;
736 }
737
738 if (net_eq(nf_ct_net(flow->ct), dev_net(dev)) &&
739 (flow->tuplehash[0].tuple.iifidx == dev->ifindex ||
740 flow->tuplehash[1].tuple.iifidx == dev->ifindex))
741 flow_offload_teardown(flow);
742 }
743
nf_flow_table_gc_cleanup(struct nf_flowtable * flowtable,struct net_device * dev)744 void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable,
745 struct net_device *dev)
746 {
747 nf_flow_table_iterate(flowtable, nf_flow_table_do_cleanup, dev);
748 flush_delayed_work(&flowtable->gc_work);
749 nf_flow_table_offload_flush(flowtable);
750 }
751
nf_flow_table_cleanup(struct net_device * dev)752 void nf_flow_table_cleanup(struct net_device *dev)
753 {
754 struct nf_flowtable *flowtable;
755
756 mutex_lock(&flowtable_lock);
757 list_for_each_entry(flowtable, &flowtables, list)
758 nf_flow_table_gc_cleanup(flowtable, dev);
759 mutex_unlock(&flowtable_lock);
760 }
761 EXPORT_SYMBOL_GPL(nf_flow_table_cleanup);
762
nf_flow_table_free(struct nf_flowtable * flow_table)763 void nf_flow_table_free(struct nf_flowtable *flow_table)
764 {
765 mutex_lock(&flowtable_lock);
766 list_del(&flow_table->list);
767 mutex_unlock(&flowtable_lock);
768
769 cancel_delayed_work_sync(&flow_table->gc_work);
770 nf_flow_table_offload_flush(flow_table);
771 /* ... no more pending work after this stage ... */
772 nf_flow_table_iterate(flow_table, nf_flow_table_do_cleanup, NULL);
773 nf_flow_table_gc_run(flow_table);
774 nf_flow_table_offload_flush_cleanup(flow_table);
775 rhashtable_destroy(&flow_table->rhashtable);
776 }
777 EXPORT_SYMBOL_GPL(nf_flow_table_free);
778
nf_flow_table_init_net(struct net * net)779 static int nf_flow_table_init_net(struct net *net)
780 {
781 net->ft.stat = alloc_percpu(struct nf_flow_table_stat);
782 return net->ft.stat ? 0 : -ENOMEM;
783 }
784
nf_flow_table_fini_net(struct net * net)785 static void nf_flow_table_fini_net(struct net *net)
786 {
787 free_percpu(net->ft.stat);
788 }
789
nf_flow_table_pernet_init(struct net * net)790 static int nf_flow_table_pernet_init(struct net *net)
791 {
792 int ret;
793
794 ret = nf_flow_table_init_net(net);
795 if (ret < 0)
796 return ret;
797
798 ret = nf_flow_table_init_proc(net);
799 if (ret < 0)
800 goto out_proc;
801
802 return 0;
803
804 out_proc:
805 nf_flow_table_fini_net(net);
806 return ret;
807 }
808
nf_flow_table_pernet_exit(struct list_head * net_exit_list)809 static void nf_flow_table_pernet_exit(struct list_head *net_exit_list)
810 {
811 struct net *net;
812
813 list_for_each_entry(net, net_exit_list, exit_list) {
814 nf_flow_table_fini_proc(net);
815 nf_flow_table_fini_net(net);
816 }
817 }
818
819 static struct pernet_operations nf_flow_table_net_ops = {
820 .init = nf_flow_table_pernet_init,
821 .exit_batch = nf_flow_table_pernet_exit,
822 };
823
nf_flow_table_module_init(void)824 static int __init nf_flow_table_module_init(void)
825 {
826 int ret;
827
828 flow_offload_cachep = KMEM_CACHE(flow_offload, SLAB_HWCACHE_ALIGN);
829 if (!flow_offload_cachep)
830 return -ENOMEM;
831
832 ret = register_pernet_subsys(&nf_flow_table_net_ops);
833 if (ret < 0)
834 goto out_pernet;
835
836 ret = nf_flow_table_offload_init();
837 if (ret)
838 goto out_offload;
839
840 ret = nf_flow_register_bpf();
841 if (ret)
842 goto out_bpf;
843
844 return 0;
845
846 out_bpf:
847 nf_flow_table_offload_exit();
848 out_offload:
849 unregister_pernet_subsys(&nf_flow_table_net_ops);
850 out_pernet:
851 kmem_cache_destroy(flow_offload_cachep);
852 return ret;
853 }
854
nf_flow_table_module_exit(void)855 static void __exit nf_flow_table_module_exit(void)
856 {
857 nf_flow_table_offload_exit();
858 unregister_pernet_subsys(&nf_flow_table_net_ops);
859 kmem_cache_destroy(flow_offload_cachep);
860 }
861
862 module_init(nf_flow_table_module_init);
863 module_exit(nf_flow_table_module_exit);
864
865 MODULE_LICENSE("GPL");
866 MODULE_AUTHOR("Pablo Neira Ayuso <pablo@netfilter.org>");
867 MODULE_DESCRIPTION("Netfilter flow table module");
868