1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/init.h>
4 #include <linux/module.h>
5 #include <linux/netfilter.h>
6 #include <linux/rhashtable.h>
7 #include <linux/ip.h>
8 #include <linux/ipv6.h>
9 #include <linux/netdevice.h>
10 #include <linux/if_ether.h>
11 #include <linux/if_vlan.h>
12 #include <net/gre.h>
13 #include <net/gso.h>
14 #include <net/ip.h>
15 #include <net/ipv6.h>
16 #include <net/ip6_route.h>
17 #include <net/ip6_tunnel.h>
18 #include <net/neighbour.h>
19 #include <net/netfilter/nf_flow_table.h>
20 #include <net/netfilter/nf_conntrack_acct.h>
21 /* For layer 4 checksum field offset. */
22 #include <linux/tcp.h>
23 #include <linux/udp.h>
24
nf_flow_state_check(struct flow_offload * flow,int proto,struct sk_buff * skb,unsigned int thoff)25 static int nf_flow_state_check(struct flow_offload *flow, int proto,
26 struct sk_buff *skb, unsigned int thoff)
27 {
28 struct tcphdr *tcph;
29
30 if (proto != IPPROTO_TCP)
31 return 0;
32
33 tcph = (void *)(skb_network_header(skb) + thoff);
34 if (tcph->syn && test_bit(NF_FLOW_CLOSING, &flow->flags)) {
35 flow_offload_teardown(flow);
36 return -1;
37 }
38
39 if ((tcph->fin || tcph->rst) &&
40 !test_bit(NF_FLOW_CLOSING, &flow->flags))
41 set_bit(NF_FLOW_CLOSING, &flow->flags);
42
43 return 0;
44 }
45
nf_flow_nat_ip_tcp(struct sk_buff * skb,unsigned int thoff,__be32 addr,__be32 new_addr)46 static void nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff,
47 __be32 addr, __be32 new_addr)
48 {
49 struct tcphdr *tcph;
50
51 tcph = (void *)(skb_network_header(skb) + thoff);
52 inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true);
53 }
54
nf_flow_nat_ip_udp(struct sk_buff * skb,unsigned int thoff,__be32 addr,__be32 new_addr)55 static void nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff,
56 __be32 addr, __be32 new_addr)
57 {
58 struct udphdr *udph;
59
60 udph = (void *)(skb_network_header(skb) + thoff);
61 if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
62 inet_proto_csum_replace4(&udph->check, skb, addr,
63 new_addr, true);
64 if (!udph->check)
65 udph->check = CSUM_MANGLED_0;
66 }
67 }
68
nf_flow_nat_ip_l4proto(struct sk_buff * skb,struct iphdr * iph,unsigned int thoff,__be32 addr,__be32 new_addr)69 static void nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph,
70 unsigned int thoff, __be32 addr,
71 __be32 new_addr)
72 {
73 switch (iph->protocol) {
74 case IPPROTO_TCP:
75 nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr);
76 break;
77 case IPPROTO_UDP:
78 nf_flow_nat_ip_udp(skb, thoff, addr, new_addr);
79 break;
80 }
81 }
82
nf_flow_snat_ip(const struct flow_offload * flow,struct sk_buff * skb,struct iphdr * iph,unsigned int thoff,enum flow_offload_tuple_dir dir)83 static void nf_flow_snat_ip(const struct flow_offload *flow,
84 struct sk_buff *skb, struct iphdr *iph,
85 unsigned int thoff, enum flow_offload_tuple_dir dir)
86 {
87 __be32 addr, new_addr;
88
89 switch (dir) {
90 case FLOW_OFFLOAD_DIR_ORIGINAL:
91 addr = iph->saddr;
92 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
93 iph->saddr = new_addr;
94 break;
95 case FLOW_OFFLOAD_DIR_REPLY:
96 addr = iph->daddr;
97 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
98 iph->daddr = new_addr;
99 break;
100 }
101 csum_replace4(&iph->check, addr, new_addr);
102
103 nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
104 }
105
nf_flow_dnat_ip(const struct flow_offload * flow,struct sk_buff * skb,struct iphdr * iph,unsigned int thoff,enum flow_offload_tuple_dir dir)106 static void nf_flow_dnat_ip(const struct flow_offload *flow,
107 struct sk_buff *skb, struct iphdr *iph,
108 unsigned int thoff, enum flow_offload_tuple_dir dir)
109 {
110 __be32 addr, new_addr;
111
112 switch (dir) {
113 case FLOW_OFFLOAD_DIR_ORIGINAL:
114 addr = iph->daddr;
115 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
116 iph->daddr = new_addr;
117 break;
118 case FLOW_OFFLOAD_DIR_REPLY:
119 addr = iph->saddr;
120 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
121 iph->saddr = new_addr;
122 break;
123 }
124 csum_replace4(&iph->check, addr, new_addr);
125
126 nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
127 }
128
nf_flow_nat_ip(const struct flow_offload * flow,struct sk_buff * skb,unsigned int thoff,enum flow_offload_tuple_dir dir,struct iphdr * iph)129 static void nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb,
130 unsigned int thoff, enum flow_offload_tuple_dir dir,
131 struct iphdr *iph)
132 {
133 if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
134 nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir);
135 nf_flow_snat_ip(flow, skb, iph, thoff, dir);
136 }
137 if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
138 nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir);
139 nf_flow_dnat_ip(flow, skb, iph, thoff, dir);
140 }
141 }
142
ip_has_options(unsigned int thoff)143 static bool ip_has_options(unsigned int thoff)
144 {
145 return thoff != sizeof(struct iphdr);
146 }
147
148 struct nf_flowtable_ctx {
149 const struct net_device *in;
150 u32 offset;
151 u32 hdrsize;
152 struct {
153 /* Tunnel IP header size */
154 u32 hdr_size;
155 /* IP tunnel protocol */
156 u8 proto;
157 } tun;
158 };
159
nf_flow_tuple_encap(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple * tuple)160 static void nf_flow_tuple_encap(struct nf_flowtable_ctx *ctx,
161 struct sk_buff *skb,
162 struct flow_offload_tuple *tuple)
163 {
164 __be16 inner_proto = skb->protocol;
165 struct vlan_ethhdr *veth;
166 struct pppoe_hdr *phdr;
167 struct ipv6hdr *ip6h;
168 struct iphdr *iph;
169 u16 offset = 0;
170 int i = 0;
171
172 if (skb_vlan_tag_present(skb)) {
173 tuple->encap[i].id = skb_vlan_tag_get(skb);
174 tuple->encap[i].proto = skb->vlan_proto;
175 i++;
176 }
177 switch (skb->protocol) {
178 case htons(ETH_P_8021Q):
179 veth = (struct vlan_ethhdr *)skb_mac_header(skb);
180 tuple->encap[i].id = ntohs(veth->h_vlan_TCI);
181 tuple->encap[i].proto = skb->protocol;
182 inner_proto = veth->h_vlan_encapsulated_proto;
183 offset += VLAN_HLEN;
184 break;
185 case htons(ETH_P_PPP_SES):
186 phdr = (struct pppoe_hdr *)skb_network_header(skb);
187 tuple->encap[i].id = ntohs(phdr->sid);
188 tuple->encap[i].proto = skb->protocol;
189 inner_proto = *((__be16 *)(phdr + 1));
190 offset += PPPOE_SES_HLEN;
191 break;
192 }
193
194 switch (inner_proto) {
195 case htons(ETH_P_IP):
196 iph = (struct iphdr *)(skb_network_header(skb) + offset);
197 if (ctx->tun.proto == IPPROTO_IPIP) {
198 tuple->tun.dst_v4.s_addr = iph->daddr;
199 tuple->tun.src_v4.s_addr = iph->saddr;
200 tuple->tun.l3_proto = IPPROTO_IPIP;
201 }
202 break;
203 case htons(ETH_P_IPV6):
204 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + offset);
205 if (ctx->tun.proto == IPPROTO_IPV6) {
206 tuple->tun.dst_v6 = ip6h->daddr;
207 tuple->tun.src_v6 = ip6h->saddr;
208 tuple->tun.l3_proto = IPPROTO_IPV6;
209 }
210 break;
211 default:
212 break;
213 }
214 }
215
nf_flow_tuple_ip(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple * tuple)216 static int nf_flow_tuple_ip(struct nf_flowtable_ctx *ctx, struct sk_buff *skb,
217 struct flow_offload_tuple *tuple)
218 {
219 struct flow_ports *ports;
220 unsigned int thoff;
221 struct iphdr *iph;
222 u8 ipproto;
223
224 if (!pskb_may_pull(skb, sizeof(*iph) + ctx->offset))
225 return -1;
226
227 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
228 thoff = (iph->ihl * 4);
229
230 if (ip_is_fragment(iph) ||
231 unlikely(ip_has_options(thoff)))
232 return -1;
233
234 thoff += ctx->offset;
235
236 ipproto = iph->protocol;
237 switch (ipproto) {
238 case IPPROTO_TCP:
239 ctx->hdrsize = sizeof(struct tcphdr);
240 break;
241 case IPPROTO_UDP:
242 ctx->hdrsize = sizeof(struct udphdr);
243 break;
244 #ifdef CONFIG_NF_CT_PROTO_GRE
245 case IPPROTO_GRE:
246 ctx->hdrsize = sizeof(struct gre_base_hdr);
247 break;
248 #endif
249 default:
250 return -1;
251 }
252
253 if (iph->ttl <= 1)
254 return -1;
255
256 if (!pskb_may_pull(skb, thoff + ctx->hdrsize))
257 return -1;
258
259 switch (ipproto) {
260 case IPPROTO_TCP:
261 case IPPROTO_UDP:
262 ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
263 tuple->src_port = ports->source;
264 tuple->dst_port = ports->dest;
265 break;
266 case IPPROTO_GRE: {
267 struct gre_base_hdr *greh;
268
269 greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff);
270 if ((greh->flags & GRE_VERSION) != GRE_VERSION_0)
271 return -1;
272 break;
273 }
274 }
275
276 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
277
278 tuple->src_v4.s_addr = iph->saddr;
279 tuple->dst_v4.s_addr = iph->daddr;
280 tuple->l3proto = AF_INET;
281 tuple->l4proto = ipproto;
282 tuple->iifidx = ctx->in->ifindex;
283 nf_flow_tuple_encap(ctx, skb, tuple);
284
285 return 0;
286 }
287
288 /* Based on ip_exceeds_mtu(). */
nf_flow_exceeds_mtu(const struct sk_buff * skb,unsigned int mtu)289 static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu)
290 {
291 if (skb->len <= mtu)
292 return false;
293
294 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
295 return false;
296
297 return true;
298 }
299
nf_flow_dst_check(struct flow_offload_tuple * tuple)300 static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple)
301 {
302 if (tuple->xmit_type != FLOW_OFFLOAD_XMIT_NEIGH &&
303 tuple->xmit_type != FLOW_OFFLOAD_XMIT_XFRM)
304 return true;
305
306 return dst_check(tuple->dst_cache, tuple->dst_cookie);
307 }
308
nf_flow_xmit_xfrm(struct sk_buff * skb,const struct nf_hook_state * state,struct dst_entry * dst)309 static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb,
310 const struct nf_hook_state *state,
311 struct dst_entry *dst)
312 {
313 skb_orphan(skb);
314 skb_dst_set_noref(skb, dst);
315 dst_output(state->net, state->sk, skb);
316 return NF_STOLEN;
317 }
318
nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx * ctx,struct sk_buff * skb)319 static bool nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx *ctx,
320 struct sk_buff *skb)
321 {
322 struct iphdr *iph;
323 u16 size;
324
325 if (!pskb_may_pull(skb, sizeof(*iph) + ctx->offset))
326 return false;
327
328 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
329 size = iph->ihl << 2;
330
331 if (ip_is_fragment(iph) || unlikely(ip_has_options(size)))
332 return false;
333
334 if (iph->ttl <= 1)
335 return false;
336
337 if (iph->protocol == IPPROTO_IPIP) {
338 ctx->tun.proto = IPPROTO_IPIP;
339 ctx->tun.hdr_size = size;
340 ctx->offset += size;
341 }
342
343 return true;
344 }
345
nf_flow_ip6_tunnel_proto(struct nf_flowtable_ctx * ctx,struct sk_buff * skb)346 static bool nf_flow_ip6_tunnel_proto(struct nf_flowtable_ctx *ctx,
347 struct sk_buff *skb)
348 {
349 #if IS_ENABLED(CONFIG_IPV6)
350 struct ipv6hdr *ip6h, _ip6h;
351 __be16 frag_off;
352 u8 nexthdr;
353 int hdrlen;
354
355 ip6h = skb_header_pointer(skb, ctx->offset, sizeof(*ip6h), &_ip6h);
356 if (!ip6h)
357 return false;
358
359 if (ip6h->hop_limit <= 1)
360 return false;
361
362 nexthdr = ip6h->nexthdr;
363 hdrlen = ipv6_skip_exthdr(skb, sizeof(*ip6h) + ctx->offset, &nexthdr,
364 &frag_off);
365 if (hdrlen < 0)
366 return false;
367
368 if (nexthdr == IPPROTO_IPV6) {
369 ctx->tun.hdr_size = hdrlen;
370 ctx->tun.proto = IPPROTO_IPV6;
371 }
372 ctx->offset += ctx->tun.hdr_size;
373
374 return true;
375 #else
376 return false;
377 #endif /* IS_ENABLED(CONFIG_IPV6) */
378 }
379
nf_flow_ip_tunnel_pop(struct nf_flowtable_ctx * ctx,struct sk_buff * skb)380 static void nf_flow_ip_tunnel_pop(struct nf_flowtable_ctx *ctx,
381 struct sk_buff *skb)
382 {
383 if (ctx->tun.proto != IPPROTO_IPIP &&
384 ctx->tun.proto != IPPROTO_IPV6)
385 return;
386
387 skb_pull(skb, ctx->tun.hdr_size);
388 skb_reset_network_header(skb);
389 }
390
nf_flow_skb_encap_protocol(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,__be16 proto)391 static bool nf_flow_skb_encap_protocol(struct nf_flowtable_ctx *ctx,
392 struct sk_buff *skb, __be16 proto)
393 {
394 __be16 inner_proto = skb->protocol;
395 struct vlan_ethhdr *veth;
396 bool ret = false;
397
398 switch (skb->protocol) {
399 case htons(ETH_P_8021Q):
400 if (!pskb_may_pull(skb, skb_mac_offset(skb) + sizeof(*veth)))
401 return false;
402
403 veth = (struct vlan_ethhdr *)skb_mac_header(skb);
404 if (veth->h_vlan_encapsulated_proto == proto) {
405 ctx->offset += VLAN_HLEN;
406 inner_proto = proto;
407 ret = true;
408 }
409 break;
410 case htons(ETH_P_PPP_SES):
411 if (nf_flow_pppoe_proto(skb, &inner_proto) &&
412 inner_proto == proto) {
413 ctx->offset += PPPOE_SES_HLEN;
414 ret = true;
415 }
416 break;
417 }
418
419 switch (inner_proto) {
420 case htons(ETH_P_IP):
421 ret = nf_flow_ip4_tunnel_proto(ctx, skb);
422 break;
423 case htons(ETH_P_IPV6):
424 ret = nf_flow_ip6_tunnel_proto(ctx, skb);
425 break;
426 default:
427 break;
428 }
429
430 return ret;
431 }
432
nf_flow_encap_pop(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple_rhash * tuplehash)433 static void nf_flow_encap_pop(struct nf_flowtable_ctx *ctx,
434 struct sk_buff *skb,
435 struct flow_offload_tuple_rhash *tuplehash)
436 {
437 struct vlan_hdr *vlan_hdr;
438 int i;
439
440 for (i = 0; i < tuplehash->tuple.encap_num; i++) {
441 if (skb_vlan_tag_present(skb)) {
442 __vlan_hwaccel_clear_tag(skb);
443 continue;
444 }
445 switch (skb->protocol) {
446 case htons(ETH_P_8021Q):
447 vlan_hdr = (struct vlan_hdr *)skb->data;
448 skb_pull_rcsum(skb, VLAN_HLEN);
449 vlan_set_encap_proto(skb, vlan_hdr);
450 skb_reset_network_header(skb);
451 break;
452 case htons(ETH_P_PPP_SES):
453 skb->protocol = __nf_flow_pppoe_proto(skb);
454 skb_pull_rcsum(skb, PPPOE_SES_HLEN);
455 skb_reset_network_header(skb);
456 break;
457 }
458 }
459
460 if (skb->protocol == htons(ETH_P_IP) ||
461 skb->protocol == htons(ETH_P_IPV6))
462 nf_flow_ip_tunnel_pop(ctx, skb);
463 }
464
465 static struct flow_offload_tuple_rhash *
nf_flow_offload_lookup(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct sk_buff * skb)466 nf_flow_offload_lookup(struct nf_flowtable_ctx *ctx,
467 struct nf_flowtable *flow_table, struct sk_buff *skb)
468 {
469 struct flow_offload_tuple tuple = {};
470
471 if (!nf_flow_skb_encap_protocol(ctx, skb, htons(ETH_P_IP)))
472 return NULL;
473
474 if (nf_flow_tuple_ip(ctx, skb, &tuple) < 0)
475 return NULL;
476
477 return flow_offload_lookup(flow_table, &tuple);
478 }
479
nf_flow_offload_forward(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct flow_offload_tuple_rhash * tuplehash,struct sk_buff * skb)480 static int nf_flow_offload_forward(struct nf_flowtable_ctx *ctx,
481 struct nf_flowtable *flow_table,
482 struct flow_offload_tuple_rhash *tuplehash,
483 struct sk_buff *skb)
484 {
485 enum flow_offload_tuple_dir dir;
486 struct flow_offload *flow;
487 unsigned int thoff, mtu;
488 struct iphdr *iph;
489
490 dir = tuplehash->tuple.dir;
491 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
492
493 mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset;
494 if (flow->tuplehash[!dir].tuple.tun_num)
495 mtu -= sizeof(*iph);
496
497 if (unlikely(nf_flow_exceeds_mtu(skb, mtu)))
498 return 0;
499
500 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
501 thoff = (iph->ihl * 4) + ctx->offset;
502 if (nf_flow_state_check(flow, iph->protocol, skb, thoff))
503 return 0;
504
505 if (!nf_flow_dst_check(&tuplehash->tuple)) {
506 flow_offload_teardown(flow);
507 return 0;
508 }
509
510 if (skb_ensure_writable(skb, thoff + ctx->hdrsize))
511 return -1;
512
513 flow_offload_refresh(flow_table, flow, false);
514
515 nf_flow_encap_pop(ctx, skb, tuplehash);
516 thoff -= ctx->offset;
517
518 iph = ip_hdr(skb);
519 nf_flow_nat_ip(flow, skb, thoff, dir, iph);
520
521 ip_decrease_ttl(iph);
522 skb_clear_tstamp(skb);
523
524 if (flow_table->flags & NF_FLOWTABLE_COUNTER)
525 nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
526
527 return 1;
528 }
529
530 /* Similar to skb_vlan_push. */
nf_flow_vlan_push(struct sk_buff * skb,__be16 proto,u16 id,u32 needed_headroom)531 static int nf_flow_vlan_push(struct sk_buff *skb, __be16 proto, u16 id,
532 u32 needed_headroom)
533 {
534 if (skb_vlan_tag_present(skb)) {
535 struct vlan_hdr *vhdr;
536
537 if (skb_cow_head(skb, needed_headroom + VLAN_HLEN))
538 return -1;
539
540 __skb_push(skb, VLAN_HLEN);
541 if (skb_mac_header_was_set(skb))
542 skb->mac_header -= VLAN_HLEN;
543
544 vhdr = (struct vlan_hdr *)skb->data;
545 skb->network_header -= VLAN_HLEN;
546 vhdr->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
547 vhdr->h_vlan_encapsulated_proto = skb->protocol;
548 skb->protocol = skb->vlan_proto;
549 skb_postpush_rcsum(skb, skb->data, VLAN_HLEN);
550 }
551 __vlan_hwaccel_put_tag(skb, proto, id);
552
553 return 0;
554 }
555
nf_flow_pppoe_push(struct sk_buff * skb,u16 id,u32 needed_headroom)556 static int nf_flow_pppoe_push(struct sk_buff *skb, u16 id,
557 u32 needed_headroom)
558 {
559 int data_len = skb->len + sizeof(__be16);
560 struct ppp_hdr {
561 struct pppoe_hdr hdr;
562 __be16 proto;
563 } *ph;
564 __be16 proto;
565
566 if (skb_cow_head(skb, needed_headroom + PPPOE_SES_HLEN))
567 return -1;
568
569 switch (skb->protocol) {
570 case htons(ETH_P_IP):
571 proto = htons(PPP_IP);
572 break;
573 case htons(ETH_P_IPV6):
574 proto = htons(PPP_IPV6);
575 break;
576 default:
577 return -1;
578 }
579
580 __skb_push(skb, PPPOE_SES_HLEN);
581 skb_reset_network_header(skb);
582
583 ph = (struct ppp_hdr *)(skb->data);
584 ph->hdr.ver = 1;
585 ph->hdr.type = 1;
586 ph->hdr.code = 0;
587 ph->hdr.sid = htons(id);
588 ph->hdr.length = htons(data_len);
589 ph->proto = proto;
590 skb->protocol = htons(ETH_P_PPP_SES);
591
592 return 0;
593 }
594
nf_flow_tunnel_ipip_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,__be32 * ip_daddr)595 static int nf_flow_tunnel_ipip_push(struct net *net, struct sk_buff *skb,
596 struct flow_offload_tuple *tuple,
597 __be32 *ip_daddr)
598 {
599 struct iphdr *iph = (struct iphdr *)skb_network_header(skb);
600 struct rtable *rt = dst_rtable(tuple->dst_cache);
601 u8 tos = iph->tos, ttl = iph->ttl;
602 __be16 frag_off = iph->frag_off;
603 u32 headroom = sizeof(*iph);
604 int err;
605
606 err = iptunnel_handle_offloads(skb, SKB_GSO_IPXIP4);
607 if (err)
608 return err;
609
610 skb_set_inner_ipproto(skb, IPPROTO_IPIP);
611 headroom += LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len;
612 err = skb_cow_head(skb, headroom);
613 if (err)
614 return err;
615
616 skb_scrub_packet(skb, true);
617 skb_clear_hash_if_not_l4(skb);
618
619 /* Push down and install the IP header. */
620 skb_push(skb, sizeof(*iph));
621 skb_reset_network_header(skb);
622
623 iph = ip_hdr(skb);
624 iph->version = 4;
625 iph->ihl = sizeof(*iph) >> 2;
626 iph->frag_off = ip_mtu_locked(&rt->dst) ? 0 : frag_off;
627 iph->protocol = tuple->tun.l3_proto;
628 iph->tos = tos;
629 iph->daddr = tuple->tun.src_v4.s_addr;
630 iph->saddr = tuple->tun.dst_v4.s_addr;
631 iph->ttl = ttl;
632 iph->tot_len = htons(skb->len);
633 __ip_select_ident(net, iph, skb_shinfo(skb)->gso_segs ?: 1);
634 ip_send_check(iph);
635
636 *ip_daddr = tuple->tun.src_v4.s_addr;
637
638 return 0;
639 }
640
nf_flow_tunnel_v4_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,__be32 * ip_daddr)641 static int nf_flow_tunnel_v4_push(struct net *net, struct sk_buff *skb,
642 struct flow_offload_tuple *tuple,
643 __be32 *ip_daddr)
644 {
645 if (tuple->tun_num)
646 return nf_flow_tunnel_ipip_push(net, skb, tuple, ip_daddr);
647
648 return 0;
649 }
650
651 struct ipv6_tel_txoption {
652 struct ipv6_txoptions ops;
653 __u8 dst_opt[8];
654 };
655
nf_flow_tunnel_ip6ip6_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,struct in6_addr ** ip6_daddr,int encap_limit)656 static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb,
657 struct flow_offload_tuple *tuple,
658 struct in6_addr **ip6_daddr,
659 int encap_limit)
660 {
661 struct ipv6hdr *ip6h = (struct ipv6hdr *)skb_network_header(skb);
662 u8 hop_limit = ip6h->hop_limit, proto = IPPROTO_IPV6;
663 struct rtable *rt = dst_rtable(tuple->dst_cache);
664 __u8 dsfield = ipv6_get_dsfield(ip6h);
665 struct flowi6 fl6 = {
666 .daddr = tuple->tun.src_v6,
667 .saddr = tuple->tun.dst_v6,
668 .flowi6_proto = proto,
669 };
670 int err, mtu;
671 u32 headroom;
672
673 err = iptunnel_handle_offloads(skb, SKB_GSO_IPXIP6);
674 if (err)
675 return err;
676
677 skb_set_inner_ipproto(skb, proto);
678 headroom = sizeof(*ip6h) + LL_RESERVED_SPACE(rt->dst.dev) +
679 rt->dst.header_len;
680 if (encap_limit)
681 headroom += 8;
682 err = skb_cow_head(skb, headroom);
683 if (err)
684 return err;
685
686 skb_scrub_packet(skb, true);
687 mtu = dst_mtu(&rt->dst) - sizeof(*ip6h);
688 if (encap_limit)
689 mtu -= 8;
690 mtu = max(mtu, IPV6_MIN_MTU);
691 skb_dst_update_pmtu_no_confirm(skb, mtu);
692
693 if (encap_limit > 0) {
694 struct ipv6_tel_txoption opt = {
695 .dst_opt[2] = IPV6_TLV_TNL_ENCAP_LIMIT,
696 .dst_opt[3] = 1,
697 .dst_opt[4] = encap_limit,
698 .dst_opt[5] = IPV6_TLV_PADN,
699 .dst_opt[6] = 1,
700 };
701 struct ipv6_opt_hdr *hopt;
702
703 opt.ops.dst1opt = (struct ipv6_opt_hdr *)opt.dst_opt;
704 opt.ops.opt_nflen = 8;
705
706 hopt = skb_push(skb, ipv6_optlen(opt.ops.dst1opt));
707 memcpy(hopt, opt.ops.dst1opt, ipv6_optlen(opt.ops.dst1opt));
708 hopt->nexthdr = IPPROTO_IPV6;
709 proto = NEXTHDR_DEST;
710 }
711
712 skb_push(skb, sizeof(*ip6h));
713 skb_reset_network_header(skb);
714
715 ip6h = ipv6_hdr(skb);
716 ip6_flow_hdr(ip6h, dsfield,
717 ip6_make_flowlabel(net, skb, fl6.flowlabel, true, &fl6));
718 ip6h->hop_limit = hop_limit;
719 ip6h->nexthdr = proto;
720 ip6h->daddr = tuple->tun.src_v6;
721 ip6h->saddr = tuple->tun.dst_v6;
722 ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(*ip6h));
723 IP6CB(skb)->nhoff = offsetof(struct ipv6hdr, nexthdr);
724
725 *ip6_daddr = &tuple->tun.src_v6;
726
727 return 0;
728 }
729
nf_flow_tunnel_v6_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,struct in6_addr ** ip6_daddr,int encap_limit)730 static int nf_flow_tunnel_v6_push(struct net *net, struct sk_buff *skb,
731 struct flow_offload_tuple *tuple,
732 struct in6_addr **ip6_daddr,
733 int encap_limit)
734 {
735 if (tuple->tun_num)
736 return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, ip6_daddr,
737 encap_limit);
738
739 return 0;
740 }
741
nf_flow_encap_push(struct sk_buff * skb,struct flow_offload_tuple * tuple,struct net_device * outdev)742 static int nf_flow_encap_push(struct sk_buff *skb,
743 struct flow_offload_tuple *tuple,
744 struct net_device *outdev)
745 {
746 u32 needed_headroom = LL_RESERVED_SPACE(outdev);
747 int i;
748
749 for (i = tuple->encap_num - 1; i >= 0; i--) {
750 switch (tuple->encap[i].proto) {
751 case htons(ETH_P_8021Q):
752 case htons(ETH_P_8021AD):
753 if (nf_flow_vlan_push(skb, tuple->encap[i].proto,
754 tuple->encap[i].id,
755 needed_headroom) < 0)
756 return -1;
757 break;
758 case htons(ETH_P_PPP_SES):
759 if (nf_flow_pppoe_push(skb, tuple->encap[i].id,
760 needed_headroom) < 0)
761 return -1;
762 break;
763 }
764 }
765
766 return 0;
767 }
768
769 struct nf_flow_xmit {
770 const void *dest;
771 const void *source;
772 struct net_device *outdev;
773 struct flow_offload_tuple *tuple;
774 bool needs_gso_segment;
775 };
776
__nf_flow_queue_xmit(struct net * net,struct sk_buff * skb,struct nf_flow_xmit * xmit)777 static void __nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
778 struct nf_flow_xmit *xmit)
779 {
780 struct net_device *dev = xmit->outdev;
781 unsigned int hh_len = LL_RESERVED_SPACE(dev);
782
783 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
784 skb = skb_expand_head(skb, hh_len);
785 if (!skb)
786 return;
787 }
788
789 skb->dev = dev;
790 dev_hard_header(skb, dev, ntohs(skb->protocol),
791 xmit->dest, xmit->source, skb->len);
792 dev_queue_xmit(skb);
793 }
794
nf_flow_encap_gso_xmit(struct net * net,struct sk_buff * skb,struct nf_flow_xmit * xmit)795 static unsigned int nf_flow_encap_gso_xmit(struct net *net, struct sk_buff *skb,
796 struct nf_flow_xmit *xmit)
797 {
798 struct sk_buff *segs, *nskb;
799
800 segs = skb_gso_segment(skb, 0);
801 if (IS_ERR(segs))
802 return NF_DROP;
803
804 if (segs)
805 consume_skb(skb);
806 else
807 segs = skb;
808
809 skb_list_walk_safe(segs, segs, nskb) {
810 skb_mark_not_on_list(segs);
811
812 if (nf_flow_encap_push(segs, xmit->tuple, xmit->outdev) < 0) {
813 kfree_skb(segs);
814 kfree_skb_list(nskb);
815 return NF_STOLEN;
816 }
817 __nf_flow_queue_xmit(net, segs, xmit);
818 }
819
820 return NF_STOLEN;
821 }
822
nf_flow_queue_xmit(struct net * net,struct sk_buff * skb,struct nf_flow_xmit * xmit)823 static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
824 struct nf_flow_xmit *xmit)
825 {
826 if (xmit->tuple->encap_num) {
827 if (skb_is_gso(skb) && xmit->needs_gso_segment)
828 return nf_flow_encap_gso_xmit(net, skb, xmit);
829
830 if (nf_flow_encap_push(skb, xmit->tuple, xmit->outdev) < 0)
831 return NF_DROP;
832 }
833
834 __nf_flow_queue_xmit(net, skb, xmit);
835
836 return NF_STOLEN;
837 }
838
839 unsigned int
nf_flow_offload_ip_hook(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)840 nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
841 const struct nf_hook_state *state)
842 {
843 struct flow_offload_tuple_rhash *tuplehash;
844 struct nf_flowtable *flow_table = priv;
845 struct flow_offload_tuple *other_tuple;
846 enum flow_offload_tuple_dir dir;
847 struct nf_flowtable_ctx ctx = {
848 .in = state->in,
849 };
850 struct nf_flow_xmit xmit = {};
851 struct flow_offload *flow;
852 struct neighbour *neigh;
853 struct rtable *rt;
854 __be32 ip_daddr;
855 int ret;
856
857 tuplehash = nf_flow_offload_lookup(&ctx, flow_table, skb);
858 if (!tuplehash)
859 return NF_ACCEPT;
860
861 ret = nf_flow_offload_forward(&ctx, flow_table, tuplehash, skb);
862 if (ret < 0)
863 return NF_DROP;
864 else if (ret == 0)
865 return NF_ACCEPT;
866
867 if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) {
868 rt = dst_rtable(tuplehash->tuple.dst_cache);
869 memset(skb->cb, 0, sizeof(struct inet_skb_parm));
870 IPCB(skb)->iif = skb->dev->ifindex;
871 IPCB(skb)->flags = IPSKB_FORWARDED;
872 return nf_flow_xmit_xfrm(skb, state, &rt->dst);
873 }
874
875 dir = tuplehash->tuple.dir;
876 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
877 other_tuple = &flow->tuplehash[!dir].tuple;
878 ip_daddr = other_tuple->src_v4.s_addr;
879
880 if (nf_flow_tunnel_v4_push(state->net, skb, other_tuple, &ip_daddr) < 0)
881 return NF_DROP;
882
883 switch (tuplehash->tuple.xmit_type) {
884 case FLOW_OFFLOAD_XMIT_NEIGH:
885 rt = dst_rtable(tuplehash->tuple.dst_cache);
886 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx);
887 if (!xmit.outdev) {
888 flow_offload_teardown(flow);
889 return NF_DROP;
890 }
891 neigh = ip_neigh_gw4(rt->dst.dev, rt_nexthop(rt, ip_daddr));
892 if (IS_ERR(neigh)) {
893 flow_offload_teardown(flow);
894 return NF_DROP;
895 }
896 xmit.dest = neigh->ha;
897 skb_dst_set_noref(skb, &rt->dst);
898 break;
899 case FLOW_OFFLOAD_XMIT_DIRECT:
900 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx);
901 if (!xmit.outdev) {
902 flow_offload_teardown(flow);
903 return NF_DROP;
904 }
905 xmit.dest = tuplehash->tuple.out.h_dest;
906 xmit.source = tuplehash->tuple.out.h_source;
907 break;
908 default:
909 WARN_ON_ONCE(1);
910 return NF_DROP;
911 }
912 xmit.tuple = other_tuple;
913 xmit.needs_gso_segment = tuplehash->tuple.needs_gso_segment;
914
915 return nf_flow_queue_xmit(state->net, skb, &xmit);
916 }
917 EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook);
918
nf_flow_nat_ipv6_tcp(struct sk_buff * skb,unsigned int thoff,struct in6_addr * addr,struct in6_addr * new_addr,struct ipv6hdr * ip6h)919 static void nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff,
920 struct in6_addr *addr,
921 struct in6_addr *new_addr,
922 struct ipv6hdr *ip6h)
923 {
924 struct tcphdr *tcph;
925
926 tcph = (void *)(skb_network_header(skb) + thoff);
927 inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32,
928 new_addr->s6_addr32, true);
929 }
930
nf_flow_nat_ipv6_udp(struct sk_buff * skb,unsigned int thoff,struct in6_addr * addr,struct in6_addr * new_addr)931 static void nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff,
932 struct in6_addr *addr,
933 struct in6_addr *new_addr)
934 {
935 struct udphdr *udph;
936
937 udph = (void *)(skb_network_header(skb) + thoff);
938 if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
939 inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32,
940 new_addr->s6_addr32, true);
941 if (!udph->check)
942 udph->check = CSUM_MANGLED_0;
943 }
944 }
945
nf_flow_nat_ipv6_l4proto(struct sk_buff * skb,struct ipv6hdr * ip6h,unsigned int thoff,struct in6_addr * addr,struct in6_addr * new_addr)946 static void nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h,
947 unsigned int thoff, struct in6_addr *addr,
948 struct in6_addr *new_addr)
949 {
950 switch (ip6h->nexthdr) {
951 case IPPROTO_TCP:
952 nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr, ip6h);
953 break;
954 case IPPROTO_UDP:
955 nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr);
956 break;
957 }
958 }
959
nf_flow_snat_ipv6(const struct flow_offload * flow,struct sk_buff * skb,struct ipv6hdr * ip6h,unsigned int thoff,enum flow_offload_tuple_dir dir)960 static void nf_flow_snat_ipv6(const struct flow_offload *flow,
961 struct sk_buff *skb, struct ipv6hdr *ip6h,
962 unsigned int thoff,
963 enum flow_offload_tuple_dir dir)
964 {
965 struct in6_addr addr, new_addr;
966
967 switch (dir) {
968 case FLOW_OFFLOAD_DIR_ORIGINAL:
969 addr = ip6h->saddr;
970 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6;
971 ip6h->saddr = new_addr;
972 break;
973 case FLOW_OFFLOAD_DIR_REPLY:
974 addr = ip6h->daddr;
975 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6;
976 ip6h->daddr = new_addr;
977 break;
978 }
979
980 nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
981 }
982
nf_flow_dnat_ipv6(const struct flow_offload * flow,struct sk_buff * skb,struct ipv6hdr * ip6h,unsigned int thoff,enum flow_offload_tuple_dir dir)983 static void nf_flow_dnat_ipv6(const struct flow_offload *flow,
984 struct sk_buff *skb, struct ipv6hdr *ip6h,
985 unsigned int thoff,
986 enum flow_offload_tuple_dir dir)
987 {
988 struct in6_addr addr, new_addr;
989
990 switch (dir) {
991 case FLOW_OFFLOAD_DIR_ORIGINAL:
992 addr = ip6h->daddr;
993 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6;
994 ip6h->daddr = new_addr;
995 break;
996 case FLOW_OFFLOAD_DIR_REPLY:
997 addr = ip6h->saddr;
998 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6;
999 ip6h->saddr = new_addr;
1000 break;
1001 }
1002
1003 nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
1004 }
1005
nf_flow_nat_ipv6(const struct flow_offload * flow,struct sk_buff * skb,enum flow_offload_tuple_dir dir,struct ipv6hdr * ip6h)1006 static void nf_flow_nat_ipv6(const struct flow_offload *flow,
1007 struct sk_buff *skb,
1008 enum flow_offload_tuple_dir dir,
1009 struct ipv6hdr *ip6h)
1010 {
1011 unsigned int thoff = sizeof(*ip6h);
1012
1013 if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
1014 nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir);
1015 nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir);
1016 }
1017 if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
1018 nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir);
1019 nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir);
1020 }
1021 }
1022
nf_flow_tuple_ipv6(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple * tuple)1023 static int nf_flow_tuple_ipv6(struct nf_flowtable_ctx *ctx, struct sk_buff *skb,
1024 struct flow_offload_tuple *tuple)
1025 {
1026 struct flow_ports *ports;
1027 struct ipv6hdr *ip6h;
1028 unsigned int thoff;
1029 u8 nexthdr;
1030
1031 thoff = sizeof(*ip6h) + ctx->offset;
1032 if (!pskb_may_pull(skb, thoff))
1033 return -1;
1034
1035 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset);
1036
1037 nexthdr = ip6h->nexthdr;
1038 switch (nexthdr) {
1039 case IPPROTO_TCP:
1040 ctx->hdrsize = sizeof(struct tcphdr);
1041 break;
1042 case IPPROTO_UDP:
1043 ctx->hdrsize = sizeof(struct udphdr);
1044 break;
1045 #ifdef CONFIG_NF_CT_PROTO_GRE
1046 case IPPROTO_GRE:
1047 ctx->hdrsize = sizeof(struct gre_base_hdr);
1048 break;
1049 #endif
1050 default:
1051 return -1;
1052 }
1053
1054 if (ip6h->hop_limit <= 1)
1055 return -1;
1056
1057 if (!pskb_may_pull(skb, thoff + ctx->hdrsize))
1058 return -1;
1059
1060 switch (nexthdr) {
1061 case IPPROTO_TCP:
1062 case IPPROTO_UDP:
1063 ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
1064 tuple->src_port = ports->source;
1065 tuple->dst_port = ports->dest;
1066 break;
1067 case IPPROTO_GRE: {
1068 struct gre_base_hdr *greh;
1069
1070 greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff);
1071 if ((greh->flags & GRE_VERSION) != GRE_VERSION_0)
1072 return -1;
1073 break;
1074 }
1075 }
1076
1077 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset);
1078
1079 tuple->src_v6 = ip6h->saddr;
1080 tuple->dst_v6 = ip6h->daddr;
1081 tuple->l3proto = AF_INET6;
1082 tuple->l4proto = nexthdr;
1083 tuple->iifidx = ctx->in->ifindex;
1084 nf_flow_tuple_encap(ctx, skb, tuple);
1085
1086 return 0;
1087 }
1088
nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct flow_offload_tuple_rhash * tuplehash,struct sk_buff * skb,int encap_limit)1089 static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx,
1090 struct nf_flowtable *flow_table,
1091 struct flow_offload_tuple_rhash *tuplehash,
1092 struct sk_buff *skb, int encap_limit)
1093 {
1094 enum flow_offload_tuple_dir dir;
1095 struct flow_offload *flow;
1096 unsigned int thoff, mtu;
1097 struct ipv6hdr *ip6h;
1098
1099 dir = tuplehash->tuple.dir;
1100 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
1101
1102 mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset;
1103 if (flow->tuplehash[!dir].tuple.tun_num) {
1104 mtu -= sizeof(*ip6h);
1105 if (encap_limit > 0)
1106 mtu -= 8; /* encap limit option */
1107 }
1108
1109 if (unlikely(nf_flow_exceeds_mtu(skb, mtu)))
1110 return 0;
1111
1112 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset);
1113 thoff = sizeof(*ip6h) + ctx->offset;
1114 if (nf_flow_state_check(flow, ip6h->nexthdr, skb, thoff))
1115 return 0;
1116
1117 if (!nf_flow_dst_check(&tuplehash->tuple)) {
1118 flow_offload_teardown(flow);
1119 return 0;
1120 }
1121
1122 if (skb_ensure_writable(skb, thoff + ctx->hdrsize))
1123 return -1;
1124
1125 flow_offload_refresh(flow_table, flow, false);
1126
1127 nf_flow_encap_pop(ctx, skb, tuplehash);
1128
1129 ip6h = ipv6_hdr(skb);
1130 nf_flow_nat_ipv6(flow, skb, dir, ip6h);
1131
1132 ip6h->hop_limit--;
1133 skb_clear_tstamp(skb);
1134
1135 if (flow_table->flags & NF_FLOWTABLE_COUNTER)
1136 nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
1137
1138 return 1;
1139 }
1140
1141 static struct flow_offload_tuple_rhash *
nf_flow_offload_ipv6_lookup(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct sk_buff * skb)1142 nf_flow_offload_ipv6_lookup(struct nf_flowtable_ctx *ctx,
1143 struct nf_flowtable *flow_table,
1144 struct sk_buff *skb)
1145 {
1146 struct flow_offload_tuple tuple = {};
1147
1148 if (!nf_flow_skb_encap_protocol(ctx, skb, htons(ETH_P_IPV6)))
1149 return NULL;
1150
1151 if (nf_flow_tuple_ipv6(ctx, skb, &tuple) < 0)
1152 return NULL;
1153
1154 return flow_offload_lookup(flow_table, &tuple);
1155 }
1156
1157 unsigned int
nf_flow_offload_ipv6_hook(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)1158 nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
1159 const struct nf_hook_state *state)
1160 {
1161 int encap_limit = IPV6_DEFAULT_TNL_ENCAP_LIMIT;
1162 struct flow_offload_tuple_rhash *tuplehash;
1163 struct nf_flowtable *flow_table = priv;
1164 struct flow_offload_tuple *other_tuple;
1165 enum flow_offload_tuple_dir dir;
1166 struct nf_flowtable_ctx ctx = {
1167 .in = state->in,
1168 };
1169 struct nf_flow_xmit xmit = {};
1170 struct in6_addr *ip6_daddr;
1171 struct flow_offload *flow;
1172 struct neighbour *neigh;
1173 struct rt6_info *rt;
1174 int ret;
1175
1176 tuplehash = nf_flow_offload_ipv6_lookup(&ctx, flow_table, skb);
1177 if (tuplehash == NULL)
1178 return NF_ACCEPT;
1179
1180 ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, skb,
1181 encap_limit);
1182 if (ret < 0)
1183 return NF_DROP;
1184 else if (ret == 0)
1185 return NF_ACCEPT;
1186
1187 if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) {
1188 rt = dst_rt6_info(tuplehash->tuple.dst_cache);
1189 memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
1190 IP6CB(skb)->iif = skb->dev->ifindex;
1191 IP6CB(skb)->flags = IP6SKB_FORWARDED;
1192 return nf_flow_xmit_xfrm(skb, state, &rt->dst);
1193 }
1194
1195 dir = tuplehash->tuple.dir;
1196 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
1197 other_tuple = &flow->tuplehash[!dir].tuple;
1198 ip6_daddr = &other_tuple->src_v6;
1199
1200 if (nf_flow_tunnel_v6_push(state->net, skb, other_tuple,
1201 &ip6_daddr, encap_limit) < 0)
1202 return NF_DROP;
1203
1204 switch (tuplehash->tuple.xmit_type) {
1205 case FLOW_OFFLOAD_XMIT_NEIGH:
1206 rt = dst_rt6_info(tuplehash->tuple.dst_cache);
1207 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx);
1208 if (!xmit.outdev) {
1209 flow_offload_teardown(flow);
1210 return NF_DROP;
1211 }
1212 neigh = ip_neigh_gw6(rt->dst.dev, rt6_nexthop(rt, ip6_daddr));
1213 if (IS_ERR(neigh)) {
1214 flow_offload_teardown(flow);
1215 return NF_DROP;
1216 }
1217 xmit.dest = neigh->ha;
1218 skb_dst_set_noref(skb, &rt->dst);
1219 break;
1220 case FLOW_OFFLOAD_XMIT_DIRECT:
1221 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx);
1222 if (!xmit.outdev) {
1223 flow_offload_teardown(flow);
1224 return NF_DROP;
1225 }
1226 xmit.dest = tuplehash->tuple.out.h_dest;
1227 xmit.source = tuplehash->tuple.out.h_source;
1228 break;
1229 default:
1230 WARN_ON_ONCE(1);
1231 return NF_DROP;
1232 }
1233 xmit.tuple = other_tuple;
1234 xmit.needs_gso_segment = tuplehash->tuple.needs_gso_segment;
1235
1236 return nf_flow_queue_xmit(state->net, skb, &xmit);
1237 }
1238 EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook);
1239