1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/init.h>
4 #include <linux/module.h>
5 #include <linux/netfilter.h>
6 #include <linux/rhashtable.h>
7 #include <linux/ip.h>
8 #include <linux/ipv6.h>
9 #include <linux/netdevice.h>
10 #include <linux/if_ether.h>
11 #include <linux/if_vlan.h>
12 #include <net/gre.h>
13 #include <net/gso.h>
14 #include <net/ip.h>
15 #include <net/ipv6.h>
16 #include <net/ip6_route.h>
17 #include <net/ip6_tunnel.h>
18 #include <net/neighbour.h>
19 #include <net/netfilter/nf_flow_table.h>
20 #include <net/netfilter/nf_conntrack_acct.h>
21 /* For layer 4 checksum field offset. */
22 #include <linux/tcp.h>
23 #include <linux/udp.h>
24
nf_flow_state_check(struct flow_offload * flow,int proto,struct sk_buff * skb,unsigned int thoff)25 static int nf_flow_state_check(struct flow_offload *flow, int proto,
26 struct sk_buff *skb, unsigned int thoff)
27 {
28 struct tcphdr *tcph;
29
30 if (proto != IPPROTO_TCP)
31 return 0;
32
33 tcph = (void *)(skb_network_header(skb) + thoff);
34 if (tcph->syn && test_bit(NF_FLOW_CLOSING, &flow->flags)) {
35 flow_offload_teardown(flow);
36 return -1;
37 }
38
39 if ((tcph->fin || tcph->rst) &&
40 !test_bit(NF_FLOW_CLOSING, &flow->flags))
41 set_bit(NF_FLOW_CLOSING, &flow->flags);
42
43 return 0;
44 }
45
nf_flow_nat_ip_tcp(struct sk_buff * skb,unsigned int thoff,__be32 addr,__be32 new_addr)46 static void nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff,
47 __be32 addr, __be32 new_addr)
48 {
49 struct tcphdr *tcph;
50
51 tcph = (void *)(skb_network_header(skb) + thoff);
52 inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true);
53 }
54
nf_flow_nat_ip_udp(struct sk_buff * skb,unsigned int thoff,__be32 addr,__be32 new_addr)55 static void nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff,
56 __be32 addr, __be32 new_addr)
57 {
58 struct udphdr *udph;
59
60 udph = (void *)(skb_network_header(skb) + thoff);
61 if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
62 inet_proto_csum_replace4(&udph->check, skb, addr,
63 new_addr, true);
64 if (!udph->check)
65 udph->check = CSUM_MANGLED_0;
66 }
67 }
68
nf_flow_nat_ip_l4proto(struct sk_buff * skb,struct iphdr * iph,unsigned int thoff,__be32 addr,__be32 new_addr)69 static void nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph,
70 unsigned int thoff, __be32 addr,
71 __be32 new_addr)
72 {
73 switch (iph->protocol) {
74 case IPPROTO_TCP:
75 nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr);
76 break;
77 case IPPROTO_UDP:
78 nf_flow_nat_ip_udp(skb, thoff, addr, new_addr);
79 break;
80 }
81 }
82
nf_flow_snat_ip(const struct flow_offload * flow,struct sk_buff * skb,struct iphdr * iph,unsigned int thoff,enum flow_offload_tuple_dir dir)83 static void nf_flow_snat_ip(const struct flow_offload *flow,
84 struct sk_buff *skb, struct iphdr *iph,
85 unsigned int thoff, enum flow_offload_tuple_dir dir)
86 {
87 __be32 addr, new_addr;
88
89 switch (dir) {
90 case FLOW_OFFLOAD_DIR_ORIGINAL:
91 addr = iph->saddr;
92 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
93 iph->saddr = new_addr;
94 break;
95 case FLOW_OFFLOAD_DIR_REPLY:
96 addr = iph->daddr;
97 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
98 iph->daddr = new_addr;
99 break;
100 }
101 csum_replace4(&iph->check, addr, new_addr);
102
103 nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
104 }
105
nf_flow_dnat_ip(const struct flow_offload * flow,struct sk_buff * skb,struct iphdr * iph,unsigned int thoff,enum flow_offload_tuple_dir dir)106 static void nf_flow_dnat_ip(const struct flow_offload *flow,
107 struct sk_buff *skb, struct iphdr *iph,
108 unsigned int thoff, enum flow_offload_tuple_dir dir)
109 {
110 __be32 addr, new_addr;
111
112 switch (dir) {
113 case FLOW_OFFLOAD_DIR_ORIGINAL:
114 addr = iph->daddr;
115 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
116 iph->daddr = new_addr;
117 break;
118 case FLOW_OFFLOAD_DIR_REPLY:
119 addr = iph->saddr;
120 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
121 iph->saddr = new_addr;
122 break;
123 }
124 csum_replace4(&iph->check, addr, new_addr);
125
126 nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
127 }
128
nf_flow_nat_ip(const struct flow_offload * flow,struct sk_buff * skb,unsigned int thoff,enum flow_offload_tuple_dir dir,struct iphdr * iph)129 static void nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb,
130 unsigned int thoff, enum flow_offload_tuple_dir dir,
131 struct iphdr *iph)
132 {
133 if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
134 nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir);
135 nf_flow_snat_ip(flow, skb, iph, thoff, dir);
136 }
137 if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
138 nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir);
139 nf_flow_dnat_ip(flow, skb, iph, thoff, dir);
140 }
141 }
142
ip_has_options(unsigned int thoff)143 static bool ip_has_options(unsigned int thoff)
144 {
145 return thoff != sizeof(struct iphdr);
146 }
147
148 struct nf_flowtable_ctx {
149 const struct net_device *in;
150 __be16 ether_type;
151 u32 offset;
152 u32 hdrsize;
153 struct {
154 /* Tunnel IP header size */
155 u32 hdr_size;
156 /* IP tunnel protocol */
157 u8 inner_proto;
158 } tun;
159 };
160
nf_flow_tuple_encap(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple * tuple)161 static void nf_flow_tuple_encap(struct nf_flowtable_ctx *ctx,
162 struct sk_buff *skb,
163 struct flow_offload_tuple *tuple)
164 {
165 struct vlan_ethhdr *veth;
166 struct pppoe_hdr *phdr;
167 struct ipv6hdr *ip6h;
168 struct iphdr *iph;
169 u16 offset = 0;
170 int i = 0;
171
172 if (skb_vlan_tag_present(skb)) {
173 tuple->encap[i].id = skb_vlan_tag_get(skb);
174 tuple->encap[i].proto = skb->vlan_proto;
175 i++;
176 }
177 switch (skb->protocol) {
178 case htons(ETH_P_8021Q):
179 veth = (struct vlan_ethhdr *)skb_mac_header(skb);
180 tuple->encap[i].id = ntohs(veth->h_vlan_TCI);
181 tuple->encap[i].proto = skb->protocol;
182 offset += VLAN_HLEN;
183 break;
184 case htons(ETH_P_PPP_SES):
185 phdr = (struct pppoe_hdr *)skb_network_header(skb);
186 tuple->encap[i].id = ntohs(phdr->sid);
187 tuple->encap[i].proto = skb->protocol;
188 offset += PPPOE_SES_HLEN;
189 break;
190 }
191
192 switch (ctx->ether_type) {
193 case htons(ETH_P_IP):
194 iph = (struct iphdr *)(skb_network_header(skb) + offset);
195 if (ctx->tun.inner_proto == IPPROTO_IPIP) {
196 tuple->tun.dst_v4.s_addr = iph->daddr;
197 tuple->tun.src_v4.s_addr = iph->saddr;
198 tuple->tun.inner_proto = IPPROTO_IPIP;
199 }
200 break;
201 case htons(ETH_P_IPV6):
202 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + offset);
203 if (ctx->tun.inner_proto == IPPROTO_IPV6) {
204 tuple->tun.dst_v6 = ip6h->daddr;
205 tuple->tun.src_v6 = ip6h->saddr;
206 tuple->tun.inner_proto = IPPROTO_IPV6;
207 }
208 break;
209 default:
210 break;
211 }
212 }
213
nf_flow_tuple_ip(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple * tuple)214 static int nf_flow_tuple_ip(struct nf_flowtable_ctx *ctx, struct sk_buff *skb,
215 struct flow_offload_tuple *tuple)
216 {
217 struct flow_ports *ports;
218 unsigned int thoff;
219 struct iphdr *iph;
220 u8 ipproto;
221
222 if (!pskb_may_pull(skb, sizeof(*iph) + ctx->offset))
223 return -1;
224
225 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
226 thoff = (iph->ihl * 4);
227
228 if (ip_is_fragment(iph) ||
229 unlikely(ip_has_options(thoff)))
230 return -1;
231
232 thoff += ctx->offset;
233
234 ipproto = iph->protocol;
235 switch (ipproto) {
236 case IPPROTO_TCP:
237 ctx->hdrsize = sizeof(struct tcphdr);
238 break;
239 case IPPROTO_UDP:
240 ctx->hdrsize = sizeof(struct udphdr);
241 break;
242 #ifdef CONFIG_NF_CT_PROTO_GRE
243 case IPPROTO_GRE:
244 ctx->hdrsize = sizeof(struct gre_base_hdr);
245 break;
246 #endif
247 default:
248 return -1;
249 }
250
251 if (iph->ttl <= 1)
252 return -1;
253
254 if (!pskb_may_pull(skb, thoff + ctx->hdrsize))
255 return -1;
256
257 switch (ipproto) {
258 case IPPROTO_TCP:
259 case IPPROTO_UDP:
260 ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
261 tuple->src_port = ports->source;
262 tuple->dst_port = ports->dest;
263 break;
264 case IPPROTO_GRE: {
265 struct gre_base_hdr *greh;
266
267 greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff);
268 if ((greh->flags & GRE_VERSION) != GRE_VERSION_0)
269 return -1;
270 break;
271 }
272 }
273
274 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
275
276 tuple->src_v4.s_addr = iph->saddr;
277 tuple->dst_v4.s_addr = iph->daddr;
278 tuple->l3proto = AF_INET;
279 tuple->l4proto = ipproto;
280 tuple->iifidx = ctx->in->ifindex;
281 nf_flow_tuple_encap(ctx, skb, tuple);
282
283 return 0;
284 }
285
286 /* Based on ip_exceeds_mtu(). */
nf_flow_exceeds_mtu(const struct sk_buff * skb,unsigned int mtu)287 static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu)
288 {
289 if (skb->len <= mtu)
290 return false;
291
292 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
293 return false;
294
295 return true;
296 }
297
nf_flow_xmit_xfrm(struct sk_buff * skb,const struct nf_hook_state * state,struct dst_entry * dst)298 static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb,
299 const struct nf_hook_state *state,
300 struct dst_entry *dst)
301 {
302 skb_orphan(skb);
303 skb_dst_drop(skb);
304 skb_dst_set_noref(skb, dst);
305 dst_output(state->net, state->sk, skb);
306 return NF_STOLEN;
307 }
308
nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx * ctx,struct sk_buff * skb)309 static bool nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx *ctx,
310 struct sk_buff *skb)
311 {
312 struct iphdr *iph;
313 u16 size;
314
315 if (!pskb_may_pull(skb, sizeof(*iph) + ctx->offset))
316 return false;
317
318 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
319 if (iph->ihl < 5)
320 return false;
321
322 size = iph->ihl << 2;
323 if (ip_is_fragment(iph) || unlikely(ip_has_options(size)))
324 return false;
325
326 if (iph->ttl <= 1)
327 return false;
328
329 if (iph->protocol == IPPROTO_IPIP) {
330 ctx->tun.inner_proto = iph->protocol;
331 ctx->tun.hdr_size = size;
332 ctx->offset += ctx->tun.hdr_size;
333 }
334
335 return true;
336 }
337
nf_flow_ip6_tunnel_proto(struct nf_flowtable_ctx * ctx,struct sk_buff * skb)338 static bool nf_flow_ip6_tunnel_proto(struct nf_flowtable_ctx *ctx,
339 struct sk_buff *skb)
340 {
341 #if IS_ENABLED(CONFIG_IPV6)
342 struct ipv6hdr *ip6h;
343
344 if (!pskb_may_pull(skb, sizeof(*ip6h) + ctx->offset))
345 return false;
346
347 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset);
348 if (ip6h->hop_limit <= 1)
349 return false;
350
351 if (ipv6_ext_hdr(ip6h->nexthdr))
352 return false;
353
354 if (ip6h->nexthdr == IPPROTO_IPV6) {
355 ctx->tun.inner_proto = ip6h->nexthdr;
356 ctx->tun.hdr_size = sizeof(*ip6h);
357 ctx->offset += ctx->tun.hdr_size;
358 }
359
360 return true;
361 #else
362 return false;
363 #endif /* IS_ENABLED(CONFIG_IPV6) */
364 }
365
nf_flow_ip_tunnel_pop(struct nf_flowtable_ctx * ctx,struct sk_buff * skb)366 static void nf_flow_ip_tunnel_pop(struct nf_flowtable_ctx *ctx,
367 struct sk_buff *skb)
368 {
369 if (ctx->tun.inner_proto != IPPROTO_IPIP &&
370 ctx->tun.inner_proto != IPPROTO_IPV6)
371 return;
372
373 skb_pull(skb, ctx->tun.hdr_size);
374 skb_reset_network_header(skb);
375 }
376
nf_flow_skb_encap_protocol(struct nf_flowtable_ctx * ctx,struct sk_buff * skb)377 static bool nf_flow_skb_encap_protocol(struct nf_flowtable_ctx *ctx,
378 struct sk_buff *skb)
379 {
380 struct vlan_ethhdr *veth;
381 __be16 ether_type;
382 bool ret = false;
383
384 switch (skb->protocol) {
385 case htons(ETH_P_8021Q):
386 if (!pskb_may_pull(skb, skb_mac_offset(skb) + sizeof(*veth)))
387 return false;
388
389 veth = (struct vlan_ethhdr *)skb_mac_header(skb);
390 ctx->ether_type = veth->h_vlan_encapsulated_proto;
391 ctx->offset += VLAN_HLEN;
392 ret = true;
393 break;
394 case htons(ETH_P_PPP_SES):
395 if (!nf_flow_pppoe_proto(skb, ðer_type))
396 return false;
397
398 ctx->ether_type = ether_type;
399 ctx->offset += PPPOE_SES_HLEN;
400 ret = true;
401 break;
402 case htons(ETH_P_IP):
403 case htons(ETH_P_IPV6):
404 ctx->ether_type = skb->protocol;
405 break;
406 default:
407 return false;
408 }
409
410 switch (ctx->ether_type) {
411 case htons(ETH_P_IP):
412 ret = nf_flow_ip4_tunnel_proto(ctx, skb);
413 break;
414 case htons(ETH_P_IPV6):
415 ret = nf_flow_ip6_tunnel_proto(ctx, skb);
416 break;
417 default:
418 break;
419 }
420
421 return ret;
422 }
423
nf_flow_encap_pop(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple_rhash * tuplehash)424 static void nf_flow_encap_pop(struct nf_flowtable_ctx *ctx,
425 struct sk_buff *skb,
426 struct flow_offload_tuple_rhash *tuplehash)
427 {
428 struct vlan_hdr *vlan_hdr;
429 int i;
430
431 for (i = 0; i < tuplehash->tuple.encap_num; i++) {
432 if (skb_vlan_tag_present(skb)) {
433 __vlan_hwaccel_clear_tag(skb);
434 continue;
435 }
436 switch (skb->protocol) {
437 case htons(ETH_P_8021Q):
438 vlan_hdr = (struct vlan_hdr *)skb->data;
439 skb_pull_rcsum(skb, VLAN_HLEN);
440 vlan_set_encap_proto(skb, vlan_hdr);
441 skb_reset_network_header(skb);
442 break;
443 case htons(ETH_P_PPP_SES):
444 skb->protocol = __nf_flow_pppoe_proto(skb);
445 skb_pull_rcsum(skb, PPPOE_SES_HLEN);
446 skb_reset_network_header(skb);
447 break;
448 }
449 }
450
451 if (skb->protocol == htons(ETH_P_IP) ||
452 skb->protocol == htons(ETH_P_IPV6))
453 nf_flow_ip_tunnel_pop(ctx, skb);
454 }
455
456 static struct flow_offload_tuple_rhash *
nf_flow_offload_lookup(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct sk_buff * skb)457 nf_flow_offload_lookup(struct nf_flowtable_ctx *ctx,
458 struct nf_flowtable *flow_table, struct sk_buff *skb)
459 {
460 struct flow_offload_tuple tuple = {};
461
462 if (nf_flow_tuple_ip(ctx, skb, &tuple) < 0)
463 return NULL;
464
465 return flow_offload_lookup(flow_table, &tuple);
466 }
467
nf_flow_offload_forward(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct flow_offload_tuple_rhash * tuplehash,struct sk_buff * skb)468 static int nf_flow_offload_forward(struct nf_flowtable_ctx *ctx,
469 struct nf_flowtable *flow_table,
470 struct flow_offload_tuple_rhash *tuplehash,
471 struct sk_buff *skb)
472 {
473 enum flow_offload_tuple_dir dir;
474 struct flow_offload *flow;
475 unsigned int thoff, mtu;
476 struct iphdr *iph;
477
478 dir = tuplehash->tuple.dir;
479 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
480
481 mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset;
482 if (flow->tuplehash[!dir].tuple.tun_num)
483 mtu -= sizeof(*iph);
484
485 if (unlikely(nf_flow_exceeds_mtu(skb, mtu)))
486 return 0;
487
488 iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset);
489 thoff = (iph->ihl * 4) + ctx->offset;
490 if (nf_flow_state_check(flow, iph->protocol, skb, thoff))
491 return 0;
492
493 if (!nf_flow_dst_check(&tuplehash->tuple)) {
494 flow_offload_teardown(flow);
495 return 0;
496 }
497
498 if (skb_ensure_writable(skb, thoff + ctx->hdrsize))
499 return -1;
500
501 flow_offload_refresh(flow_table, flow, false);
502
503 nf_flow_encap_pop(ctx, skb, tuplehash);
504 thoff -= ctx->offset;
505
506 iph = ip_hdr(skb);
507 nf_flow_nat_ip(flow, skb, thoff, dir, iph);
508
509 ip_decrease_ttl(iph);
510 skb_clear_tstamp(skb);
511
512 if (flow_table->flags & NF_FLOWTABLE_COUNTER)
513 nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
514
515 return 1;
516 }
517
518 /* Similar to skb_vlan_push. */
nf_flow_vlan_push(struct sk_buff * skb,__be16 proto,u16 id,u32 needed_headroom)519 static int nf_flow_vlan_push(struct sk_buff *skb, __be16 proto, u16 id,
520 u32 needed_headroom)
521 {
522 if (skb_vlan_tag_present(skb)) {
523 struct vlan_hdr *vhdr;
524
525 if (skb_cow_head(skb, needed_headroom + VLAN_HLEN))
526 return -1;
527
528 __skb_push(skb, VLAN_HLEN);
529 if (skb_mac_header_was_set(skb))
530 skb->mac_header -= VLAN_HLEN;
531
532 vhdr = (struct vlan_hdr *)skb->data;
533 skb->network_header -= VLAN_HLEN;
534 vhdr->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
535 vhdr->h_vlan_encapsulated_proto = skb->protocol;
536 skb->protocol = skb->vlan_proto;
537 skb_postpush_rcsum(skb, skb->data, VLAN_HLEN);
538 }
539 __vlan_hwaccel_put_tag(skb, proto, id);
540
541 return 0;
542 }
543
nf_flow_pppoe_push(struct sk_buff * skb,u16 id,u32 needed_headroom)544 static int nf_flow_pppoe_push(struct sk_buff *skb, u16 id,
545 u32 needed_headroom)
546 {
547 int data_len = skb->len + sizeof(__be16);
548 struct ppp_hdr {
549 struct pppoe_hdr hdr;
550 __be16 proto;
551 } *ph;
552 __be16 proto;
553
554 if (skb_cow_head(skb, needed_headroom + PPPOE_SES_HLEN))
555 return -1;
556
557 switch (skb->protocol) {
558 case htons(ETH_P_IP):
559 proto = htons(PPP_IP);
560 break;
561 case htons(ETH_P_IPV6):
562 proto = htons(PPP_IPV6);
563 break;
564 default:
565 return -1;
566 }
567
568 __skb_push(skb, PPPOE_SES_HLEN);
569 skb_reset_network_header(skb);
570
571 ph = (struct ppp_hdr *)(skb->data);
572 ph->hdr.ver = 1;
573 ph->hdr.type = 1;
574 ph->hdr.code = 0;
575 ph->hdr.sid = htons(id);
576 ph->hdr.length = htons(data_len);
577 ph->proto = proto;
578 skb->protocol = htons(ETH_P_PPP_SES);
579
580 return 0;
581 }
582
nf_flow_tunnel_ipip_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,struct dst_entry * dst,__be32 * ip_daddr)583 static int nf_flow_tunnel_ipip_push(struct net *net, struct sk_buff *skb,
584 struct flow_offload_tuple *tuple,
585 struct dst_entry *dst, __be32 *ip_daddr)
586 {
587 struct iphdr *iph = (struct iphdr *)skb_network_header(skb);
588 struct rtable *rt = dst_rtable(dst);
589 u8 tos = iph->tos, ttl = iph->ttl;
590 __be16 frag_off = iph->frag_off;
591 u32 headroom = sizeof(*iph);
592 int err;
593
594 err = iptunnel_handle_offloads(skb, SKB_GSO_IPXIP4);
595 if (err)
596 return err;
597
598 skb_set_inner_ipproto(skb, IPPROTO_IPIP);
599 headroom += LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len;
600 err = skb_cow_head(skb, headroom);
601 if (err)
602 return err;
603
604 skb_scrub_packet(skb, true);
605 skb_clear_hash_if_not_l4(skb);
606
607 /* Push down and install the IP header. */
608 skb_push(skb, sizeof(*iph));
609 skb_reset_network_header(skb);
610
611 iph = ip_hdr(skb);
612 iph->version = 4;
613 iph->ihl = sizeof(*iph) >> 2;
614 iph->frag_off = ip_mtu_locked(&rt->dst) ? 0 : frag_off;
615 iph->protocol = tuple->tun.inner_proto;
616 iph->tos = tos;
617 iph->daddr = tuple->tun.src_v4.s_addr;
618 iph->saddr = tuple->tun.dst_v4.s_addr;
619 iph->ttl = ttl;
620 iph->tot_len = htons(skb->len);
621 __ip_select_ident(net, iph, skb_shinfo(skb)->gso_segs ?: 1);
622 ip_send_check(iph);
623
624 *ip_daddr = tuple->tun.src_v4.s_addr;
625
626 return 0;
627 }
628
nf_flow_tunnel_v4_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,struct dst_entry * dst,__be32 * ip_daddr)629 static int nf_flow_tunnel_v4_push(struct net *net, struct sk_buff *skb,
630 struct flow_offload_tuple *tuple,
631 struct dst_entry *dst, __be32 *ip_daddr)
632 {
633 if (tuple->tun_num)
634 return nf_flow_tunnel_ipip_push(net, skb, tuple, dst, ip_daddr);
635
636 return 0;
637 }
638
nf_flow_tunnel_ip6ip6_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,struct dst_entry * dst,struct in6_addr ** ip6_daddr)639 static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb,
640 struct flow_offload_tuple *tuple,
641 struct dst_entry *dst,
642 struct in6_addr **ip6_daddr)
643 {
644 struct ipv6hdr *ip6h = (struct ipv6hdr *)skb_network_header(skb);
645 __u8 dsfield = ipv6_get_dsfield(ip6h);
646 struct rtable *rt = dst_rtable(dst);
647 struct flowi6 fl6 = {
648 .daddr = tuple->tun.src_v6,
649 .saddr = tuple->tun.dst_v6,
650 .flowi6_proto = IPPROTO_IPV6,
651 };
652 u8 hop_limit = ip6h->hop_limit;
653 int err, mtu;
654 u32 headroom;
655
656 err = iptunnel_handle_offloads(skb, SKB_GSO_IPXIP6);
657 if (err)
658 return err;
659
660 skb_set_inner_ipproto(skb, IPPROTO_IPV6);
661 headroom = sizeof(*ip6h) + LL_RESERVED_SPACE(rt->dst.dev) +
662 rt->dst.header_len;
663 err = skb_cow_head(skb, headroom);
664 if (err)
665 return err;
666
667 skb_scrub_packet(skb, true);
668 mtu = dst_mtu(&rt->dst) - sizeof(*ip6h);
669 mtu = max(mtu, IPV6_MIN_MTU);
670 skb_dst_update_pmtu_no_confirm(skb, mtu);
671
672 skb_push(skb, sizeof(*ip6h));
673 skb_reset_network_header(skb);
674
675 ip6h = ipv6_hdr(skb);
676 ip6_flow_hdr(ip6h, dsfield,
677 ip6_make_flowlabel(net, skb, fl6.flowlabel, true, &fl6));
678 ip6h->hop_limit = hop_limit;
679 ip6h->nexthdr = IPPROTO_IPV6;
680 ip6h->daddr = tuple->tun.src_v6;
681 ip6h->saddr = tuple->tun.dst_v6;
682 ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(*ip6h));
683 IP6CB(skb)->nhoff = offsetof(struct ipv6hdr, nexthdr);
684
685 *ip6_daddr = &tuple->tun.src_v6;
686
687 return 0;
688 }
689
nf_flow_tunnel_v6_push(struct net * net,struct sk_buff * skb,struct flow_offload_tuple * tuple,struct dst_entry * dst,struct in6_addr ** ip6_daddr)690 static int nf_flow_tunnel_v6_push(struct net *net, struct sk_buff *skb,
691 struct flow_offload_tuple *tuple,
692 struct dst_entry *dst,
693 struct in6_addr **ip6_daddr)
694 {
695 if (tuple->tun_num)
696 return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, dst, ip6_daddr);
697
698 return 0;
699 }
700
nf_flow_encap_push(struct sk_buff * skb,struct flow_offload_tuple * tuple,struct net_device * outdev)701 static int nf_flow_encap_push(struct sk_buff *skb,
702 struct flow_offload_tuple *tuple,
703 struct net_device *outdev)
704 {
705 u32 needed_headroom = LL_RESERVED_SPACE(outdev);
706 int i;
707
708 for (i = tuple->encap_num - 1; i >= 0; i--) {
709 switch (tuple->encap[i].proto) {
710 case htons(ETH_P_8021Q):
711 case htons(ETH_P_8021AD):
712 if (nf_flow_vlan_push(skb, tuple->encap[i].proto,
713 tuple->encap[i].id,
714 needed_headroom) < 0)
715 return -1;
716 break;
717 case htons(ETH_P_PPP_SES):
718 if (nf_flow_pppoe_push(skb, tuple->encap[i].id,
719 needed_headroom) < 0)
720 return -1;
721 break;
722 }
723 }
724
725 return 0;
726 }
727
728 struct nf_flow_xmit {
729 const void *dest;
730 const void *source;
731 struct net_device *outdev;
732 struct flow_offload_tuple *tuple;
733 bool needs_gso_segment;
734 };
735
__nf_flow_queue_xmit(struct net * net,struct sk_buff * skb,struct nf_flow_xmit * xmit)736 static void __nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
737 struct nf_flow_xmit *xmit)
738 {
739 struct net_device *dev = xmit->outdev;
740 unsigned int hh_len = LL_RESERVED_SPACE(dev);
741
742 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
743 skb = skb_expand_head(skb, hh_len);
744 if (!skb)
745 return;
746 }
747
748 skb->dev = dev;
749 dev_hard_header(skb, dev, ntohs(skb->protocol),
750 xmit->dest, xmit->source, skb->len);
751 dev_queue_xmit(skb);
752 }
753
nf_flow_encap_gso_xmit(struct net * net,struct sk_buff * skb,struct nf_flow_xmit * xmit)754 static unsigned int nf_flow_encap_gso_xmit(struct net *net, struct sk_buff *skb,
755 struct nf_flow_xmit *xmit)
756 {
757 struct sk_buff *segs, *nskb;
758
759 segs = skb_gso_segment(skb, 0);
760 if (IS_ERR(segs))
761 return NF_DROP;
762
763 if (segs)
764 consume_skb(skb);
765 else
766 segs = skb;
767
768 skb_list_walk_safe(segs, segs, nskb) {
769 skb_mark_not_on_list(segs);
770
771 if (nf_flow_encap_push(segs, xmit->tuple, xmit->outdev) < 0) {
772 kfree_skb(segs);
773 kfree_skb_list(nskb);
774 return NF_STOLEN;
775 }
776 __nf_flow_queue_xmit(net, segs, xmit);
777 }
778
779 return NF_STOLEN;
780 }
781
nf_flow_queue_xmit(struct net * net,struct sk_buff * skb,struct nf_flow_xmit * xmit)782 static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
783 struct nf_flow_xmit *xmit)
784 {
785 if (xmit->tuple->encap_num) {
786 if (skb_is_gso(skb) && xmit->needs_gso_segment)
787 return nf_flow_encap_gso_xmit(net, skb, xmit);
788
789 if (nf_flow_encap_push(skb, xmit->tuple, xmit->outdev) < 0)
790 return NF_DROP;
791 }
792
793 __nf_flow_queue_xmit(net, skb, xmit);
794
795 return NF_STOLEN;
796 }
797
nf_flow_queue_xmit4(struct sk_buff * skb,struct flow_offload_tuple_rhash * tuplehash,const struct nf_hook_state * state)798 static int nf_flow_queue_xmit4(struct sk_buff *skb,
799 struct flow_offload_tuple_rhash *tuplehash,
800 const struct nf_hook_state *state)
801 {
802 struct flow_offload_tuple *other_tuple;
803 enum flow_offload_tuple_dir dir;
804 struct nf_flow_xmit xmit = {};
805 struct flow_offload *flow;
806 struct neighbour *neigh;
807 struct rtable *rt;
808 __be32 ip_daddr;
809
810 if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) {
811 rt = dst_rtable(tuplehash->tuple.dst_cache);
812 memset(skb->cb, 0, sizeof(struct inet_skb_parm));
813 IPCB(skb)->iif = skb->dev->ifindex;
814 IPCB(skb)->flags = IPSKB_FORWARDED;
815 return nf_flow_xmit_xfrm(skb, state, &rt->dst);
816 }
817
818 dir = tuplehash->tuple.dir;
819 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
820 other_tuple = &flow->tuplehash[!dir].tuple;
821 ip_daddr = other_tuple->src_v4.s_addr;
822
823 if (nf_flow_tunnel_v4_push(state->net, skb, other_tuple,
824 tuplehash->tuple.dst_cache, &ip_daddr) < 0)
825 return NF_DROP;
826
827 switch (tuplehash->tuple.xmit_type) {
828 case FLOW_OFFLOAD_XMIT_NEIGH:
829 rt = dst_rtable(tuplehash->tuple.dst_cache);
830 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx);
831 if (!xmit.outdev) {
832 flow_offload_teardown(flow);
833 return NF_DROP;
834 }
835 neigh = ip_neigh_gw4(rt->dst.dev, rt_nexthop(rt, ip_daddr));
836 if (IS_ERR(neigh)) {
837 flow_offload_teardown(flow);
838 return NF_DROP;
839 }
840 xmit.dest = neigh->ha;
841 skb_dst_drop(skb);
842 skb_dst_set_noref(skb, &rt->dst);
843 break;
844 case FLOW_OFFLOAD_XMIT_DIRECT:
845 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx);
846 if (!xmit.outdev) {
847 flow_offload_teardown(flow);
848 return NF_DROP;
849 }
850 xmit.dest = tuplehash->tuple.out.h_dest;
851 xmit.source = tuplehash->tuple.out.h_source;
852 break;
853 default:
854 WARN_ON_ONCE(1);
855 return NF_DROP;
856 }
857 xmit.tuple = other_tuple;
858 xmit.needs_gso_segment = tuplehash->tuple.needs_gso_segment;
859
860 return nf_flow_queue_xmit(state->net, skb, &xmit);
861 }
862
863 unsigned int
nf_flow_offload_ip_hook(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)864 nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
865 const struct nf_hook_state *state)
866 {
867 struct flow_offload_tuple_rhash *tuplehash;
868 struct nf_flowtable *flow_table = priv;
869 struct nf_flowtable_ctx ctx = {
870 .in = state->in,
871 };
872 int ret;
873
874 if (!nf_flow_skb_encap_protocol(&ctx, skb))
875 return NF_ACCEPT;
876
877 if (unlikely(ctx.ether_type != htons(ETH_P_IP)))
878 return NF_ACCEPT;
879
880 tuplehash = nf_flow_offload_lookup(&ctx, flow_table, skb);
881 if (!tuplehash)
882 return NF_ACCEPT;
883
884 ret = nf_flow_offload_forward(&ctx, flow_table, tuplehash, skb);
885 if (ret < 0)
886 return NF_DROP;
887 else if (ret == 0)
888 return NF_ACCEPT;
889
890 return nf_flow_queue_xmit4(skb, tuplehash, state);
891 }
892 EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook);
893
nf_flow_nat_ipv6_tcp(struct sk_buff * skb,unsigned int thoff,struct in6_addr * addr,struct in6_addr * new_addr,struct ipv6hdr * ip6h)894 static void nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff,
895 struct in6_addr *addr,
896 struct in6_addr *new_addr,
897 struct ipv6hdr *ip6h)
898 {
899 struct tcphdr *tcph;
900
901 tcph = (void *)(skb_network_header(skb) + thoff);
902 inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32,
903 new_addr->s6_addr32, true);
904 }
905
nf_flow_nat_ipv6_udp(struct sk_buff * skb,unsigned int thoff,struct in6_addr * addr,struct in6_addr * new_addr)906 static void nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff,
907 struct in6_addr *addr,
908 struct in6_addr *new_addr)
909 {
910 struct udphdr *udph;
911
912 udph = (void *)(skb_network_header(skb) + thoff);
913 if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
914 inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32,
915 new_addr->s6_addr32, true);
916 if (!udph->check)
917 udph->check = CSUM_MANGLED_0;
918 }
919 }
920
nf_flow_nat_ipv6_l4proto(struct sk_buff * skb,struct ipv6hdr * ip6h,unsigned int thoff,struct in6_addr * addr,struct in6_addr * new_addr)921 static void nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h,
922 unsigned int thoff, struct in6_addr *addr,
923 struct in6_addr *new_addr)
924 {
925 switch (ip6h->nexthdr) {
926 case IPPROTO_TCP:
927 nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr, ip6h);
928 break;
929 case IPPROTO_UDP:
930 nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr);
931 break;
932 }
933 }
934
nf_flow_snat_ipv6(const struct flow_offload * flow,struct sk_buff * skb,struct ipv6hdr * ip6h,unsigned int thoff,enum flow_offload_tuple_dir dir)935 static void nf_flow_snat_ipv6(const struct flow_offload *flow,
936 struct sk_buff *skb, struct ipv6hdr *ip6h,
937 unsigned int thoff,
938 enum flow_offload_tuple_dir dir)
939 {
940 struct in6_addr addr, new_addr;
941
942 switch (dir) {
943 case FLOW_OFFLOAD_DIR_ORIGINAL:
944 addr = ip6h->saddr;
945 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6;
946 ip6h->saddr = new_addr;
947 break;
948 case FLOW_OFFLOAD_DIR_REPLY:
949 addr = ip6h->daddr;
950 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6;
951 ip6h->daddr = new_addr;
952 break;
953 }
954
955 nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
956 }
957
nf_flow_dnat_ipv6(const struct flow_offload * flow,struct sk_buff * skb,struct ipv6hdr * ip6h,unsigned int thoff,enum flow_offload_tuple_dir dir)958 static void nf_flow_dnat_ipv6(const struct flow_offload *flow,
959 struct sk_buff *skb, struct ipv6hdr *ip6h,
960 unsigned int thoff,
961 enum flow_offload_tuple_dir dir)
962 {
963 struct in6_addr addr, new_addr;
964
965 switch (dir) {
966 case FLOW_OFFLOAD_DIR_ORIGINAL:
967 addr = ip6h->daddr;
968 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6;
969 ip6h->daddr = new_addr;
970 break;
971 case FLOW_OFFLOAD_DIR_REPLY:
972 addr = ip6h->saddr;
973 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6;
974 ip6h->saddr = new_addr;
975 break;
976 }
977
978 nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
979 }
980
nf_flow_nat_ipv6(const struct flow_offload * flow,struct sk_buff * skb,enum flow_offload_tuple_dir dir,struct ipv6hdr * ip6h)981 static void nf_flow_nat_ipv6(const struct flow_offload *flow,
982 struct sk_buff *skb,
983 enum flow_offload_tuple_dir dir,
984 struct ipv6hdr *ip6h)
985 {
986 unsigned int thoff = sizeof(*ip6h);
987
988 if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
989 nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir);
990 nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir);
991 }
992 if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
993 nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir);
994 nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir);
995 }
996 }
997
nf_flow_tuple_ipv6(struct nf_flowtable_ctx * ctx,struct sk_buff * skb,struct flow_offload_tuple * tuple)998 static int nf_flow_tuple_ipv6(struct nf_flowtable_ctx *ctx, struct sk_buff *skb,
999 struct flow_offload_tuple *tuple)
1000 {
1001 struct flow_ports *ports;
1002 struct ipv6hdr *ip6h;
1003 unsigned int thoff;
1004 u8 nexthdr;
1005
1006 thoff = sizeof(*ip6h) + ctx->offset;
1007 if (!pskb_may_pull(skb, thoff))
1008 return -1;
1009
1010 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset);
1011
1012 nexthdr = ip6h->nexthdr;
1013 switch (nexthdr) {
1014 case IPPROTO_TCP:
1015 ctx->hdrsize = sizeof(struct tcphdr);
1016 break;
1017 case IPPROTO_UDP:
1018 ctx->hdrsize = sizeof(struct udphdr);
1019 break;
1020 #ifdef CONFIG_NF_CT_PROTO_GRE
1021 case IPPROTO_GRE:
1022 ctx->hdrsize = sizeof(struct gre_base_hdr);
1023 break;
1024 #endif
1025 default:
1026 return -1;
1027 }
1028
1029 if (ip6h->hop_limit <= 1)
1030 return -1;
1031
1032 if (!pskb_may_pull(skb, thoff + ctx->hdrsize))
1033 return -1;
1034
1035 switch (nexthdr) {
1036 case IPPROTO_TCP:
1037 case IPPROTO_UDP:
1038 ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
1039 tuple->src_port = ports->source;
1040 tuple->dst_port = ports->dest;
1041 break;
1042 case IPPROTO_GRE: {
1043 struct gre_base_hdr *greh;
1044
1045 greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff);
1046 if ((greh->flags & GRE_VERSION) != GRE_VERSION_0)
1047 return -1;
1048 break;
1049 }
1050 }
1051
1052 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset);
1053
1054 tuple->src_v6 = ip6h->saddr;
1055 tuple->dst_v6 = ip6h->daddr;
1056 tuple->l3proto = AF_INET6;
1057 tuple->l4proto = nexthdr;
1058 tuple->iifidx = ctx->in->ifindex;
1059 nf_flow_tuple_encap(ctx, skb, tuple);
1060
1061 return 0;
1062 }
1063
nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct flow_offload_tuple_rhash * tuplehash,struct sk_buff * skb)1064 static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx,
1065 struct nf_flowtable *flow_table,
1066 struct flow_offload_tuple_rhash *tuplehash,
1067 struct sk_buff *skb)
1068 {
1069 enum flow_offload_tuple_dir dir;
1070 struct flow_offload *flow;
1071 unsigned int thoff, mtu;
1072 struct ipv6hdr *ip6h;
1073
1074 dir = tuplehash->tuple.dir;
1075 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
1076
1077 mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset;
1078 if (flow->tuplehash[!dir].tuple.tun_num)
1079 mtu -= sizeof(*ip6h);
1080
1081 if (unlikely(nf_flow_exceeds_mtu(skb, mtu)))
1082 return 0;
1083
1084 ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset);
1085 thoff = sizeof(*ip6h) + ctx->offset;
1086 if (nf_flow_state_check(flow, ip6h->nexthdr, skb, thoff))
1087 return 0;
1088
1089 if (!nf_flow_dst_check(&tuplehash->tuple)) {
1090 flow_offload_teardown(flow);
1091 return 0;
1092 }
1093
1094 if (skb_ensure_writable(skb, thoff + ctx->hdrsize))
1095 return -1;
1096
1097 flow_offload_refresh(flow_table, flow, false);
1098
1099 nf_flow_encap_pop(ctx, skb, tuplehash);
1100
1101 ip6h = ipv6_hdr(skb);
1102 nf_flow_nat_ipv6(flow, skb, dir, ip6h);
1103
1104 ip6h->hop_limit--;
1105 skb_clear_tstamp(skb);
1106
1107 if (flow_table->flags & NF_FLOWTABLE_COUNTER)
1108 nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
1109
1110 return 1;
1111 }
1112
1113 static struct flow_offload_tuple_rhash *
nf_flow_offload_ipv6_lookup(struct nf_flowtable_ctx * ctx,struct nf_flowtable * flow_table,struct sk_buff * skb)1114 nf_flow_offload_ipv6_lookup(struct nf_flowtable_ctx *ctx,
1115 struct nf_flowtable *flow_table,
1116 struct sk_buff *skb)
1117 {
1118 struct flow_offload_tuple tuple = {};
1119
1120 if (nf_flow_tuple_ipv6(ctx, skb, &tuple) < 0)
1121 return NULL;
1122
1123 return flow_offload_lookup(flow_table, &tuple);
1124 }
1125
nf_flow_queue_xmit6(struct sk_buff * skb,struct flow_offload_tuple_rhash * tuplehash,const struct nf_hook_state * state)1126 static int nf_flow_queue_xmit6(struct sk_buff *skb,
1127 struct flow_offload_tuple_rhash *tuplehash,
1128 const struct nf_hook_state *state)
1129 {
1130 struct flow_offload_tuple *other_tuple;
1131 enum flow_offload_tuple_dir dir;
1132 struct nf_flow_xmit xmit = {};
1133 struct in6_addr *ip6_daddr;
1134 struct flow_offload *flow;
1135 struct neighbour *neigh;
1136 struct rt6_info *rt;
1137
1138 if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) {
1139 rt = dst_rt6_info(tuplehash->tuple.dst_cache);
1140 memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
1141 IP6CB(skb)->iif = skb->dev->ifindex;
1142 IP6CB(skb)->flags = IP6SKB_FORWARDED;
1143 return nf_flow_xmit_xfrm(skb, state, &rt->dst);
1144 }
1145
1146 dir = tuplehash->tuple.dir;
1147 flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
1148 other_tuple = &flow->tuplehash[!dir].tuple;
1149 ip6_daddr = &other_tuple->src_v6;
1150
1151 if (nf_flow_tunnel_v6_push(state->net, skb, other_tuple,
1152 tuplehash->tuple.dst_cache,
1153 &ip6_daddr) < 0)
1154 return NF_DROP;
1155
1156 switch (tuplehash->tuple.xmit_type) {
1157 case FLOW_OFFLOAD_XMIT_NEIGH:
1158 rt = dst_rt6_info(tuplehash->tuple.dst_cache);
1159 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx);
1160 if (!xmit.outdev) {
1161 flow_offload_teardown(flow);
1162 return NF_DROP;
1163 }
1164 neigh = ip_neigh_gw6(rt->dst.dev, rt6_nexthop(rt, ip6_daddr));
1165 if (IS_ERR(neigh)) {
1166 flow_offload_teardown(flow);
1167 return NF_DROP;
1168 }
1169 xmit.dest = neigh->ha;
1170 skb_dst_drop(skb);
1171 skb_dst_set_noref(skb, &rt->dst);
1172 break;
1173 case FLOW_OFFLOAD_XMIT_DIRECT:
1174 xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx);
1175 if (!xmit.outdev) {
1176 flow_offload_teardown(flow);
1177 return NF_DROP;
1178 }
1179 xmit.dest = tuplehash->tuple.out.h_dest;
1180 xmit.source = tuplehash->tuple.out.h_source;
1181 break;
1182 default:
1183 WARN_ON_ONCE(1);
1184 return NF_DROP;
1185 }
1186 xmit.tuple = other_tuple;
1187 xmit.needs_gso_segment = tuplehash->tuple.needs_gso_segment;
1188
1189 return nf_flow_queue_xmit(state->net, skb, &xmit);
1190 }
1191
1192 unsigned int
nf_flow_offload_ipv6_hook(void * priv,struct sk_buff * skb,const struct nf_hook_state * state)1193 nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
1194 const struct nf_hook_state *state)
1195 {
1196 struct flow_offload_tuple_rhash *tuplehash;
1197 struct nf_flowtable *flow_table = priv;
1198 struct nf_flowtable_ctx ctx = {
1199 .in = state->in,
1200 };
1201 int ret;
1202
1203 if (!nf_flow_skb_encap_protocol(&ctx, skb))
1204 return NF_ACCEPT;
1205
1206 if (unlikely(ctx.ether_type != htons(ETH_P_IPV6)))
1207 return NF_ACCEPT;
1208
1209 tuplehash = nf_flow_offload_ipv6_lookup(&ctx, flow_table, skb);
1210 if (!tuplehash)
1211 return NF_ACCEPT;
1212
1213 ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, skb);
1214 if (ret < 0)
1215 return NF_DROP;
1216 else if (ret == 0)
1217 return NF_ACCEPT;
1218
1219 return nf_flow_queue_xmit6(skb, tuplehash, state);
1220 }
1221 EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook);
1222