1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * xfrm_output.c - Common IPsec encapsulation code.
4 *
5 * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
6 */
7
8 #include <linux/errno.h>
9 #include <linux/module.h>
10 #include <linux/netdevice.h>
11 #include <linux/netfilter.h>
12 #include <linux/skbuff.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <net/dst.h>
16 #include <net/gso.h>
17 #include <net/icmp.h>
18 #include <net/inet_ecn.h>
19 #include <net/xfrm.h>
20
21 #if IS_ENABLED(CONFIG_IPV6)
22 #include <net/ip6_route.h>
23 #endif
24
25 #include "xfrm_inout.h"
26
27 static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb);
28 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb);
29
xfrm_skb_check_space(struct sk_buff * skb)30 static int xfrm_skb_check_space(struct sk_buff *skb)
31 {
32 struct dst_entry *dst = skb_dst(skb);
33 int nhead = dst->header_len + LL_RESERVED_SPACE(dst->dev)
34 - skb_headroom(skb);
35 int ntail = dst->dev->needed_tailroom - skb_tailroom(skb);
36
37 if (nhead <= 0) {
38 if (ntail <= 0)
39 return 0;
40 nhead = 0;
41 } else if (ntail < 0)
42 ntail = 0;
43
44 return pskb_expand_head(skb, nhead, ntail, GFP_ATOMIC);
45 }
46
47 /* Children define the path of the packet through the
48 * Linux networking. Thus, destinations are stackable.
49 */
50
skb_dst_pop(struct sk_buff * skb)51 static struct dst_entry *skb_dst_pop(struct sk_buff *skb)
52 {
53 struct dst_entry *child = dst_clone(xfrm_dst_child(skb_dst(skb)));
54
55 skb_dst_drop(skb);
56 return child;
57 }
58
59 /* Add encapsulation header.
60 *
61 * The IP header will be moved forward to make space for the encapsulation
62 * header.
63 */
xfrm4_transport_output(struct xfrm_state * x,struct sk_buff * skb)64 static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
65 {
66 struct iphdr *iph = ip_hdr(skb);
67 int ihl = iph->ihl * 4;
68
69 if (!skb->inner_protocol)
70 skb_set_inner_transport_header(skb,
71 skb_transport_offset(skb));
72
73 skb_set_network_header(skb, -x->props.header_len);
74 skb->mac_header = skb->network_header +
75 offsetof(struct iphdr, protocol);
76 skb->transport_header = skb->network_header + ihl;
77 __skb_pull(skb, ihl);
78 memmove(skb_network_header(skb), iph, ihl);
79 return 0;
80 }
81
82 #if IS_ENABLED(CONFIG_IPV6_MIP6)
mip6_rthdr_offset(struct sk_buff * skb,u8 ** nexthdr,int type)83 static int mip6_rthdr_offset(struct sk_buff *skb, u8 **nexthdr, int type)
84 {
85 const unsigned char *nh = skb_network_header(skb);
86 unsigned int offset = sizeof(struct ipv6hdr);
87 unsigned int packet_len;
88 int found_rhdr = 0;
89
90 packet_len = skb_tail_pointer(skb) - nh;
91 *nexthdr = &ipv6_hdr(skb)->nexthdr;
92
93 while (offset <= packet_len) {
94 struct ipv6_opt_hdr *exthdr;
95
96 switch (**nexthdr) {
97 case NEXTHDR_HOP:
98 break;
99 case NEXTHDR_ROUTING:
100 if (type == IPPROTO_ROUTING && offset + 3 <= packet_len) {
101 struct ipv6_rt_hdr *rt;
102
103 rt = (struct ipv6_rt_hdr *)(nh + offset);
104 if (rt->type != 0)
105 return offset;
106 }
107 found_rhdr = 1;
108 break;
109 case NEXTHDR_DEST:
110 /* HAO MUST NOT appear more than once.
111 * XXX: It is better to try to find by the end of
112 * XXX: packet if HAO exists.
113 */
114 if (ipv6_find_tlv(skb, offset, IPV6_TLV_HAO) >= 0) {
115 net_dbg_ratelimited("mip6: hao exists already, override\n");
116 return offset;
117 }
118
119 if (found_rhdr)
120 return offset;
121
122 break;
123 default:
124 return offset;
125 }
126
127 if (offset + sizeof(struct ipv6_opt_hdr) > packet_len)
128 return -EINVAL;
129
130 exthdr = (struct ipv6_opt_hdr *)(skb_network_header(skb) +
131 offset);
132 offset += ipv6_optlen(exthdr);
133 if (offset > IPV6_MAXPLEN)
134 return -EINVAL;
135 *nexthdr = &exthdr->nexthdr;
136 }
137
138 return -EINVAL;
139 }
140 #endif
141
142 #if IS_ENABLED(CONFIG_IPV6)
xfrm6_hdr_offset(struct xfrm_state * x,struct sk_buff * skb,u8 ** prevhdr)143 static int xfrm6_hdr_offset(struct xfrm_state *x, struct sk_buff *skb, u8 **prevhdr)
144 {
145 switch (x->type->proto) {
146 #if IS_ENABLED(CONFIG_IPV6_MIP6)
147 case IPPROTO_DSTOPTS:
148 case IPPROTO_ROUTING:
149 return mip6_rthdr_offset(skb, prevhdr, x->type->proto);
150 #endif
151 default:
152 break;
153 }
154
155 return ip6_find_1stfragopt(skb, prevhdr);
156 }
157 #endif
158
159 /* Add encapsulation header.
160 *
161 * The IP header and mutable extension headers will be moved forward to make
162 * space for the encapsulation header.
163 */
xfrm6_transport_output(struct xfrm_state * x,struct sk_buff * skb)164 static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)
165 {
166 #if IS_ENABLED(CONFIG_IPV6)
167 struct ipv6hdr *iph;
168 u8 *prevhdr;
169 int hdr_len;
170
171 iph = ipv6_hdr(skb);
172 if (!skb->inner_protocol)
173 skb_set_inner_transport_header(skb,
174 skb_transport_offset(skb));
175
176 hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);
177 if (hdr_len < 0)
178 return hdr_len;
179 skb_set_mac_header(skb,
180 (prevhdr - x->props.header_len) - skb->data);
181 skb_set_network_header(skb, -x->props.header_len);
182 skb->transport_header = skb->network_header + hdr_len;
183 __skb_pull(skb, hdr_len);
184 memmove(ipv6_hdr(skb), iph, hdr_len);
185 return 0;
186 #else
187 WARN_ON_ONCE(1);
188 return -EAFNOSUPPORT;
189 #endif
190 }
191
192 /* Add route optimization header space.
193 *
194 * The IP header and mutable extension headers will be moved forward to make
195 * space for the route optimization header.
196 */
xfrm6_ro_output(struct xfrm_state * x,struct sk_buff * skb)197 static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb)
198 {
199 #if IS_ENABLED(CONFIG_IPV6)
200 struct ipv6hdr *iph;
201 u8 *prevhdr;
202 int hdr_len;
203
204 iph = ipv6_hdr(skb);
205
206 hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);
207 if (hdr_len < 0)
208 return hdr_len;
209 skb_set_mac_header(skb,
210 (prevhdr - x->props.header_len) - skb->data);
211 skb_set_network_header(skb, -x->props.header_len);
212 skb->transport_header = skb->network_header + hdr_len;
213 __skb_pull(skb, hdr_len);
214 memmove(ipv6_hdr(skb), iph, hdr_len);
215
216 return 0;
217 #else
218 WARN_ON_ONCE(1);
219 return -EAFNOSUPPORT;
220 #endif
221 }
222
223 /* Add encapsulation header.
224 *
225 * The top IP header will be constructed per draft-nikander-esp-beet-mode-06.txt.
226 */
xfrm4_beet_encap_add(struct xfrm_state * x,struct sk_buff * skb)227 static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
228 {
229 struct ip_beet_phdr *ph;
230 struct iphdr *top_iph;
231 int hdrlen, optlen;
232
233 hdrlen = 0;
234 optlen = XFRM_MODE_SKB_CB(skb)->optlen;
235 if (unlikely(optlen))
236 hdrlen += IPV4_BEET_PHMAXLEN - (optlen & 4);
237
238 skb_set_network_header(skb, -x->props.header_len - hdrlen +
239 (XFRM_MODE_SKB_CB(skb)->ihl - sizeof(*top_iph)));
240 if (x->sel.family != AF_INET6)
241 skb->network_header += IPV4_BEET_PHMAXLEN;
242 skb->mac_header = skb->network_header +
243 offsetof(struct iphdr, protocol);
244 skb->transport_header = skb->network_header + sizeof(*top_iph);
245
246 xfrm4_beet_make_header(skb);
247
248 ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdrlen);
249
250 top_iph = ip_hdr(skb);
251
252 if (unlikely(optlen)) {
253 if (WARN_ON(optlen < 0))
254 return -EINVAL;
255
256 ph->padlen = 4 - (optlen & 4);
257 ph->hdrlen = optlen / 8;
258 ph->nexthdr = top_iph->protocol;
259 if (ph->padlen)
260 memset(ph + 1, IPOPT_NOP, ph->padlen);
261
262 top_iph->protocol = IPPROTO_BEETPH;
263 top_iph->ihl = sizeof(struct iphdr) / 4;
264 }
265
266 top_iph->saddr = x->props.saddr.a4;
267 top_iph->daddr = x->id.daddr.a4;
268
269 return 0;
270 }
271
272 /* Add encapsulation header.
273 *
274 * The top IP header will be constructed per RFC 2401.
275 */
xfrm4_tunnel_encap_add(struct xfrm_state * x,struct sk_buff * skb)276 static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
277 {
278 bool small_ipv6 = (skb->protocol == htons(ETH_P_IPV6)) && (skb->len <= IPV6_MIN_MTU);
279 struct dst_entry *dst = skb_dst(skb);
280 struct iphdr *top_iph;
281 int flags;
282
283 if (!skb->inner_protocol) {
284 skb_set_inner_network_header(skb, skb_network_offset(skb));
285 skb_set_inner_transport_header(skb, skb_transport_offset(skb));
286 }
287
288 skb_set_network_header(skb, -x->props.header_len);
289 skb->mac_header = skb->network_header +
290 offsetof(struct iphdr, protocol);
291 skb->transport_header = skb->network_header + sizeof(*top_iph);
292 top_iph = ip_hdr(skb);
293
294 top_iph->ihl = 5;
295 top_iph->version = 4;
296
297 top_iph->protocol = xfrm_af2proto(skb_dst(skb)->ops->family);
298
299 /* DS disclosing depends on XFRM_SA_XFLAG_DONT_ENCAP_DSCP */
300 if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
301 top_iph->tos = 0;
302 else
303 top_iph->tos = XFRM_MODE_SKB_CB(skb)->tos;
304 top_iph->tos = INET_ECN_encapsulate(top_iph->tos,
305 XFRM_MODE_SKB_CB(skb)->tos);
306
307 flags = x->props.flags;
308 if (flags & XFRM_STATE_NOECN)
309 IP_ECN_clear(top_iph);
310
311 top_iph->frag_off = (flags & XFRM_STATE_NOPMTUDISC) || small_ipv6 ?
312 0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF));
313
314 top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst));
315
316 top_iph->saddr = x->props.saddr.a4;
317 top_iph->daddr = x->id.daddr.a4;
318 ip_select_ident(dev_net(dst->dev), skb, NULL);
319
320 return 0;
321 }
322
323 #if IS_ENABLED(CONFIG_IPV6)
xfrm6_tunnel_encap_add(struct xfrm_state * x,struct sk_buff * skb)324 static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
325 {
326 struct dst_entry *dst = skb_dst(skb);
327 struct ipv6hdr *top_iph;
328 int dsfield;
329
330 if (!skb->inner_protocol) {
331 skb_set_inner_network_header(skb, skb_network_offset(skb));
332 skb_set_inner_transport_header(skb, skb_transport_offset(skb));
333 }
334
335 skb_set_network_header(skb, -x->props.header_len);
336 skb->mac_header = skb->network_header +
337 offsetof(struct ipv6hdr, nexthdr);
338 skb->transport_header = skb->network_header + sizeof(*top_iph);
339 top_iph = ipv6_hdr(skb);
340
341 top_iph->version = 6;
342
343 memcpy(top_iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl,
344 sizeof(top_iph->flow_lbl));
345 top_iph->nexthdr = xfrm_af2proto(skb_dst(skb)->ops->family);
346
347 if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
348 dsfield = 0;
349 else
350 dsfield = XFRM_MODE_SKB_CB(skb)->tos;
351 dsfield = INET_ECN_encapsulate(dsfield, XFRM_MODE_SKB_CB(skb)->tos);
352 if (x->props.flags & XFRM_STATE_NOECN)
353 dsfield &= ~INET_ECN_MASK;
354 ipv6_change_dsfield(top_iph, 0, dsfield);
355 top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst));
356 top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
357 top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
358 return 0;
359 }
360
xfrm6_beet_encap_add(struct xfrm_state * x,struct sk_buff * skb)361 static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
362 {
363 struct ipv6hdr *top_iph;
364 struct ip_beet_phdr *ph;
365 int optlen, hdr_len;
366
367 hdr_len = 0;
368 optlen = XFRM_MODE_SKB_CB(skb)->optlen;
369 if (unlikely(optlen))
370 hdr_len += IPV4_BEET_PHMAXLEN - (optlen & 4);
371
372 skb_set_network_header(skb, -x->props.header_len - hdr_len);
373 if (x->sel.family != AF_INET6)
374 skb->network_header += IPV4_BEET_PHMAXLEN;
375 skb->mac_header = skb->network_header +
376 offsetof(struct ipv6hdr, nexthdr);
377 skb->transport_header = skb->network_header + sizeof(*top_iph);
378 ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdr_len);
379
380 xfrm6_beet_make_header(skb);
381
382 top_iph = ipv6_hdr(skb);
383 if (unlikely(optlen)) {
384 if (WARN_ON(optlen < 0))
385 return -EINVAL;
386
387 ph->padlen = 4 - (optlen & 4);
388 ph->hdrlen = optlen / 8;
389 ph->nexthdr = top_iph->nexthdr;
390 if (ph->padlen)
391 memset(ph + 1, IPOPT_NOP, ph->padlen);
392
393 top_iph->nexthdr = IPPROTO_BEETPH;
394 }
395
396 top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
397 top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
398 return 0;
399 }
400 #endif
401
402 /* Add encapsulation header.
403 *
404 * On exit, the transport header will be set to the start of the
405 * encapsulation header to be filled in by x->type->output and the mac
406 * header will be set to the nextheader (protocol for IPv4) field of the
407 * extension header directly preceding the encapsulation header, or in
408 * its absence, that of the top IP header.
409 * The value of the network header will always point to the top IP header
410 * while skb->data will point to the payload.
411 */
xfrm4_prepare_output(struct xfrm_state * x,struct sk_buff * skb)412 static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
413 {
414 int err;
415
416 err = xfrm_inner_extract_output(x, skb);
417 if (err)
418 return err;
419
420 IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;
421 skb->protocol = htons(ETH_P_IP);
422
423 switch (x->props.mode) {
424 case XFRM_MODE_BEET:
425 return xfrm4_beet_encap_add(x, skb);
426 case XFRM_MODE_TUNNEL:
427 return xfrm4_tunnel_encap_add(x, skb);
428 }
429
430 WARN_ON_ONCE(1);
431 return -EOPNOTSUPP;
432 }
433
xfrm6_prepare_output(struct xfrm_state * x,struct sk_buff * skb)434 static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
435 {
436 #if IS_ENABLED(CONFIG_IPV6)
437 int err;
438
439 err = xfrm_inner_extract_output(x, skb);
440 if (err)
441 return err;
442
443 skb->ignore_df = 1;
444 skb->protocol = htons(ETH_P_IPV6);
445
446 switch (x->props.mode) {
447 case XFRM_MODE_BEET:
448 return xfrm6_beet_encap_add(x, skb);
449 case XFRM_MODE_TUNNEL:
450 return xfrm6_tunnel_encap_add(x, skb);
451 default:
452 WARN_ON_ONCE(1);
453 return -EOPNOTSUPP;
454 }
455 #endif
456 WARN_ON_ONCE(1);
457 return -EAFNOSUPPORT;
458 }
459
xfrm_outer_mode_output(struct xfrm_state * x,struct sk_buff * skb)460 static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
461 {
462 switch (x->props.mode) {
463 case XFRM_MODE_BEET:
464 case XFRM_MODE_TUNNEL:
465 if (x->props.family == AF_INET)
466 return xfrm4_prepare_output(x, skb);
467 if (x->props.family == AF_INET6)
468 return xfrm6_prepare_output(x, skb);
469 break;
470 case XFRM_MODE_TRANSPORT:
471 if (x->props.family == AF_INET)
472 return xfrm4_transport_output(x, skb);
473 if (x->props.family == AF_INET6)
474 return xfrm6_transport_output(x, skb);
475 break;
476 case XFRM_MODE_ROUTEOPTIMIZATION:
477 if (x->props.family == AF_INET6)
478 return xfrm6_ro_output(x, skb);
479 WARN_ON_ONCE(1);
480 break;
481 default:
482 if (x->mode_cbs && x->mode_cbs->prepare_output)
483 return x->mode_cbs->prepare_output(x, skb);
484 WARN_ON_ONCE(1);
485 break;
486 }
487
488 return -EOPNOTSUPP;
489 }
490
491 #if IS_ENABLED(CONFIG_NET_PKTGEN)
pktgen_xfrm_outer_mode_output(struct xfrm_state * x,struct sk_buff * skb)492 int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
493 {
494 return xfrm_outer_mode_output(x, skb);
495 }
496 EXPORT_SYMBOL_GPL(pktgen_xfrm_outer_mode_output);
497 #endif
498
xfrm_output_one(struct sk_buff * skb,int err)499 static int xfrm_output_one(struct sk_buff *skb, int err)
500 {
501 struct dst_entry *dst = skb_dst(skb);
502 struct xfrm_state *x = dst->xfrm;
503 struct net *net = xs_net(x);
504
505 if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
506 goto resume;
507
508 do {
509 err = xfrm_skb_check_space(skb);
510 if (err) {
511 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
512 goto error_nolock;
513 }
514
515 skb->mark = xfrm_smark_get(skb->mark, x);
516
517 err = xfrm_outer_mode_output(x, skb);
518 if (err) {
519 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
520 goto error_nolock;
521 }
522
523 spin_lock_bh(&x->lock);
524
525 if (unlikely(x->km.state != XFRM_STATE_VALID)) {
526 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEINVALID);
527 err = -EINVAL;
528 goto error;
529 }
530
531 err = xfrm_state_check_expire(x);
532 if (err) {
533 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEEXPIRED);
534 goto error;
535 }
536
537 err = xfrm_replay_overflow(x, skb);
538 if (err) {
539 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATESEQERROR);
540 goto error;
541 }
542
543 x->curlft.bytes += skb->len;
544 x->curlft.packets++;
545 x->lastused = ktime_get_real_seconds();
546
547 spin_unlock_bh(&x->lock);
548
549 skb_dst_force(skb);
550 if (!skb_dst(skb)) {
551 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
552 err = -EHOSTUNREACH;
553 goto error_nolock;
554 }
555
556 if (xfrm_offload(skb)) {
557 x->type_offload->encap(x, skb);
558 } else {
559 /* Inner headers are invalid now. */
560 skb->encapsulation = 0;
561
562 err = x->type->output(x, skb);
563 if (err == -EINPROGRESS)
564 goto out;
565 }
566
567 resume:
568 if (err) {
569 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
570 goto error_nolock;
571 }
572
573 dst = skb_dst_pop(skb);
574 if (!dst) {
575 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
576 err = -EHOSTUNREACH;
577 goto error_nolock;
578 }
579 skb_dst_set(skb, dst);
580 x = dst->xfrm;
581 } while (x && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL));
582
583 return 0;
584
585 error:
586 spin_unlock_bh(&x->lock);
587 error_nolock:
588 kfree_skb(skb);
589 out:
590 return err;
591 }
592
xfrm_output_resume(struct sock * sk,struct sk_buff * skb,int err)593 int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err)
594 {
595 struct net *net = xs_net(skb_dst(skb)->xfrm);
596
597 while (likely((err = xfrm_output_one(skb, err)) == 0)) {
598 nf_reset_ct(skb);
599
600 err = skb_dst(skb)->ops->local_out(net, sk, skb);
601 if (unlikely(err != 1))
602 goto out;
603
604 if (!skb_dst(skb)->xfrm)
605 return dst_output(net, sk, skb);
606
607 err = nf_hook(skb_dst(skb)->ops->family,
608 NF_INET_POST_ROUTING, net, sk, skb,
609 NULL, skb_dst(skb)->dev, xfrm_output2);
610 if (unlikely(err != 1))
611 goto out;
612 }
613
614 if (err == -EINPROGRESS)
615 err = 0;
616
617 out:
618 return err;
619 }
620 EXPORT_SYMBOL_GPL(xfrm_output_resume);
621
xfrm_dev_direct_output(struct sock * sk,struct xfrm_state * x,struct sk_buff * skb)622 static int xfrm_dev_direct_output(struct sock *sk, struct xfrm_state *x,
623 struct sk_buff *skb)
624 {
625 struct dst_entry *dst = skb_dst(skb);
626 struct net *net = xs_net(x);
627 int err;
628
629 dst = skb_dst_pop(skb);
630 if (!dst) {
631 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
632 kfree_skb(skb);
633 return -EHOSTUNREACH;
634 }
635 skb_dst_set(skb, dst);
636 nf_reset_ct(skb);
637
638 err = skb_dst(skb)->ops->local_out(net, sk, skb);
639 if (unlikely(err != 1))
640 return err;
641
642 /* In transport mode, network destination is
643 * directly reachable, while in tunnel mode,
644 * inner packet network may not be. In packet
645 * offload type, HW is responsible for hard
646 * header packet mangling so directly xmit skb
647 * to netdevice.
648 */
649 skb->dev = x->xso.dev;
650 __skb_push(skb, skb->dev->hard_header_len);
651 return dev_queue_xmit(skb);
652 }
653
xfrm_output2(struct net * net,struct sock * sk,struct sk_buff * skb)654 static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
655 {
656 return xfrm_output_resume(sk, skb, 1);
657 }
658
xfrm_output_gso(struct net * net,struct sock * sk,struct sk_buff * skb)659 static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb)
660 {
661 struct sk_buff *segs, *nskb;
662
663 BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET);
664 BUILD_BUG_ON(sizeof(*IP6CB(skb)) > SKB_GSO_CB_OFFSET);
665 segs = skb_gso_segment(skb, 0);
666 kfree_skb(skb);
667 if (IS_ERR(segs))
668 return PTR_ERR(segs);
669 if (segs == NULL)
670 return -EINVAL;
671
672 skb_list_walk_safe(segs, segs, nskb) {
673 int err;
674
675 skb_mark_not_on_list(segs);
676 err = xfrm_output2(net, sk, segs);
677
678 if (unlikely(err)) {
679 kfree_skb_list(nskb);
680 return err;
681 }
682 }
683
684 return 0;
685 }
686
687 /* For partial checksum offload, the outer header checksum is calculated
688 * by software and the inner header checksum is calculated by hardware.
689 * This requires hardware to know the inner packet type to calculate
690 * the inner header checksum. Save inner ip protocol here to avoid
691 * traversing the packet in the vendor's xmit code.
692 * For IPsec tunnel mode save the ip protocol from the IP header of the
693 * plain text packet. Otherwise If the encap type is IPIP, just save
694 * skb->inner_ipproto in any other case get the ip protocol from the IP
695 * header.
696 */
xfrm_get_inner_ipproto(struct sk_buff * skb,struct xfrm_state * x)697 static void xfrm_get_inner_ipproto(struct sk_buff *skb, struct xfrm_state *x)
698 {
699 struct xfrm_offload *xo = xfrm_offload(skb);
700 const struct ethhdr *eth;
701
702 if (!xo)
703 return;
704
705 if (x->outer_mode.encap == XFRM_MODE_TUNNEL) {
706 switch (skb_dst(skb)->ops->family) {
707 case AF_INET:
708 xo->inner_ipproto = ip_hdr(skb)->protocol;
709 break;
710 case AF_INET6:
711 xo->inner_ipproto = ipv6_hdr(skb)->nexthdr;
712 break;
713 default:
714 break;
715 }
716
717 return;
718 }
719 if (x->outer_mode.encap == XFRM_MODE_IPTFS) {
720 xo->inner_ipproto = IPPROTO_AGGFRAG;
721 return;
722 }
723
724 /* non-Tunnel Mode */
725 if (!skb->encapsulation)
726 return;
727
728 if (skb->inner_protocol_type == ENCAP_TYPE_IPPROTO) {
729 xo->inner_ipproto = skb->inner_ipproto;
730 return;
731 }
732
733 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER)
734 return;
735
736 eth = (struct ethhdr *)skb_inner_mac_header(skb);
737
738 switch (ntohs(eth->h_proto)) {
739 case ETH_P_IPV6:
740 xo->inner_ipproto = inner_ipv6_hdr(skb)->nexthdr;
741 break;
742 case ETH_P_IP:
743 xo->inner_ipproto = inner_ip_hdr(skb)->protocol;
744 break;
745 }
746 }
747
xfrm_output(struct sock * sk,struct sk_buff * skb)748 int xfrm_output(struct sock *sk, struct sk_buff *skb)
749 {
750 struct net *net = dev_net(skb_dst(skb)->dev);
751 struct xfrm_state *x = skb_dst(skb)->xfrm;
752 int family;
753 int err;
754
755 family = (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) ? x->outer_mode.family
756 : skb_dst(skb)->ops->family;
757
758 switch (family) {
759 case AF_INET:
760 memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
761 IPCB(skb)->flags |= IPSKB_XFRM_TRANSFORMED;
762 break;
763 case AF_INET6:
764 memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
765
766 IP6CB(skb)->flags |= IP6SKB_XFRM_TRANSFORMED;
767 break;
768 }
769
770 if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) {
771 if (!xfrm_dev_offload_ok(skb, x)) {
772 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
773 kfree_skb(skb);
774 return -EHOSTUNREACH;
775 }
776
777 /* Exclusive direct xmit for tunnel mode, as
778 * some filtering or matching rules may apply
779 * in transport mode.
780 * Locally generated packets also require
781 * the normal XFRM path for L2 header setup,
782 * as the hardware needs the L2 header to match
783 * for encryption, so skip direct output as well.
784 */
785 if (x->props.mode == XFRM_MODE_TUNNEL && !skb->sk)
786 return xfrm_dev_direct_output(sk, x, skb);
787
788 return xfrm_output_resume(sk, skb, 0);
789 }
790
791 secpath_reset(skb);
792
793 if (xfrm_dev_offload_ok(skb, x)) {
794 struct sec_path *sp;
795
796 sp = secpath_set(skb);
797 if (!sp) {
798 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
799 kfree_skb(skb);
800 return -ENOMEM;
801 }
802
803 sp->olen++;
804 sp->xvec[sp->len++] = x;
805 xfrm_state_hold(x);
806
807 xfrm_get_inner_ipproto(skb, x);
808 skb->encapsulation = 1;
809
810 if (skb_is_gso(skb)) {
811 if (skb->inner_protocol && x->props.mode == XFRM_MODE_TUNNEL)
812 return xfrm_output_gso(net, sk, skb);
813
814 skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;
815 goto out;
816 }
817
818 if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)
819 goto out;
820 } else {
821 if (skb_is_gso(skb))
822 return xfrm_output_gso(net, sk, skb);
823 }
824
825 if (skb->ip_summed == CHECKSUM_PARTIAL) {
826 err = skb_checksum_help(skb);
827 if (err) {
828 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
829 kfree_skb(skb);
830 return err;
831 }
832 }
833
834 out:
835 return xfrm_output2(net, sk, skb);
836 }
837 EXPORT_SYMBOL_GPL(xfrm_output);
838
xfrm4_tunnel_check_size(struct sk_buff * skb)839 int xfrm4_tunnel_check_size(struct sk_buff *skb)
840 {
841 int mtu, ret = 0;
842
843 if (IPCB(skb)->flags & IPSKB_XFRM_TUNNEL_SIZE)
844 goto out;
845
846 if (!(ip_hdr(skb)->frag_off & htons(IP_DF)) || skb->ignore_df)
847 goto out;
848
849 mtu = dst_mtu(skb_dst(skb));
850 if ((!skb_is_gso(skb) && skb->len > mtu) ||
851 (skb_is_gso(skb) &&
852 !skb_gso_validate_network_len(skb, ip_skb_dst_mtu(skb->sk, skb)))) {
853 skb->protocol = htons(ETH_P_IP);
854
855 if (skb->sk && sk_fullsock(skb->sk))
856 xfrm_local_error(skb, mtu);
857 else
858 icmp_send(skb, ICMP_DEST_UNREACH,
859 ICMP_FRAG_NEEDED, htonl(mtu));
860 ret = -EMSGSIZE;
861 }
862 out:
863 return ret;
864 }
865 EXPORT_SYMBOL_GPL(xfrm4_tunnel_check_size);
866
xfrm4_extract_output(struct xfrm_state * x,struct sk_buff * skb)867 static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
868 {
869 int err;
870
871 if (x->outer_mode.encap == XFRM_MODE_BEET &&
872 ip_is_fragment(ip_hdr(skb))) {
873 net_warn_ratelimited("BEET mode doesn't support inner IPv4 fragments\n");
874 return -EAFNOSUPPORT;
875 }
876
877 err = xfrm4_tunnel_check_size(skb);
878 if (err)
879 return err;
880
881 XFRM_MODE_SKB_CB(skb)->protocol = ip_hdr(skb)->protocol;
882
883 xfrm4_extract_header(skb);
884 return 0;
885 }
886
887 #if IS_ENABLED(CONFIG_IPV6)
xfrm6_tunnel_check_size(struct sk_buff * skb)888 int xfrm6_tunnel_check_size(struct sk_buff *skb)
889 {
890 int mtu, ret = 0;
891 struct dst_entry *dst = skb_dst(skb);
892 struct sock *sk = skb_to_full_sk(skb);
893
894 if (skb->ignore_df)
895 goto out;
896
897 mtu = dst_mtu(dst);
898 if (mtu < IPV6_MIN_MTU)
899 mtu = IPV6_MIN_MTU;
900
901 if ((!skb_is_gso(skb) && skb->len > mtu) ||
902 (skb_is_gso(skb) &&
903 !skb_gso_validate_network_len(skb, ip6_skb_dst_mtu(skb)))) {
904 skb->dev = dst->dev;
905 skb->protocol = htons(ETH_P_IPV6);
906
907 if (xfrm6_local_dontfrag(sk))
908 xfrm6_local_rxpmtu(skb, mtu);
909 else if (sk)
910 xfrm_local_error(skb, mtu);
911 else
912 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
913 ret = -EMSGSIZE;
914 }
915 out:
916 return ret;
917 }
918 EXPORT_SYMBOL_GPL(xfrm6_tunnel_check_size);
919 #endif
920
xfrm6_extract_output(struct xfrm_state * x,struct sk_buff * skb)921 static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)
922 {
923 #if IS_ENABLED(CONFIG_IPV6)
924 int err;
925
926 err = xfrm6_tunnel_check_size(skb);
927 if (err)
928 return err;
929
930 XFRM_MODE_SKB_CB(skb)->protocol = ipv6_hdr(skb)->nexthdr;
931
932 xfrm6_extract_header(skb);
933 return 0;
934 #else
935 WARN_ON_ONCE(1);
936 return -EAFNOSUPPORT;
937 #endif
938 }
939
xfrm_inner_extract_output(struct xfrm_state * x,struct sk_buff * skb)940 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb)
941 {
942 switch (skb->protocol) {
943 case htons(ETH_P_IP):
944 return xfrm4_extract_output(x, skb);
945 case htons(ETH_P_IPV6):
946 return xfrm6_extract_output(x, skb);
947 }
948
949 return -EAFNOSUPPORT;
950 }
951
xfrm_local_error(struct sk_buff * skb,int mtu)952 void xfrm_local_error(struct sk_buff *skb, int mtu)
953 {
954 unsigned int proto;
955 struct xfrm_state_afinfo *afinfo;
956
957 if (skb->protocol == htons(ETH_P_IP))
958 proto = AF_INET;
959 else if (skb->protocol == htons(ETH_P_IPV6) &&
960 skb->sk->sk_family == AF_INET6)
961 proto = AF_INET6;
962 else
963 return;
964
965 afinfo = xfrm_state_get_afinfo(proto);
966 if (afinfo) {
967 afinfo->local_error(skb, mtu);
968 rcu_read_unlock();
969 }
970 }
971 EXPORT_SYMBOL_GPL(xfrm_local_error);
972