28 static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb);
29 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb);
31 static int xfrm_skb_check_space(struct sk_buff *skb)  in xfrm_skb_check_space()  argument
33 	struct dst_entry *dst = skb_dst(skb);  in xfrm_skb_check_space()
35 		- skb_headroom(skb);  in xfrm_skb_check_space()
36 	int ntail = dst->dev->needed_tailroom - skb_tailroom(skb);  in xfrm_skb_check_space()
45 	return pskb_expand_head(skb, nhead, ntail, GFP_ATOMIC);  in xfrm_skb_check_space()
52 static struct dst_entry *skb_dst_pop(struct sk_buff *skb)  in skb_dst_pop()  argument
54 	struct dst_entry *child = dst_clone(xfrm_dst_child(skb_dst(skb)));  in skb_dst_pop()
56 	skb_dst_drop(skb);  in skb_dst_pop()
65 static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_transport_output()  argument
67 	struct iphdr *iph = ip_hdr(skb);  in xfrm4_transport_output()
70 	skb_set_inner_transport_header(skb, skb_transport_offset(skb));  in xfrm4_transport_output()
72 	skb_set_network_header(skb, -x->props.header_len);  in xfrm4_transport_output()
73 	skb->mac_header = skb->network_header +  in xfrm4_transport_output()
75 	skb->transport_header = skb->network_header + ihl;  in xfrm4_transport_output()
76 	__skb_pull(skb, ihl);  in xfrm4_transport_output()
77 	memmove(skb_network_header(skb), iph, ihl);  in xfrm4_transport_output()
82 static int mip6_rthdr_offset(struct sk_buff *skb, u8 **nexthdr, int type)  in mip6_rthdr_offset()  argument
84 	const unsigned char *nh = skb_network_header(skb);  in mip6_rthdr_offset()
89 	packet_len = skb_tail_pointer(skb) - nh;  in mip6_rthdr_offset()
90 	*nexthdr = &ipv6_hdr(skb)->nexthdr;  in mip6_rthdr_offset()
113 			if (ipv6_find_tlv(skb, offset, IPV6_TLV_HAO) >= 0) {  in mip6_rthdr_offset()
129 		exthdr = (struct ipv6_opt_hdr *)(skb_network_header(skb) +  in mip6_rthdr_offset()
142 static int xfrm6_hdr_offset(struct xfrm_state *x, struct sk_buff *skb, u8 **prevhdr)  in xfrm6_hdr_offset()  argument
148 		return mip6_rthdr_offset(skb, prevhdr, x->type->proto);  in xfrm6_hdr_offset()
154 	return ip6_find_1stfragopt(skb, prevhdr);  in xfrm6_hdr_offset()
163 static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_transport_output()  argument
170 	iph = ipv6_hdr(skb);  in xfrm6_transport_output()
171 	skb_set_inner_transport_header(skb, skb_transport_offset(skb));  in xfrm6_transport_output()
173 	hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);  in xfrm6_transport_output()
176 	skb_set_mac_header(skb,  in xfrm6_transport_output()
177 			   (prevhdr - x->props.header_len) - skb->data);  in xfrm6_transport_output()
178 	skb_set_network_header(skb, -x->props.header_len);  in xfrm6_transport_output()
179 	skb->transport_header = skb->network_header + hdr_len;  in xfrm6_transport_output()
180 	__skb_pull(skb, hdr_len);  in xfrm6_transport_output()
181 	memmove(ipv6_hdr(skb), iph, hdr_len);  in xfrm6_transport_output()
194 static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_ro_output()  argument
201 	iph = ipv6_hdr(skb);  in xfrm6_ro_output()
203 	hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);  in xfrm6_ro_output()
206 	skb_set_mac_header(skb,  in xfrm6_ro_output()
207 			   (prevhdr - x->props.header_len) - skb->data);  in xfrm6_ro_output()
208 	skb_set_network_header(skb, -x->props.header_len);  in xfrm6_ro_output()
209 	skb->transport_header = skb->network_header + hdr_len;  in xfrm6_ro_output()
210 	__skb_pull(skb, hdr_len);  in xfrm6_ro_output()
211 	memmove(ipv6_hdr(skb), iph, hdr_len);  in xfrm6_ro_output()
224 static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_beet_encap_add()  argument
231 	optlen = XFRM_MODE_SKB_CB(skb)->optlen;  in xfrm4_beet_encap_add()
235 	skb_set_network_header(skb, -x->props.header_len - hdrlen +  in xfrm4_beet_encap_add()
236 			       (XFRM_MODE_SKB_CB(skb)->ihl - sizeof(*top_iph)));  in xfrm4_beet_encap_add()
238 		skb->network_header += IPV4_BEET_PHMAXLEN;  in xfrm4_beet_encap_add()
239 	skb->mac_header = skb->network_header +  in xfrm4_beet_encap_add()
241 	skb->transport_header = skb->network_header + sizeof(*top_iph);  in xfrm4_beet_encap_add()
243 	xfrm4_beet_make_header(skb);  in xfrm4_beet_encap_add()
245 	ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdrlen);  in xfrm4_beet_encap_add()
247 	top_iph = ip_hdr(skb);  in xfrm4_beet_encap_add()
273 static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_tunnel_encap_add()  argument
275 	bool small_ipv6 = (skb->protocol == htons(ETH_P_IPV6)) && (skb->len <= IPV6_MIN_MTU);  in xfrm4_tunnel_encap_add()
276 	struct dst_entry *dst = skb_dst(skb);  in xfrm4_tunnel_encap_add()
280 	skb_set_inner_network_header(skb, skb_network_offset(skb));  in xfrm4_tunnel_encap_add()
281 	skb_set_inner_transport_header(skb, skb_transport_offset(skb));  in xfrm4_tunnel_encap_add()
283 	skb_set_network_header(skb, -x->props.header_len);  in xfrm4_tunnel_encap_add()
284 	skb->mac_header = skb->network_header +  in xfrm4_tunnel_encap_add()
286 	skb->transport_header = skb->network_header + sizeof(*top_iph);  in xfrm4_tunnel_encap_add()
287 	top_iph = ip_hdr(skb);  in xfrm4_tunnel_encap_add()
292 	top_iph->protocol = xfrm_af2proto(skb_dst(skb)->ops->family);  in xfrm4_tunnel_encap_add()
298 		top_iph->tos = XFRM_MODE_SKB_CB(skb)->tos;  in xfrm4_tunnel_encap_add()
300 					    XFRM_MODE_SKB_CB(skb)->tos);  in xfrm4_tunnel_encap_add()
307 		0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF));  in xfrm4_tunnel_encap_add()
313 	ip_select_ident(dev_net(dst->dev), skb, NULL);  in xfrm4_tunnel_encap_add()
319 static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_tunnel_encap_add()  argument
321 	struct dst_entry *dst = skb_dst(skb);  in xfrm6_tunnel_encap_add()
325 	skb_set_inner_network_header(skb, skb_network_offset(skb));  in xfrm6_tunnel_encap_add()
326 	skb_set_inner_transport_header(skb, skb_transport_offset(skb));  in xfrm6_tunnel_encap_add()
328 	skb_set_network_header(skb, -x->props.header_len);  in xfrm6_tunnel_encap_add()
329 	skb->mac_header = skb->network_header +  in xfrm6_tunnel_encap_add()
331 	skb->transport_header = skb->network_header + sizeof(*top_iph);  in xfrm6_tunnel_encap_add()
332 	top_iph = ipv6_hdr(skb);  in xfrm6_tunnel_encap_add()
336 	memcpy(top_iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl,  in xfrm6_tunnel_encap_add()
338 	top_iph->nexthdr = xfrm_af2proto(skb_dst(skb)->ops->family);  in xfrm6_tunnel_encap_add()
343 		dsfield = XFRM_MODE_SKB_CB(skb)->tos;  in xfrm6_tunnel_encap_add()
344 	dsfield = INET_ECN_encapsulate(dsfield, XFRM_MODE_SKB_CB(skb)->tos);  in xfrm6_tunnel_encap_add()
354 static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_beet_encap_add()  argument
361 	optlen = XFRM_MODE_SKB_CB(skb)->optlen;  in xfrm6_beet_encap_add()
365 	skb_set_network_header(skb, -x->props.header_len - hdr_len);  in xfrm6_beet_encap_add()
367 		skb->network_header += IPV4_BEET_PHMAXLEN;  in xfrm6_beet_encap_add()
368 	skb->mac_header = skb->network_header +  in xfrm6_beet_encap_add()
370 	skb->transport_header = skb->network_header + sizeof(*top_iph);  in xfrm6_beet_encap_add()
371 	ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdr_len);  in xfrm6_beet_encap_add()
373 	xfrm6_beet_make_header(skb);  in xfrm6_beet_encap_add()
375 	top_iph = ipv6_hdr(skb);  in xfrm6_beet_encap_add()
405 static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_prepare_output()  argument
409 	err = xfrm_inner_extract_output(x, skb);  in xfrm4_prepare_output()
413 	IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;  in xfrm4_prepare_output()
414 	skb->protocol = htons(ETH_P_IP);  in xfrm4_prepare_output()
418 		return xfrm4_beet_encap_add(x, skb);  in xfrm4_prepare_output()
420 		return xfrm4_tunnel_encap_add(x, skb);  in xfrm4_prepare_output()
427 static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_prepare_output()  argument
432 	err = xfrm_inner_extract_output(x, skb);  in xfrm6_prepare_output()
436 	skb->ignore_df = 1;  in xfrm6_prepare_output()
437 	skb->protocol = htons(ETH_P_IPV6);  in xfrm6_prepare_output()
441 		return xfrm6_beet_encap_add(x, skb);  in xfrm6_prepare_output()
443 		return xfrm6_tunnel_encap_add(x, skb);  in xfrm6_prepare_output()
453 static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm_outer_mode_output()  argument
459 			return xfrm4_prepare_output(x, skb);  in xfrm_outer_mode_output()
461 			return xfrm6_prepare_output(x, skb);  in xfrm_outer_mode_output()
465 			return xfrm4_transport_output(x, skb);  in xfrm_outer_mode_output()
467 			return xfrm6_transport_output(x, skb);  in xfrm_outer_mode_output()
471 			return xfrm6_ro_output(x, skb);  in xfrm_outer_mode_output()
476 			return x->mode_cbs->prepare_output(x, skb);  in xfrm_outer_mode_output()
485 int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)  in pktgen_xfrm_outer_mode_output()  argument
487 	return xfrm_outer_mode_output(x, skb);  in pktgen_xfrm_outer_mode_output()
492 static int xfrm_output_one(struct sk_buff *skb, int err)  in xfrm_output_one()  argument
494 	struct dst_entry *dst = skb_dst(skb);  in xfrm_output_one()
502 		err = xfrm_skb_check_space(skb);  in xfrm_output_one()
508 		skb->mark = xfrm_smark_get(skb->mark, x);  in xfrm_output_one()
510 		err = xfrm_outer_mode_output(x, skb);  in xfrm_output_one()
530 		err = xfrm_replay_overflow(x, skb);  in xfrm_output_one()
536 		x->curlft.bytes += skb->len;  in xfrm_output_one()
542 		skb_dst_force(skb);  in xfrm_output_one()
543 		if (!skb_dst(skb)) {  in xfrm_output_one()
549 		if (xfrm_offload(skb)) {  in xfrm_output_one()
550 			x->type_offload->encap(x, skb);  in xfrm_output_one()
553 			skb->encapsulation = 0;  in xfrm_output_one()
555 			err = x->type->output(x, skb);  in xfrm_output_one()
566 		dst = skb_dst_pop(skb);  in xfrm_output_one()
572 		skb_dst_set(skb, dst);  in xfrm_output_one()
581 	kfree_skb(skb);  in xfrm_output_one()
586 int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err)  in xfrm_output_resume()  argument
588 	struct net *net = xs_net(skb_dst(skb)->xfrm);  in xfrm_output_resume()
590 	while (likely((err = xfrm_output_one(skb, err)) == 0)) {  in xfrm_output_resume()
591 		nf_reset_ct(skb);  in xfrm_output_resume()
593 		err = skb_dst(skb)->ops->local_out(net, sk, skb);  in xfrm_output_resume()
597 		if (!skb_dst(skb)->xfrm)  in xfrm_output_resume()
598 			return dst_output(net, sk, skb);  in xfrm_output_resume()
600 		err = nf_hook(skb_dst(skb)->ops->family,  in xfrm_output_resume()
601 			      NF_INET_POST_ROUTING, net, sk, skb,  in xfrm_output_resume()
602 			      NULL, skb_dst(skb)->dev, xfrm_output2);  in xfrm_output_resume()
615 static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb)  in xfrm_output2()  argument
617 	return xfrm_output_resume(sk, skb, 1);  in xfrm_output2()
620 static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb)  in xfrm_output_gso()  argument
624 	BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET);  in xfrm_output_gso()
625 	BUILD_BUG_ON(sizeof(*IP6CB(skb)) > SKB_GSO_CB_OFFSET);  in xfrm_output_gso()
626 	segs = skb_gso_segment(skb, 0);  in xfrm_output_gso()
627 	kfree_skb(skb);  in xfrm_output_gso()
658 static void xfrm_get_inner_ipproto(struct sk_buff *skb, struct xfrm_state *x)  in xfrm_get_inner_ipproto()  argument
660 	struct xfrm_offload *xo = xfrm_offload(skb);  in xfrm_get_inner_ipproto()
669 			xo->inner_ipproto = ip_hdr(skb)->protocol;  in xfrm_get_inner_ipproto()
672 			xo->inner_ipproto = ipv6_hdr(skb)->nexthdr;  in xfrm_get_inner_ipproto()
686 	if (!skb->encapsulation)  in xfrm_get_inner_ipproto()
689 	if (skb->inner_protocol_type == ENCAP_TYPE_IPPROTO) {  in xfrm_get_inner_ipproto()
690 		xo->inner_ipproto = skb->inner_ipproto;  in xfrm_get_inner_ipproto()
694 	if (skb->inner_protocol_type != ENCAP_TYPE_ETHER)  in xfrm_get_inner_ipproto()
697 	eth = (struct ethhdr *)skb_inner_mac_header(skb);  in xfrm_get_inner_ipproto()
701 		xo->inner_ipproto = inner_ipv6_hdr(skb)->nexthdr;  in xfrm_get_inner_ipproto()
704 		xo->inner_ipproto = inner_ip_hdr(skb)->protocol;  in xfrm_get_inner_ipproto()
709 int xfrm_output(struct sock *sk, struct sk_buff *skb)  in xfrm_output()  argument
711 	struct net *net = dev_net(skb_dst(skb)->dev);  in xfrm_output()
712 	struct xfrm_state *x = skb_dst(skb)->xfrm;  in xfrm_output()
717 		: skb_dst(skb)->ops->family;  in xfrm_output()
721 		memset(IPCB(skb), 0, sizeof(*IPCB(skb)));  in xfrm_output()
722 		IPCB(skb)->flags |= IPSKB_XFRM_TRANSFORMED;  in xfrm_output()
725 		memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));  in xfrm_output()
727 		IP6CB(skb)->flags |= IP6SKB_XFRM_TRANSFORMED;  in xfrm_output()
732 		if (!xfrm_dev_offload_ok(skb, x)) {  in xfrm_output()
734 			kfree_skb(skb);  in xfrm_output()
738 		return xfrm_output_resume(sk, skb, 0);  in xfrm_output()
741 	secpath_reset(skb);  in xfrm_output()
743 	if (xfrm_dev_offload_ok(skb, x)) {  in xfrm_output()
746 		sp = secpath_set(skb);  in xfrm_output()
749 			kfree_skb(skb);  in xfrm_output()
757 		xfrm_get_inner_ipproto(skb, x);  in xfrm_output()
758 		skb->encapsulation = 1;  in xfrm_output()
760 		if (skb_is_gso(skb)) {  in xfrm_output()
761 			if (skb->inner_protocol)  in xfrm_output()
762 				return xfrm_output_gso(net, sk, skb);  in xfrm_output()
764 			skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;  in xfrm_output()
771 		if (skb_is_gso(skb))  in xfrm_output()
772 			return xfrm_output_gso(net, sk, skb);  in xfrm_output()
775 	if (skb->ip_summed == CHECKSUM_PARTIAL) {  in xfrm_output()
776 		err = skb_checksum_help(skb);  in xfrm_output()
779 			kfree_skb(skb);  in xfrm_output()
785 	return xfrm_output2(net, sk, skb);  in xfrm_output()
789 static int xfrm4_tunnel_check_size(struct sk_buff *skb)  in xfrm4_tunnel_check_size()  argument
793 	if (IPCB(skb)->flags & IPSKB_XFRM_TUNNEL_SIZE)  in xfrm4_tunnel_check_size()
796 	if (!(ip_hdr(skb)->frag_off & htons(IP_DF)) || skb->ignore_df)  in xfrm4_tunnel_check_size()
799 	mtu = dst_mtu(skb_dst(skb));  in xfrm4_tunnel_check_size()
800 	if ((!skb_is_gso(skb) && skb->len > mtu) ||  in xfrm4_tunnel_check_size()
801 	    (skb_is_gso(skb) &&  in xfrm4_tunnel_check_size()
802 	     !skb_gso_validate_network_len(skb, ip_skb_dst_mtu(skb->sk, skb)))) {  in xfrm4_tunnel_check_size()
803 		skb->protocol = htons(ETH_P_IP);  in xfrm4_tunnel_check_size()
805 		if (skb->sk && sk_fullsock(skb->sk))  in xfrm4_tunnel_check_size()
806 			xfrm_local_error(skb, mtu);  in xfrm4_tunnel_check_size()
808 			icmp_send(skb, ICMP_DEST_UNREACH,  in xfrm4_tunnel_check_size()
816 static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm4_extract_output()  argument
821 	    ip_is_fragment(ip_hdr(skb))) {  in xfrm4_extract_output()
826 	err = xfrm4_tunnel_check_size(skb);  in xfrm4_extract_output()
830 	XFRM_MODE_SKB_CB(skb)->protocol = ip_hdr(skb)->protocol;  in xfrm4_extract_output()
832 	xfrm4_extract_header(skb);  in xfrm4_extract_output()
837 static int xfrm6_tunnel_check_size(struct sk_buff *skb)  in xfrm6_tunnel_check_size()  argument
840 	struct dst_entry *dst = skb_dst(skb);  in xfrm6_tunnel_check_size()
841 	struct sock *sk = skb_to_full_sk(skb);  in xfrm6_tunnel_check_size()
843 	if (skb->ignore_df)  in xfrm6_tunnel_check_size()
850 	if ((!skb_is_gso(skb) && skb->len > mtu) ||  in xfrm6_tunnel_check_size()
851 	    (skb_is_gso(skb) &&  in xfrm6_tunnel_check_size()
852 	     !skb_gso_validate_network_len(skb, ip6_skb_dst_mtu(skb)))) {  in xfrm6_tunnel_check_size()
853 		skb->dev = dst->dev;  in xfrm6_tunnel_check_size()
854 		skb->protocol = htons(ETH_P_IPV6);  in xfrm6_tunnel_check_size()
857 			ipv6_stub->xfrm6_local_rxpmtu(skb, mtu);  in xfrm6_tunnel_check_size()
859 			xfrm_local_error(skb, mtu);  in xfrm6_tunnel_check_size()
861 			icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);  in xfrm6_tunnel_check_size()
869 static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm6_extract_output()  argument
874 	err = xfrm6_tunnel_check_size(skb);  in xfrm6_extract_output()
878 	XFRM_MODE_SKB_CB(skb)->protocol = ipv6_hdr(skb)->nexthdr;  in xfrm6_extract_output()
880 	xfrm6_extract_header(skb);  in xfrm6_extract_output()
888 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb)  in xfrm_inner_extract_output()  argument
890 	switch (skb->protocol) {  in xfrm_inner_extract_output()
892 		return xfrm4_extract_output(x, skb);  in xfrm_inner_extract_output()
894 		return xfrm6_extract_output(x, skb);  in xfrm_inner_extract_output()
900 void xfrm_local_error(struct sk_buff *skb, int mtu)  in xfrm_local_error()  argument
905 	if (skb->protocol == htons(ETH_P_IP))  in xfrm_local_error()
907 	else if (skb->protocol == htons(ETH_P_IPV6) &&  in xfrm_local_error()
908 		 skb->sk->sk_family == AF_INET6)  in xfrm_local_error()
915 		afinfo->local_error(skb, mtu);  in xfrm_local_error()