175a6faf6SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
27785bba2SSteffen Klassert /*
37785bba2SSteffen Klassert * IPV6 GSO/GRO offload support
47785bba2SSteffen Klassert * Linux INET implementation
57785bba2SSteffen Klassert *
67785bba2SSteffen Klassert * Copyright (C) 2016 secunet Security Networks AG
77785bba2SSteffen Klassert * Author: Steffen Klassert <steffen.klassert@secunet.com>
87785bba2SSteffen Klassert *
97785bba2SSteffen Klassert * ESP GRO support
107785bba2SSteffen Klassert */
117785bba2SSteffen Klassert
127785bba2SSteffen Klassert #include <linux/skbuff.h>
137785bba2SSteffen Klassert #include <linux/init.h>
147785bba2SSteffen Klassert #include <net/protocol.h>
157785bba2SSteffen Klassert #include <crypto/aead.h>
167785bba2SSteffen Klassert #include <crypto/authenc.h>
177785bba2SSteffen Klassert #include <linux/err.h>
187785bba2SSteffen Klassert #include <linux/module.h>
194721031cSEric Dumazet #include <net/gro.h>
20d457a0e3SEric Dumazet #include <net/gso.h>
217785bba2SSteffen Klassert #include <net/ip.h>
227785bba2SSteffen Klassert #include <net/xfrm.h>
237785bba2SSteffen Klassert #include <net/esp.h>
247785bba2SSteffen Klassert #include <linux/scatterlist.h>
257785bba2SSteffen Klassert #include <linux/kernel.h>
267785bba2SSteffen Klassert #include <linux/slab.h>
277785bba2SSteffen Klassert #include <linux/spinlock.h>
287785bba2SSteffen Klassert #include <net/ip6_route.h>
297785bba2SSteffen Klassert #include <net/ipv6.h>
307785bba2SSteffen Klassert #include <linux/icmpv6.h>
317785bba2SSteffen Klassert
esp6_nexthdr_esp_offset(struct ipv6hdr * ipv6_hdr,int nhlen)32ca3a1b85SYossi Kuperman static __u16 esp6_nexthdr_esp_offset(struct ipv6hdr *ipv6_hdr, int nhlen)
33ca3a1b85SYossi Kuperman {
34ca3a1b85SYossi Kuperman int off = sizeof(struct ipv6hdr);
35ca3a1b85SYossi Kuperman struct ipv6_opt_hdr *exthdr;
36ca3a1b85SYossi Kuperman
37221ddb72SSteffen Klassert /* ESP or ESPINUDP */
38221ddb72SSteffen Klassert if (likely(ipv6_hdr->nexthdr == NEXTHDR_ESP ||
39221ddb72SSteffen Klassert ipv6_hdr->nexthdr == NEXTHDR_UDP))
40ca3a1b85SYossi Kuperman return offsetof(struct ipv6hdr, nexthdr);
41ca3a1b85SYossi Kuperman
42ca3a1b85SYossi Kuperman while (off < nhlen) {
43ca3a1b85SYossi Kuperman exthdr = (void *)ipv6_hdr + off;
44ca3a1b85SYossi Kuperman if (exthdr->nexthdr == NEXTHDR_ESP)
45ca3a1b85SYossi Kuperman return off;
46ca3a1b85SYossi Kuperman
47ca3a1b85SYossi Kuperman off += ipv6_optlen(exthdr);
48ca3a1b85SYossi Kuperman }
49ca3a1b85SYossi Kuperman
50ca3a1b85SYossi Kuperman return 0;
51ca3a1b85SYossi Kuperman }
52ca3a1b85SYossi Kuperman
esp6_gro_receive(struct list_head * head,struct sk_buff * skb)53d4546c25SDavid Miller static struct sk_buff *esp6_gro_receive(struct list_head *head,
547785bba2SSteffen Klassert struct sk_buff *skb)
557785bba2SSteffen Klassert {
567785bba2SSteffen Klassert int offset = skb_gro_offset(skb);
577785bba2SSteffen Klassert struct xfrm_offload *xo;
587785bba2SSteffen Klassert struct xfrm_state *x;
59221ddb72SSteffen Klassert int encap_type = 0;
607785bba2SSteffen Klassert __be32 seq;
617785bba2SSteffen Klassert __be32 spi;
62ca3a1b85SYossi Kuperman int nhoff;
637785bba2SSteffen Klassert
64221ddb72SSteffen Klassert if (NAPI_GRO_CB(skb)->proto == IPPROTO_UDP)
65221ddb72SSteffen Klassert encap_type = UDP_ENCAP_ESPINUDP;
66221ddb72SSteffen Klassert
67374d1b5aSSteffen Klassert if (!pskb_pull(skb, offset))
68374d1b5aSSteffen Klassert return NULL;
697785bba2SSteffen Klassert
70e91001baSColin Ian King if (xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq) != 0)
717785bba2SSteffen Klassert goto out;
727785bba2SSteffen Klassert
73bcd1f8a4SSteffen Klassert xo = xfrm_offload(skb);
74bcd1f8a4SSteffen Klassert if (!xo || !(xo->flags & CRYPTO_DONE)) {
750ca64da1SFlorian Westphal struct sec_path *sp = secpath_set(skb);
760ca64da1SFlorian Westphal
770ca64da1SFlorian Westphal if (!sp)
787785bba2SSteffen Klassert goto out;
797785bba2SSteffen Klassert
800ca64da1SFlorian Westphal if (sp->len == XFRM_MAX_DEPTH)
816ed69184SMyungho Jung goto out_reset;
827785bba2SSteffen Klassert
837785bba2SSteffen Klassert x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
847785bba2SSteffen Klassert (xfrm_address_t *)&ipv6_hdr(skb)->daddr,
857785bba2SSteffen Klassert spi, IPPROTO_ESP, AF_INET6);
86*15f5fe9eSAntony Antony
87*15f5fe9eSAntony Antony if (unlikely(x && x->dir && x->dir != XFRM_SA_DIR_IN)) {
88*15f5fe9eSAntony Antony /* non-offload path will record the error and audit log */
89*15f5fe9eSAntony Antony xfrm_state_put(x);
90*15f5fe9eSAntony Antony x = NULL;
91*15f5fe9eSAntony Antony }
92*15f5fe9eSAntony Antony
937785bba2SSteffen Klassert if (!x)
946ed69184SMyungho Jung goto out_reset;
957785bba2SSteffen Klassert
964e4362d2SUlrich Weber skb->mark = xfrm_smark_get(skb->mark, x);
974e4362d2SUlrich Weber
980ca64da1SFlorian Westphal sp->xvec[sp->len++] = x;
990ca64da1SFlorian Westphal sp->olen++;
1007785bba2SSteffen Klassert
1017785bba2SSteffen Klassert xo = xfrm_offload(skb);
102db87668aSXin Long if (!xo)
1036ed69184SMyungho Jung goto out_reset;
1047785bba2SSteffen Klassert }
105bcd1f8a4SSteffen Klassert
1067785bba2SSteffen Klassert xo->flags |= XFRM_GRO;
1077785bba2SSteffen Klassert
108ca3a1b85SYossi Kuperman nhoff = esp6_nexthdr_esp_offset(ipv6_hdr(skb), offset);
109ca3a1b85SYossi Kuperman if (!nhoff)
110ca3a1b85SYossi Kuperman goto out;
111ca3a1b85SYossi Kuperman
112ca3a1b85SYossi Kuperman IP6CB(skb)->nhoff = nhoff;
1137785bba2SSteffen Klassert XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
1147785bba2SSteffen Klassert XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
1157785bba2SSteffen Klassert XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
1167785bba2SSteffen Klassert XFRM_SPI_SKB_CB(skb)->seq = seq;
1177785bba2SSteffen Klassert
1187785bba2SSteffen Klassert /* We don't need to handle errors from xfrm_input, it does all
1197785bba2SSteffen Klassert * the error handling and frees the resources on error. */
120221ddb72SSteffen Klassert xfrm_input(skb, IPPROTO_ESP, spi, encap_type);
1217785bba2SSteffen Klassert
1227785bba2SSteffen Klassert return ERR_PTR(-EINPROGRESS);
1236ed69184SMyungho Jung out_reset:
1246ed69184SMyungho Jung secpath_reset(skb);
1257785bba2SSteffen Klassert out:
1267785bba2SSteffen Klassert skb_push(skb, offset);
1277785bba2SSteffen Klassert NAPI_GRO_CB(skb)->same_flow = 0;
1287785bba2SSteffen Klassert NAPI_GRO_CB(skb)->flush = 1;
1297785bba2SSteffen Klassert
1307785bba2SSteffen Klassert return NULL;
1317785bba2SSteffen Klassert }
1327785bba2SSteffen Klassert
esp6_gso_encap(struct xfrm_state * x,struct sk_buff * skb)1337862b405SSteffen Klassert static void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb)
1347862b405SSteffen Klassert {
1357862b405SSteffen Klassert struct ip_esp_hdr *esph;
1367862b405SSteffen Klassert struct ipv6hdr *iph = ipv6_hdr(skb);
1377862b405SSteffen Klassert struct xfrm_offload *xo = xfrm_offload(skb);
1383c96ec56SXin Long u8 proto = iph->nexthdr;
1397862b405SSteffen Klassert
1407862b405SSteffen Klassert skb_push(skb, -skb_network_offset(skb));
1413c96ec56SXin Long
1423c96ec56SXin Long if (x->outer_mode.encap == XFRM_MODE_TRANSPORT) {
1433c96ec56SXin Long __be16 frag;
1443c96ec56SXin Long
1453c96ec56SXin Long ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &proto, &frag);
1463c96ec56SXin Long }
1473c96ec56SXin Long
1487862b405SSteffen Klassert esph = ip_esp_hdr(skb);
1497862b405SSteffen Klassert *skb_mac_header(skb) = IPPROTO_ESP;
1507862b405SSteffen Klassert
1517862b405SSteffen Klassert esph->spi = x->id.spi;
1527862b405SSteffen Klassert esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
1537862b405SSteffen Klassert
1547862b405SSteffen Klassert xo->proto = proto;
1557862b405SSteffen Klassert }
1567862b405SSteffen Klassert
xfrm6_tunnel_gso_segment(struct xfrm_state * x,struct sk_buff * skb,netdev_features_t features)1577613b92bSFlorian Westphal static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x,
1587613b92bSFlorian Westphal struct sk_buff *skb,
1597613b92bSFlorian Westphal netdev_features_t features)
1607613b92bSFlorian Westphal {
16126dbd66eSSabrina Dubroca __be16 type = x->inner_mode.family == AF_INET ? htons(ETH_P_IP)
16226dbd66eSSabrina Dubroca : htons(ETH_P_IPV6);
16326dbd66eSSabrina Dubroca
16426dbd66eSSabrina Dubroca return skb_eth_gso_segment(skb, features, type);
1657613b92bSFlorian Westphal }
1667613b92bSFlorian Westphal
xfrm6_transport_gso_segment(struct xfrm_state * x,struct sk_buff * skb,netdev_features_t features)1677613b92bSFlorian Westphal static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x,
1687613b92bSFlorian Westphal struct sk_buff *skb,
1697613b92bSFlorian Westphal netdev_features_t features)
1707613b92bSFlorian Westphal {
1717613b92bSFlorian Westphal const struct net_offload *ops;
1727613b92bSFlorian Westphal struct sk_buff *segs = ERR_PTR(-EINVAL);
1737613b92bSFlorian Westphal struct xfrm_offload *xo = xfrm_offload(skb);
1747613b92bSFlorian Westphal
1757613b92bSFlorian Westphal skb->transport_header += x->props.header_len;
1767613b92bSFlorian Westphal ops = rcu_dereference(inet6_offloads[xo->proto]);
1777613b92bSFlorian Westphal if (likely(ops && ops->callbacks.gso_segment))
1787613b92bSFlorian Westphal segs = ops->callbacks.gso_segment(skb, features);
1797613b92bSFlorian Westphal
1807613b92bSFlorian Westphal return segs;
1817613b92bSFlorian Westphal }
1827613b92bSFlorian Westphal
xfrm6_beet_gso_segment(struct xfrm_state * x,struct sk_buff * skb,netdev_features_t features)1837f9e40ebSXin Long static struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x,
1847f9e40ebSXin Long struct sk_buff *skb,
1857f9e40ebSXin Long netdev_features_t features)
1867f9e40ebSXin Long {
1877f9e40ebSXin Long struct xfrm_offload *xo = xfrm_offload(skb);
1887f9e40ebSXin Long struct sk_buff *segs = ERR_PTR(-EINVAL);
1897f9e40ebSXin Long const struct net_offload *ops;
19025a44ae9SXin Long u8 proto = xo->proto;
1917f9e40ebSXin Long
1927f9e40ebSXin Long skb->transport_header += x->props.header_len;
1937f9e40ebSXin Long
1947f9e40ebSXin Long if (x->sel.family != AF_INET6) {
1957f9e40ebSXin Long skb->transport_header -=
1967f9e40ebSXin Long (sizeof(struct ipv6hdr) - sizeof(struct iphdr));
1977f9e40ebSXin Long
1987f9e40ebSXin Long if (proto == IPPROTO_BEETPH) {
19956b1b7c6SXin Long struct ip_beet_phdr *ph =
20056b1b7c6SXin Long (struct ip_beet_phdr *)skb->data;
2017f9e40ebSXin Long
2027f9e40ebSXin Long skb->transport_header += ph->hdrlen * 8;
2037f9e40ebSXin Long proto = ph->nexthdr;
20456b1b7c6SXin Long } else {
20556b1b7c6SXin Long skb->transport_header -= IPV4_BEET_PHMAXLEN;
2067f9e40ebSXin Long }
2077f9e40ebSXin Long
2087f9e40ebSXin Long if (proto == IPPROTO_TCP)
2097f9e40ebSXin Long skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
21056b1b7c6SXin Long } else {
21125a44ae9SXin Long __be16 frag;
21225a44ae9SXin Long
21325a44ae9SXin Long skb->transport_header +=
21425a44ae9SXin Long ipv6_skip_exthdr(skb, 0, &proto, &frag);
2157f9e40ebSXin Long }
2167f9e40ebSXin Long
217053c8fdfSSteffen Klassert if (proto == IPPROTO_IPIP)
218053c8fdfSSteffen Klassert skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
219053c8fdfSSteffen Klassert
2207f9e40ebSXin Long __skb_pull(skb, skb_transport_offset(skb));
2217f9e40ebSXin Long ops = rcu_dereference(inet6_offloads[proto]);
2227f9e40ebSXin Long if (likely(ops && ops->callbacks.gso_segment))
2237f9e40ebSXin Long segs = ops->callbacks.gso_segment(skb, features);
2247f9e40ebSXin Long
2257f9e40ebSXin Long return segs;
2267f9e40ebSXin Long }
2277f9e40ebSXin Long
xfrm6_outer_mode_gso_segment(struct xfrm_state * x,struct sk_buff * skb,netdev_features_t features)2287613b92bSFlorian Westphal static struct sk_buff *xfrm6_outer_mode_gso_segment(struct xfrm_state *x,
2297613b92bSFlorian Westphal struct sk_buff *skb,
2307613b92bSFlorian Westphal netdev_features_t features)
2317613b92bSFlorian Westphal {
232c9500d7bSFlorian Westphal switch (x->outer_mode.encap) {
2337613b92bSFlorian Westphal case XFRM_MODE_TUNNEL:
2347613b92bSFlorian Westphal return xfrm6_tunnel_gso_segment(x, skb, features);
2357613b92bSFlorian Westphal case XFRM_MODE_TRANSPORT:
2367613b92bSFlorian Westphal return xfrm6_transport_gso_segment(x, skb, features);
2377f9e40ebSXin Long case XFRM_MODE_BEET:
2387f9e40ebSXin Long return xfrm6_beet_gso_segment(x, skb, features);
2397613b92bSFlorian Westphal }
2407613b92bSFlorian Westphal
2417613b92bSFlorian Westphal return ERR_PTR(-EOPNOTSUPP);
2427613b92bSFlorian Westphal }
2437613b92bSFlorian Westphal
esp6_gso_segment(struct sk_buff * skb,netdev_features_t features)2447862b405SSteffen Klassert static struct sk_buff *esp6_gso_segment(struct sk_buff *skb,
2457862b405SSteffen Klassert netdev_features_t features)
2467862b405SSteffen Klassert {
2477862b405SSteffen Klassert struct xfrm_state *x;
2487862b405SSteffen Klassert struct ip_esp_hdr *esph;
2497862b405SSteffen Klassert struct crypto_aead *aead;
2507862b405SSteffen Klassert netdev_features_t esp_features = features;
2517862b405SSteffen Klassert struct xfrm_offload *xo = xfrm_offload(skb);
2522294be0fSFlorian Westphal struct sec_path *sp;
2537862b405SSteffen Klassert
254ffa6f571SColin Ian King if (!xo)
2553dca3f38SSteffen Klassert return ERR_PTR(-EINVAL);
2567862b405SSteffen Klassert
257121d57afSWillem de Bruijn if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP))
2585ca11440SDavid S. Miller return ERR_PTR(-EINVAL);
2597862b405SSteffen Klassert
2602294be0fSFlorian Westphal sp = skb_sec_path(skb);
2612294be0fSFlorian Westphal x = sp->xvec[sp->len - 1];
2627862b405SSteffen Klassert aead = x->data;
2637862b405SSteffen Klassert esph = ip_esp_hdr(skb);
2647862b405SSteffen Klassert
2657862b405SSteffen Klassert if (esph->spi != x->id.spi)
2663dca3f38SSteffen Klassert return ERR_PTR(-EINVAL);
2677862b405SSteffen Klassert
2687862b405SSteffen Klassert if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))
2693dca3f38SSteffen Klassert return ERR_PTR(-EINVAL);
2707862b405SSteffen Klassert
2717862b405SSteffen Klassert __skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));
2727862b405SSteffen Klassert
2737862b405SSteffen Klassert skb->encap_hdr_csum = 1;
2747862b405SSteffen Klassert
275fcb662deSShannon Nelson if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev)
276154deab6SXin Long esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK |
277154deab6SXin Long NETIF_F_SCTP_CRC);
2785211fcfbSShannon Nelson else if (!(features & NETIF_F_HW_ESP_TX_CSUM))
279154deab6SXin Long esp_features = features & ~(NETIF_F_CSUM_MASK |
280154deab6SXin Long NETIF_F_SCTP_CRC);
2817862b405SSteffen Klassert
2827862b405SSteffen Klassert xo->flags |= XFRM_GSO_SEGMENT;
2837862b405SSteffen Klassert
2847613b92bSFlorian Westphal return xfrm6_outer_mode_gso_segment(x, skb, esp_features);
2857862b405SSteffen Klassert }
2867862b405SSteffen Klassert
esp6_input_tail(struct xfrm_state * x,struct sk_buff * skb)287383d0350SSteffen Klassert static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb)
288383d0350SSteffen Klassert {
289383d0350SSteffen Klassert struct crypto_aead *aead = x->data;
290e51a6472SIlan Tayari struct xfrm_offload *xo = xfrm_offload(skb);
291383d0350SSteffen Klassert
292383d0350SSteffen Klassert if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead)))
293383d0350SSteffen Klassert return -EINVAL;
294383d0350SSteffen Klassert
295e51a6472SIlan Tayari if (!(xo->flags & CRYPTO_DONE))
296383d0350SSteffen Klassert skb->ip_summed = CHECKSUM_NONE;
297383d0350SSteffen Klassert
298383d0350SSteffen Klassert return esp6_input_done2(skb, 0);
299383d0350SSteffen Klassert }
300383d0350SSteffen Klassert
esp6_xmit(struct xfrm_state * x,struct sk_buff * skb,netdev_features_t features)301383d0350SSteffen Klassert static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features)
302383d0350SSteffen Klassert {
3033dca3f38SSteffen Klassert int len;
304383d0350SSteffen Klassert int err;
305383d0350SSteffen Klassert int alen;
306383d0350SSteffen Klassert int blksize;
307383d0350SSteffen Klassert struct xfrm_offload *xo;
308383d0350SSteffen Klassert struct crypto_aead *aead;
309383d0350SSteffen Klassert struct esp_info esp;
310383d0350SSteffen Klassert bool hw_offload = true;
3113dca3f38SSteffen Klassert __u32 seq;
312383d0350SSteffen Klassert
313383d0350SSteffen Klassert esp.inplace = true;
314383d0350SSteffen Klassert
315383d0350SSteffen Klassert xo = xfrm_offload(skb);
316383d0350SSteffen Klassert
317383d0350SSteffen Klassert if (!xo)
318383d0350SSteffen Klassert return -EINVAL;
319383d0350SSteffen Klassert
320fcb662deSShannon Nelson if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) {
321383d0350SSteffen Klassert xo->flags |= CRYPTO_FALLBACK;
3228f92e03eSIlan Tayari hw_offload = false;
323383d0350SSteffen Klassert }
324383d0350SSteffen Klassert
325383d0350SSteffen Klassert esp.proto = xo->proto;
326383d0350SSteffen Klassert
327383d0350SSteffen Klassert /* skb is pure payload to encrypt */
328383d0350SSteffen Klassert
329383d0350SSteffen Klassert aead = x->data;
330383d0350SSteffen Klassert alen = crypto_aead_authsize(aead);
331383d0350SSteffen Klassert
332383d0350SSteffen Klassert esp.tfclen = 0;
333383d0350SSteffen Klassert /* XXX: Add support for tfc padding here. */
334383d0350SSteffen Klassert
335383d0350SSteffen Klassert blksize = ALIGN(crypto_aead_blocksize(aead), 4);
336383d0350SSteffen Klassert esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
337383d0350SSteffen Klassert esp.plen = esp.clen - skb->len - esp.tfclen;
338383d0350SSteffen Klassert esp.tailen = esp.tfclen + esp.plen + alen;
339383d0350SSteffen Klassert
340f076835aSJunlin Yang if (!hw_offload || !skb_is_gso(skb)) {
341383d0350SSteffen Klassert esp.nfrags = esp6_output_head(x, skb, &esp);
342383d0350SSteffen Klassert if (esp.nfrags < 0)
343383d0350SSteffen Klassert return esp.nfrags;
344383d0350SSteffen Klassert }
345383d0350SSteffen Klassert
3463dca3f38SSteffen Klassert seq = xo->seq.low;
3473dca3f38SSteffen Klassert
3480146dca7SSabrina Dubroca esp.esph = ip_esp_hdr(skb);
3490146dca7SSabrina Dubroca esp.esph->spi = x->id.spi;
350383d0350SSteffen Klassert
351383d0350SSteffen Klassert skb_push(skb, -skb_network_offset(skb));
352383d0350SSteffen Klassert
353383d0350SSteffen Klassert if (xo->flags & XFRM_GSO_SEGMENT) {
3540146dca7SSabrina Dubroca esp.esph->seq_no = htonl(seq);
3553dca3f38SSteffen Klassert
3563dca3f38SSteffen Klassert if (!skb_is_gso(skb))
3573dca3f38SSteffen Klassert xo->seq.low++;
3583dca3f38SSteffen Klassert else
3593dca3f38SSteffen Klassert xo->seq.low += skb_shinfo(skb)->gso_segs;
3603dca3f38SSteffen Klassert }
3613dca3f38SSteffen Klassert
3624b549cccSChristian Langrock if (xo->seq.low < seq)
3634b549cccSChristian Langrock xo->seq.hi++;
3644b549cccSChristian Langrock
3653dca3f38SSteffen Klassert esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));
366383d0350SSteffen Klassert
367383d0350SSteffen Klassert len = skb->len - sizeof(struct ipv6hdr);
368383d0350SSteffen Klassert if (len > IPV6_MAXPLEN)
369383d0350SSteffen Klassert len = 0;
370383d0350SSteffen Klassert
371383d0350SSteffen Klassert ipv6_hdr(skb)->payload_len = htons(len);
372383d0350SSteffen Klassert
373c7dbf4c0SSteffen Klassert if (hw_offload) {
374c7dbf4c0SSteffen Klassert if (!skb_ext_add(skb, SKB_EXT_SEC_PATH))
375c7dbf4c0SSteffen Klassert return -ENOMEM;
376c7dbf4c0SSteffen Klassert
377c7dbf4c0SSteffen Klassert xo = xfrm_offload(skb);
378c7dbf4c0SSteffen Klassert if (!xo)
379c7dbf4c0SSteffen Klassert return -EINVAL;
380c7dbf4c0SSteffen Klassert
381c7dbf4c0SSteffen Klassert xo->flags |= XFRM_XMIT;
382383d0350SSteffen Klassert return 0;
383c7dbf4c0SSteffen Klassert }
384383d0350SSteffen Klassert
385383d0350SSteffen Klassert err = esp6_output_tail(x, skb, &esp);
3864ff0308fSSteffen Klassert if (err)
387383d0350SSteffen Klassert return err;
388383d0350SSteffen Klassert
389383d0350SSteffen Klassert secpath_reset(skb);
390383d0350SSteffen Klassert
391f015b900SSebastian Andrzej Siewior if (skb_needs_linearize(skb, skb->dev->features) &&
392f015b900SSebastian Andrzej Siewior __skb_linearize(skb))
393f015b900SSebastian Andrzej Siewior return -ENOMEM;
394383d0350SSteffen Klassert return 0;
395383d0350SSteffen Klassert }
396383d0350SSteffen Klassert
3977785bba2SSteffen Klassert static const struct net_offload esp6_offload = {
3987785bba2SSteffen Klassert .callbacks = {
3997785bba2SSteffen Klassert .gro_receive = esp6_gro_receive,
4007862b405SSteffen Klassert .gso_segment = esp6_gso_segment,
4017785bba2SSteffen Klassert },
4027785bba2SSteffen Klassert };
4037785bba2SSteffen Klassert
404383d0350SSteffen Klassert static const struct xfrm_type_offload esp6_type_offload = {
405383d0350SSteffen Klassert .owner = THIS_MODULE,
406383d0350SSteffen Klassert .proto = IPPROTO_ESP,
407383d0350SSteffen Klassert .input_tail = esp6_input_tail,
408383d0350SSteffen Klassert .xmit = esp6_xmit,
4097862b405SSteffen Klassert .encap = esp6_gso_encap,
410383d0350SSteffen Klassert };
411383d0350SSteffen Klassert
esp6_offload_init(void)4127785bba2SSteffen Klassert static int __init esp6_offload_init(void)
4137785bba2SSteffen Klassert {
414383d0350SSteffen Klassert if (xfrm_register_type_offload(&esp6_type_offload, AF_INET6) < 0) {
415383d0350SSteffen Klassert pr_info("%s: can't add xfrm type offload\n", __func__);
416383d0350SSteffen Klassert return -EAGAIN;
417383d0350SSteffen Klassert }
418383d0350SSteffen Klassert
4197785bba2SSteffen Klassert return inet6_add_offload(&esp6_offload, IPPROTO_ESP);
4207785bba2SSteffen Klassert }
4217785bba2SSteffen Klassert
esp6_offload_exit(void)4227785bba2SSteffen Klassert static void __exit esp6_offload_exit(void)
4237785bba2SSteffen Klassert {
4244f518e80SFlorian Westphal xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6);
4257785bba2SSteffen Klassert inet6_del_offload(&esp6_offload, IPPROTO_ESP);
4267785bba2SSteffen Klassert }
4277785bba2SSteffen Klassert
4287785bba2SSteffen Klassert module_init(esp6_offload_init);
4297785bba2SSteffen Klassert module_exit(esp6_offload_exit);
4307785bba2SSteffen Klassert MODULE_LICENSE("GPL");
4317785bba2SSteffen Klassert MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
432ffdb5211SIlan Tayari MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP);
43367c20de3SRob Gill MODULE_DESCRIPTION("IPV6 GSO/GRO offload support");
434