1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * IPV6 GSO/GRO offload support 4 * Linux INET6 implementation 5 * 6 * TCPv6 GSO/GRO support 7 */ 8 #include <linux/indirect_call_wrapper.h> 9 #include <linux/skbuff.h> 10 #include <net/inet6_hashtables.h> 11 #include <net/gro.h> 12 #include <net/protocol.h> 13 #include <net/tcp.h> 14 #include <net/ip6_checksum.h> 15 #include "ip6_offload.h" 16 17 static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, 18 struct tcphdr *th) 19 { 20 #if IS_ENABLED(CONFIG_IPV6) 21 const struct ipv6hdr *hdr; 22 struct sk_buff *p; 23 struct sock *sk; 24 struct net *net; 25 int iif, sdif; 26 27 p = tcp_gro_lookup(head, th); 28 if (p) { 29 /* flist GRO applies to consecutive non-GSO skbs */ 30 if (!skb_is_gso(skb) || !NAPI_GRO_CB(p)->is_flist) { 31 NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; 32 return; 33 } 34 35 /* Fall back to the regular GRO path */ 36 if (NAPI_GRO_CB(p)->count == 1) 37 NAPI_GRO_CB(p)->is_flist = 0; 38 39 NAPI_GRO_CB(skb)->is_flist = 0; 40 41 return; 42 } 43 44 inet6_get_iif_sdif(skb, &iif, &sdif); 45 hdr = skb_gro_network_header(skb); 46 net = dev_net_rcu(skb->dev); 47 sk = __inet6_lookup_established(net, &hdr->saddr, th->source, 48 &hdr->daddr, ntohs(th->dest), 49 iif, sdif); 50 NAPI_GRO_CB(skb)->is_flist = !sk && !skb_is_gso(skb); 51 if (sk) 52 sock_gen_put(sk); 53 #endif /* IS_ENABLED(CONFIG_IPV6) */ 54 } 55 56 static __always_inline struct sk_buff *tcp6_gro_receive(struct list_head *head, 57 struct sk_buff *skb) 58 { 59 struct tcphdr *th; 60 61 /* Don't bother verifying checksum if we're going to flush anyway. */ 62 if (!NAPI_GRO_CB(skb)->flush && 63 skb_gro_checksum_validate(skb, IPPROTO_TCP, 64 ip6_gro_compute_pseudo)) 65 goto flush; 66 67 th = tcp_gro_pull_header(skb); 68 if (!th) 69 goto flush; 70 71 if (unlikely(skb->dev->features & NETIF_F_GRO_FRAGLIST)) 72 tcp6_check_fraglist_gro(head, skb, th); 73 74 return tcp_gro_receive(head, skb, th); 75 76 flush: 77 NAPI_GRO_CB(skb)->flush = 1; 78 return NULL; 79 } 80 81 static __always_inline int tcp6_gro_complete(struct sk_buff *skb, int thoff) 82 { 83 const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation]; 84 const struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + offset); 85 struct tcphdr *th = tcp_hdr(skb); 86 87 if (unlikely(NAPI_GRO_CB(skb)->is_flist)) { 88 skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6; 89 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; 90 91 __skb_incr_checksum_unnecessary(skb); 92 93 return 0; 94 } 95 96 th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr, 97 &iph->daddr, 0); 98 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6; 99 100 tcp_gro_complete(skb); 101 return 0; 102 } 103 104 static void __tcpv6_gso_segment_csum(struct sk_buff *seg, 105 struct in6_addr *oldip, 106 const struct in6_addr *newip, 107 __be16 *oldport, __be16 newport) 108 { 109 struct tcphdr *th = tcp_hdr(seg); 110 111 if (!ipv6_addr_equal(oldip, newip)) { 112 inet_proto_csum_replace16(&th->check, seg, 113 oldip->s6_addr32, 114 newip->s6_addr32, 115 true); 116 *oldip = *newip; 117 } 118 119 if (*oldport == newport) 120 return; 121 122 inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false); 123 *oldport = newport; 124 } 125 126 static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs) 127 { 128 const struct tcphdr *th; 129 const struct ipv6hdr *iph; 130 struct sk_buff *seg; 131 struct tcphdr *th2; 132 struct ipv6hdr *iph2; 133 134 seg = segs; 135 th = tcp_hdr(seg); 136 iph = ipv6_hdr(seg); 137 th2 = tcp_hdr(seg->next); 138 iph2 = ipv6_hdr(seg->next); 139 140 if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) && 141 ipv6_addr_equal(&iph->saddr, &iph2->saddr) && 142 ipv6_addr_equal(&iph->daddr, &iph2->daddr)) 143 return segs; 144 145 while ((seg = seg->next)) { 146 th2 = tcp_hdr(seg); 147 iph2 = ipv6_hdr(seg); 148 149 __tcpv6_gso_segment_csum(seg, &iph2->saddr, &iph->saddr, 150 &th2->source, th->source); 151 __tcpv6_gso_segment_csum(seg, &iph2->daddr, &iph->daddr, 152 &th2->dest, th->dest); 153 } 154 155 return segs; 156 } 157 158 static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb, 159 netdev_features_t features) 160 { 161 skb = skb_segment_list(skb, features, skb_mac_header_len(skb)); 162 if (IS_ERR(skb)) 163 return skb; 164 165 return __tcpv6_gso_segment_list_csum(skb); 166 } 167 168 static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb, 169 netdev_features_t features) 170 { 171 struct tcphdr *th; 172 173 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)) 174 return ERR_PTR(-EINVAL); 175 176 if (!pskb_may_pull(skb, sizeof(*th))) 177 return ERR_PTR(-EINVAL); 178 179 if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) { 180 struct tcphdr *th = tcp_hdr(skb); 181 182 if ((skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size) && 183 !(skb_shinfo(skb)->gso_type & SKB_GSO_DODGY)) 184 return __tcp6_gso_segment_list(skb, features); 185 186 skb->ip_summed = CHECKSUM_NONE; 187 } 188 189 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) { 190 const struct ipv6hdr *ipv6h = ipv6_hdr(skb); 191 struct tcphdr *th = tcp_hdr(skb); 192 193 /* Set up pseudo header, usually expect stack to have done 194 * this. 195 */ 196 197 th->check = 0; 198 skb->ip_summed = CHECKSUM_PARTIAL; 199 __tcp_v6_send_check(skb, &ipv6h->saddr, &ipv6h->daddr); 200 } 201 202 return tcp_gso_segment(skb, features); 203 } 204 205 int __init tcpv6_offload_init(void) 206 { 207 net_hotdata.tcpv6_offload = (struct net_offload) { 208 .callbacks = { 209 .gso_segment = tcp6_gso_segment, 210 .gro_receive = tcp6_gro_receive, 211 .gro_complete = tcp6_gro_complete, 212 }, 213 }; 214 return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP); 215 } 216