1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * The Internet Protocol (IP) output module. 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Donald Becker, <becker@super.org> 12 * Alan Cox, <Alan.Cox@linux.org> 13 * Richard Underwood 14 * Stefan Becker, <stefanb@yello.ping.de> 15 * Jorge Cwik, <jorge@laser.satlink.net> 16 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 17 * Hirokazu Takahashi, <taka@valinux.co.jp> 18 * 19 * See ip_input.c for original log 20 * 21 * Fixes: 22 * Alan Cox : Missing nonblock feature in ip_build_xmit. 23 * Mike Kilburn : htons() missing in ip_build_xmit. 24 * Bradford Johnson: Fix faulty handling of some frames when 25 * no route is found. 26 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit 27 * (in case if packet not accepted by 28 * output firewall rules) 29 * Mike McLagan : Routing by source 30 * Alexey Kuznetsov: use new route cache 31 * Andi Kleen: Fix broken PMTU recovery and remove 32 * some redundant tests. 33 * Vitaly E. Lavrov : Transparent proxy revived after year coma. 34 * Andi Kleen : Replace ip_reply with ip_send_reply. 35 * Andi Kleen : Split fast and slow ip_build_xmit path 36 * for decreased register pressure on x86 37 * and more readability. 38 * Marc Boucher : When call_out_firewall returns FW_QUEUE, 39 * silently drop skb instead of failing with -EPERM. 40 * Detlev Wengorz : Copy protocol for fragments. 41 * Hirokazu Takahashi: HW checksumming for outgoing UDP 42 * datagrams. 43 * Hirokazu Takahashi: sendfile() on UDP works now. 44 */ 45 46 #include <linux/uaccess.h> 47 #include <linux/module.h> 48 #include <linux/types.h> 49 #include <linux/kernel.h> 50 #include <linux/mm.h> 51 #include <linux/string.h> 52 #include <linux/errno.h> 53 #include <linux/highmem.h> 54 #include <linux/slab.h> 55 56 #include <linux/socket.h> 57 #include <linux/sockios.h> 58 #include <linux/in.h> 59 #include <linux/inet.h> 60 #include <linux/netdevice.h> 61 #include <linux/etherdevice.h> 62 #include <linux/proc_fs.h> 63 #include <linux/stat.h> 64 #include <linux/init.h> 65 66 #include <net/snmp.h> 67 #include <net/ip.h> 68 #include <net/protocol.h> 69 #include <net/route.h> 70 #include <net/xfrm.h> 71 #include <linux/skbuff.h> 72 #include <net/sock.h> 73 #include <net/arp.h> 74 #include <net/icmp.h> 75 #include <net/checksum.h> 76 #include <net/inetpeer.h> 77 #include <net/inet_ecn.h> 78 #include <net/lwtunnel.h> 79 #include <linux/bpf-cgroup.h> 80 #include <linux/igmp.h> 81 #include <linux/netfilter_ipv4.h> 82 #include <linux/netfilter_bridge.h> 83 #include <linux/netlink.h> 84 #include <linux/tcp.h> 85 86 static int 87 ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 88 unsigned int mtu, 89 int (*output)(struct net *, struct sock *, struct sk_buff *)); 90 91 /* Generate a checksum for an outgoing IP datagram. */ 92 void ip_send_check(struct iphdr *iph) 93 { 94 iph->check = 0; 95 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 96 } 97 EXPORT_SYMBOL(ip_send_check); 98 99 int __ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb) 100 { 101 struct iphdr *iph = ip_hdr(skb); 102 103 iph->tot_len = htons(skb->len); 104 ip_send_check(iph); 105 106 /* if egress device is enslaved to an L3 master device pass the 107 * skb to its handler for processing 108 */ 109 skb = l3mdev_ip_out(sk, skb); 110 if (unlikely(!skb)) 111 return 0; 112 113 skb->protocol = htons(ETH_P_IP); 114 115 return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, 116 net, sk, skb, NULL, skb_dst(skb)->dev, 117 dst_output); 118 } 119 120 int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb) 121 { 122 int err; 123 124 err = __ip_local_out(net, sk, skb); 125 if (likely(err == 1)) 126 err = dst_output(net, sk, skb); 127 128 return err; 129 } 130 EXPORT_SYMBOL_GPL(ip_local_out); 131 132 static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst) 133 { 134 int ttl = inet->uc_ttl; 135 136 if (ttl < 0) 137 ttl = ip4_dst_hoplimit(dst); 138 return ttl; 139 } 140 141 /* 142 * Add an ip header to a skbuff and send it out. 143 * 144 */ 145 int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk, 146 __be32 saddr, __be32 daddr, struct ip_options_rcu *opt, 147 u8 tos) 148 { 149 struct inet_sock *inet = inet_sk(sk); 150 struct rtable *rt = skb_rtable(skb); 151 struct net *net = sock_net(sk); 152 struct iphdr *iph; 153 154 /* Build the IP header. */ 155 skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0)); 156 skb_reset_network_header(skb); 157 iph = ip_hdr(skb); 158 iph->version = 4; 159 iph->ihl = 5; 160 iph->tos = tos; 161 iph->ttl = ip_select_ttl(inet, &rt->dst); 162 iph->daddr = (opt && opt->opt.srr ? opt->opt.faddr : daddr); 163 iph->saddr = saddr; 164 iph->protocol = sk->sk_protocol; 165 /* Do not bother generating IPID for small packets (eg SYNACK) */ 166 if (skb->len <= IPV4_MIN_MTU || ip_dont_fragment(sk, &rt->dst)) { 167 iph->frag_off = htons(IP_DF); 168 iph->id = 0; 169 } else { 170 iph->frag_off = 0; 171 /* TCP packets here are SYNACK with fat IPv4/TCP options. 172 * Avoid using the hashed IP ident generator. 173 */ 174 if (sk->sk_protocol == IPPROTO_TCP) 175 iph->id = (__force __be16)prandom_u32(); 176 else 177 __ip_select_ident(net, iph, 1); 178 } 179 180 if (opt && opt->opt.optlen) { 181 iph->ihl += opt->opt.optlen>>2; 182 ip_options_build(skb, &opt->opt, daddr, rt, 0); 183 } 184 185 skb->priority = sk->sk_priority; 186 if (!skb->mark) 187 skb->mark = sk->sk_mark; 188 189 /* Send it out. */ 190 return ip_local_out(net, skb->sk, skb); 191 } 192 EXPORT_SYMBOL_GPL(ip_build_and_send_pkt); 193 194 static int ip_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) 195 { 196 struct dst_entry *dst = skb_dst(skb); 197 struct rtable *rt = (struct rtable *)dst; 198 struct net_device *dev = dst->dev; 199 unsigned int hh_len = LL_RESERVED_SPACE(dev); 200 struct neighbour *neigh; 201 bool is_v6gw = false; 202 203 if (rt->rt_type == RTN_MULTICAST) { 204 IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTMCAST, skb->len); 205 } else if (rt->rt_type == RTN_BROADCAST) 206 IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTBCAST, skb->len); 207 208 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { 209 skb = skb_expand_head(skb, hh_len); 210 if (!skb) 211 return -ENOMEM; 212 } 213 214 if (lwtunnel_xmit_redirect(dst->lwtstate)) { 215 int res = lwtunnel_xmit(skb); 216 217 if (res < 0 || res == LWTUNNEL_XMIT_DONE) 218 return res; 219 } 220 221 rcu_read_lock_bh(); 222 neigh = ip_neigh_for_gw(rt, skb, &is_v6gw); 223 if (!IS_ERR(neigh)) { 224 int res; 225 226 sock_confirm_neigh(skb, neigh); 227 /* if crossing protocols, can not use the cached header */ 228 res = neigh_output(neigh, skb, is_v6gw); 229 rcu_read_unlock_bh(); 230 return res; 231 } 232 rcu_read_unlock_bh(); 233 234 net_dbg_ratelimited("%s: No header cache and no neighbour!\n", 235 __func__); 236 kfree_skb(skb); 237 return -EINVAL; 238 } 239 240 static int ip_finish_output_gso(struct net *net, struct sock *sk, 241 struct sk_buff *skb, unsigned int mtu) 242 { 243 struct sk_buff *segs, *nskb; 244 netdev_features_t features; 245 int ret = 0; 246 247 /* common case: seglen is <= mtu 248 */ 249 if (skb_gso_validate_network_len(skb, mtu)) 250 return ip_finish_output2(net, sk, skb); 251 252 /* Slowpath - GSO segment length exceeds the egress MTU. 253 * 254 * This can happen in several cases: 255 * - Forwarding of a TCP GRO skb, when DF flag is not set. 256 * - Forwarding of an skb that arrived on a virtualization interface 257 * (virtio-net/vhost/tap) with TSO/GSO size set by other network 258 * stack. 259 * - Local GSO skb transmitted on an NETIF_F_TSO tunnel stacked over an 260 * interface with a smaller MTU. 261 * - Arriving GRO skb (or GSO skb in a virtualized environment) that is 262 * bridged to a NETIF_F_TSO tunnel stacked over an interface with an 263 * insufficient MTU. 264 */ 265 features = netif_skb_features(skb); 266 BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET); 267 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 268 if (IS_ERR_OR_NULL(segs)) { 269 kfree_skb(skb); 270 return -ENOMEM; 271 } 272 273 consume_skb(skb); 274 275 skb_list_walk_safe(segs, segs, nskb) { 276 int err; 277 278 skb_mark_not_on_list(segs); 279 err = ip_fragment(net, sk, segs, mtu, ip_finish_output2); 280 281 if (err && ret == 0) 282 ret = err; 283 } 284 285 return ret; 286 } 287 288 static int __ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 289 { 290 unsigned int mtu; 291 292 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM) 293 /* Policy lookup after SNAT yielded a new policy */ 294 if (skb_dst(skb)->xfrm) { 295 IPCB(skb)->flags |= IPSKB_REROUTED; 296 return dst_output(net, sk, skb); 297 } 298 #endif 299 mtu = ip_skb_dst_mtu(sk, skb); 300 if (skb_is_gso(skb)) 301 return ip_finish_output_gso(net, sk, skb, mtu); 302 303 if (skb->len > mtu || IPCB(skb)->frag_max_size) 304 return ip_fragment(net, sk, skb, mtu, ip_finish_output2); 305 306 return ip_finish_output2(net, sk, skb); 307 } 308 309 static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 310 { 311 int ret; 312 313 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); 314 switch (ret) { 315 case NET_XMIT_SUCCESS: 316 return __ip_finish_output(net, sk, skb); 317 case NET_XMIT_CN: 318 return __ip_finish_output(net, sk, skb) ? : ret; 319 default: 320 kfree_skb(skb); 321 return ret; 322 } 323 } 324 325 static int ip_mc_finish_output(struct net *net, struct sock *sk, 326 struct sk_buff *skb) 327 { 328 struct rtable *new_rt; 329 bool do_cn = false; 330 int ret, err; 331 332 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); 333 switch (ret) { 334 case NET_XMIT_CN: 335 do_cn = true; 336 fallthrough; 337 case NET_XMIT_SUCCESS: 338 break; 339 default: 340 kfree_skb(skb); 341 return ret; 342 } 343 344 /* Reset rt_iif so that inet_iif() will return skb->skb_iif. Setting 345 * this to non-zero causes ipi_ifindex in in_pktinfo to be overwritten, 346 * see ipv4_pktinfo_prepare(). 347 */ 348 new_rt = rt_dst_clone(net->loopback_dev, skb_rtable(skb)); 349 if (new_rt) { 350 new_rt->rt_iif = 0; 351 skb_dst_drop(skb); 352 skb_dst_set(skb, &new_rt->dst); 353 } 354 355 err = dev_loopback_xmit(net, sk, skb); 356 return (do_cn && err) ? ret : err; 357 } 358 359 int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb) 360 { 361 struct rtable *rt = skb_rtable(skb); 362 struct net_device *dev = rt->dst.dev; 363 364 /* 365 * If the indicated interface is up and running, send the packet. 366 */ 367 IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len); 368 369 skb->dev = dev; 370 skb->protocol = htons(ETH_P_IP); 371 372 /* 373 * Multicasts are looped back for other local users 374 */ 375 376 if (rt->rt_flags&RTCF_MULTICAST) { 377 if (sk_mc_loop(sk) 378 #ifdef CONFIG_IP_MROUTE 379 /* Small optimization: do not loopback not local frames, 380 which returned after forwarding; they will be dropped 381 by ip_mr_input in any case. 382 Note, that local frames are looped back to be delivered 383 to local recipients. 384 385 This check is duplicated in ip_mr_input at the moment. 386 */ 387 && 388 ((rt->rt_flags & RTCF_LOCAL) || 389 !(IPCB(skb)->flags & IPSKB_FORWARDED)) 390 #endif 391 ) { 392 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 393 if (newskb) 394 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, 395 net, sk, newskb, NULL, newskb->dev, 396 ip_mc_finish_output); 397 } 398 399 /* Multicasts with ttl 0 must not go beyond the host */ 400 401 if (ip_hdr(skb)->ttl == 0) { 402 kfree_skb(skb); 403 return 0; 404 } 405 } 406 407 if (rt->rt_flags&RTCF_BROADCAST) { 408 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 409 if (newskb) 410 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, 411 net, sk, newskb, NULL, newskb->dev, 412 ip_mc_finish_output); 413 } 414 415 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, 416 net, sk, skb, NULL, skb->dev, 417 ip_finish_output, 418 !(IPCB(skb)->flags & IPSKB_REROUTED)); 419 } 420 421 int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb) 422 { 423 struct net_device *dev = skb_dst(skb)->dev, *indev = skb->dev; 424 425 IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len); 426 427 skb->dev = dev; 428 skb->protocol = htons(ETH_P_IP); 429 430 return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, 431 net, sk, skb, indev, dev, 432 ip_finish_output, 433 !(IPCB(skb)->flags & IPSKB_REROUTED)); 434 } 435 EXPORT_SYMBOL(ip_output); 436 437 /* 438 * copy saddr and daddr, possibly using 64bit load/stores 439 * Equivalent to : 440 * iph->saddr = fl4->saddr; 441 * iph->daddr = fl4->daddr; 442 */ 443 static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4) 444 { 445 BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) != 446 offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr)); 447 448 iph->saddr = fl4->saddr; 449 iph->daddr = fl4->daddr; 450 } 451 452 /* Note: skb->sk can be different from sk, in case of tunnels */ 453 int __ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl, 454 __u8 tos) 455 { 456 struct inet_sock *inet = inet_sk(sk); 457 struct net *net = sock_net(sk); 458 struct ip_options_rcu *inet_opt; 459 struct flowi4 *fl4; 460 struct rtable *rt; 461 struct iphdr *iph; 462 int res; 463 464 /* Skip all of this if the packet is already routed, 465 * f.e. by something like SCTP. 466 */ 467 rcu_read_lock(); 468 inet_opt = rcu_dereference(inet->inet_opt); 469 fl4 = &fl->u.ip4; 470 rt = skb_rtable(skb); 471 if (rt) 472 goto packet_routed; 473 474 /* Make sure we can route this packet. */ 475 rt = (struct rtable *)__sk_dst_check(sk, 0); 476 if (!rt) { 477 __be32 daddr; 478 479 /* Use correct destination address if we have options. */ 480 daddr = inet->inet_daddr; 481 if (inet_opt && inet_opt->opt.srr) 482 daddr = inet_opt->opt.faddr; 483 484 /* If this fails, retransmit mechanism of transport layer will 485 * keep trying until route appears or the connection times 486 * itself out. 487 */ 488 rt = ip_route_output_ports(net, fl4, sk, 489 daddr, inet->inet_saddr, 490 inet->inet_dport, 491 inet->inet_sport, 492 sk->sk_protocol, 493 RT_CONN_FLAGS_TOS(sk, tos), 494 sk->sk_bound_dev_if); 495 if (IS_ERR(rt)) 496 goto no_route; 497 sk_setup_caps(sk, &rt->dst); 498 } 499 skb_dst_set_noref(skb, &rt->dst); 500 501 packet_routed: 502 if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway) 503 goto no_route; 504 505 /* OK, we know where to send it, allocate and build IP header. */ 506 skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0)); 507 skb_reset_network_header(skb); 508 iph = ip_hdr(skb); 509 *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (tos & 0xff)); 510 if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df) 511 iph->frag_off = htons(IP_DF); 512 else 513 iph->frag_off = 0; 514 iph->ttl = ip_select_ttl(inet, &rt->dst); 515 iph->protocol = sk->sk_protocol; 516 ip_copy_addrs(iph, fl4); 517 518 /* Transport layer set skb->h.foo itself. */ 519 520 if (inet_opt && inet_opt->opt.optlen) { 521 iph->ihl += inet_opt->opt.optlen >> 2; 522 ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0); 523 } 524 525 ip_select_ident_segs(net, skb, sk, 526 skb_shinfo(skb)->gso_segs ?: 1); 527 528 /* TODO : should we use skb->sk here instead of sk ? */ 529 skb->priority = sk->sk_priority; 530 skb->mark = sk->sk_mark; 531 532 res = ip_local_out(net, sk, skb); 533 rcu_read_unlock(); 534 return res; 535 536 no_route: 537 rcu_read_unlock(); 538 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 539 kfree_skb(skb); 540 return -EHOSTUNREACH; 541 } 542 EXPORT_SYMBOL(__ip_queue_xmit); 543 544 int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl) 545 { 546 return __ip_queue_xmit(sk, skb, fl, inet_sk(sk)->tos); 547 } 548 EXPORT_SYMBOL(ip_queue_xmit); 549 550 static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from) 551 { 552 to->pkt_type = from->pkt_type; 553 to->priority = from->priority; 554 to->protocol = from->protocol; 555 to->skb_iif = from->skb_iif; 556 skb_dst_drop(to); 557 skb_dst_copy(to, from); 558 to->dev = from->dev; 559 to->mark = from->mark; 560 561 skb_copy_hash(to, from); 562 563 #ifdef CONFIG_NET_SCHED 564 to->tc_index = from->tc_index; 565 #endif 566 nf_copy(to, from); 567 skb_ext_copy(to, from); 568 #if IS_ENABLED(CONFIG_IP_VS) 569 to->ipvs_property = from->ipvs_property; 570 #endif 571 skb_copy_secmark(to, from); 572 } 573 574 static int ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 575 unsigned int mtu, 576 int (*output)(struct net *, struct sock *, struct sk_buff *)) 577 { 578 struct iphdr *iph = ip_hdr(skb); 579 580 if ((iph->frag_off & htons(IP_DF)) == 0) 581 return ip_do_fragment(net, sk, skb, output); 582 583 if (unlikely(!skb->ignore_df || 584 (IPCB(skb)->frag_max_size && 585 IPCB(skb)->frag_max_size > mtu))) { 586 IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS); 587 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, 588 htonl(mtu)); 589 kfree_skb(skb); 590 return -EMSGSIZE; 591 } 592 593 return ip_do_fragment(net, sk, skb, output); 594 } 595 596 void ip_fraglist_init(struct sk_buff *skb, struct iphdr *iph, 597 unsigned int hlen, struct ip_fraglist_iter *iter) 598 { 599 unsigned int first_len = skb_pagelen(skb); 600 601 iter->frag = skb_shinfo(skb)->frag_list; 602 skb_frag_list_init(skb); 603 604 iter->offset = 0; 605 iter->iph = iph; 606 iter->hlen = hlen; 607 608 skb->data_len = first_len - skb_headlen(skb); 609 skb->len = first_len; 610 iph->tot_len = htons(first_len); 611 iph->frag_off = htons(IP_MF); 612 ip_send_check(iph); 613 } 614 EXPORT_SYMBOL(ip_fraglist_init); 615 616 void ip_fraglist_prepare(struct sk_buff *skb, struct ip_fraglist_iter *iter) 617 { 618 unsigned int hlen = iter->hlen; 619 struct iphdr *iph = iter->iph; 620 struct sk_buff *frag; 621 622 frag = iter->frag; 623 frag->ip_summed = CHECKSUM_NONE; 624 skb_reset_transport_header(frag); 625 __skb_push(frag, hlen); 626 skb_reset_network_header(frag); 627 memcpy(skb_network_header(frag), iph, hlen); 628 iter->iph = ip_hdr(frag); 629 iph = iter->iph; 630 iph->tot_len = htons(frag->len); 631 ip_copy_metadata(frag, skb); 632 iter->offset += skb->len - hlen; 633 iph->frag_off = htons(iter->offset >> 3); 634 if (frag->next) 635 iph->frag_off |= htons(IP_MF); 636 /* Ready, complete checksum */ 637 ip_send_check(iph); 638 } 639 EXPORT_SYMBOL(ip_fraglist_prepare); 640 641 void ip_frag_init(struct sk_buff *skb, unsigned int hlen, 642 unsigned int ll_rs, unsigned int mtu, bool DF, 643 struct ip_frag_state *state) 644 { 645 struct iphdr *iph = ip_hdr(skb); 646 647 state->DF = DF; 648 state->hlen = hlen; 649 state->ll_rs = ll_rs; 650 state->mtu = mtu; 651 652 state->left = skb->len - hlen; /* Space per frame */ 653 state->ptr = hlen; /* Where to start from */ 654 655 state->offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3; 656 state->not_last_frag = iph->frag_off & htons(IP_MF); 657 } 658 EXPORT_SYMBOL(ip_frag_init); 659 660 static void ip_frag_ipcb(struct sk_buff *from, struct sk_buff *to, 661 bool first_frag) 662 { 663 /* Copy the flags to each fragment. */ 664 IPCB(to)->flags = IPCB(from)->flags; 665 666 /* ANK: dirty, but effective trick. Upgrade options only if 667 * the segment to be fragmented was THE FIRST (otherwise, 668 * options are already fixed) and make it ONCE 669 * on the initial skb, so that all the following fragments 670 * will inherit fixed options. 671 */ 672 if (first_frag) 673 ip_options_fragment(from); 674 } 675 676 struct sk_buff *ip_frag_next(struct sk_buff *skb, struct ip_frag_state *state) 677 { 678 unsigned int len = state->left; 679 struct sk_buff *skb2; 680 struct iphdr *iph; 681 682 /* IF: it doesn't fit, use 'mtu' - the data space left */ 683 if (len > state->mtu) 684 len = state->mtu; 685 /* IF: we are not sending up to and including the packet end 686 then align the next start on an eight byte boundary */ 687 if (len < state->left) { 688 len &= ~7; 689 } 690 691 /* Allocate buffer */ 692 skb2 = alloc_skb(len + state->hlen + state->ll_rs, GFP_ATOMIC); 693 if (!skb2) 694 return ERR_PTR(-ENOMEM); 695 696 /* 697 * Set up data on packet 698 */ 699 700 ip_copy_metadata(skb2, skb); 701 skb_reserve(skb2, state->ll_rs); 702 skb_put(skb2, len + state->hlen); 703 skb_reset_network_header(skb2); 704 skb2->transport_header = skb2->network_header + state->hlen; 705 706 /* 707 * Charge the memory for the fragment to any owner 708 * it might possess 709 */ 710 711 if (skb->sk) 712 skb_set_owner_w(skb2, skb->sk); 713 714 /* 715 * Copy the packet header into the new buffer. 716 */ 717 718 skb_copy_from_linear_data(skb, skb_network_header(skb2), state->hlen); 719 720 /* 721 * Copy a block of the IP datagram. 722 */ 723 if (skb_copy_bits(skb, state->ptr, skb_transport_header(skb2), len)) 724 BUG(); 725 state->left -= len; 726 727 /* 728 * Fill in the new header fields. 729 */ 730 iph = ip_hdr(skb2); 731 iph->frag_off = htons((state->offset >> 3)); 732 if (state->DF) 733 iph->frag_off |= htons(IP_DF); 734 735 /* 736 * Added AC : If we are fragmenting a fragment that's not the 737 * last fragment then keep MF on each bit 738 */ 739 if (state->left > 0 || state->not_last_frag) 740 iph->frag_off |= htons(IP_MF); 741 state->ptr += len; 742 state->offset += len; 743 744 iph->tot_len = htons(len + state->hlen); 745 746 ip_send_check(iph); 747 748 return skb2; 749 } 750 EXPORT_SYMBOL(ip_frag_next); 751 752 /* 753 * This IP datagram is too large to be sent in one piece. Break it up into 754 * smaller pieces (each of size equal to IP header plus 755 * a block of the data of the original IP data part) that will yet fit in a 756 * single device frame, and queue such a frame for sending. 757 */ 758 759 int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 760 int (*output)(struct net *, struct sock *, struct sk_buff *)) 761 { 762 struct iphdr *iph; 763 struct sk_buff *skb2; 764 struct rtable *rt = skb_rtable(skb); 765 unsigned int mtu, hlen, ll_rs; 766 struct ip_fraglist_iter iter; 767 ktime_t tstamp = skb->tstamp; 768 struct ip_frag_state state; 769 int err = 0; 770 771 /* for offloaded checksums cleanup checksum before fragmentation */ 772 if (skb->ip_summed == CHECKSUM_PARTIAL && 773 (err = skb_checksum_help(skb))) 774 goto fail; 775 776 /* 777 * Point into the IP datagram header. 778 */ 779 780 iph = ip_hdr(skb); 781 782 mtu = ip_skb_dst_mtu(sk, skb); 783 if (IPCB(skb)->frag_max_size && IPCB(skb)->frag_max_size < mtu) 784 mtu = IPCB(skb)->frag_max_size; 785 786 /* 787 * Setup starting values. 788 */ 789 790 hlen = iph->ihl * 4; 791 mtu = mtu - hlen; /* Size of data space */ 792 IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE; 793 ll_rs = LL_RESERVED_SPACE(rt->dst.dev); 794 795 /* When frag_list is given, use it. First, check its validity: 796 * some transformers could create wrong frag_list or break existing 797 * one, it is not prohibited. In this case fall back to copying. 798 * 799 * LATER: this step can be merged to real generation of fragments, 800 * we can switch to copy when see the first bad fragment. 801 */ 802 if (skb_has_frag_list(skb)) { 803 struct sk_buff *frag, *frag2; 804 unsigned int first_len = skb_pagelen(skb); 805 806 if (first_len - hlen > mtu || 807 ((first_len - hlen) & 7) || 808 ip_is_fragment(iph) || 809 skb_cloned(skb) || 810 skb_headroom(skb) < ll_rs) 811 goto slow_path; 812 813 skb_walk_frags(skb, frag) { 814 /* Correct geometry. */ 815 if (frag->len > mtu || 816 ((frag->len & 7) && frag->next) || 817 skb_headroom(frag) < hlen + ll_rs) 818 goto slow_path_clean; 819 820 /* Partially cloned skb? */ 821 if (skb_shared(frag)) 822 goto slow_path_clean; 823 824 BUG_ON(frag->sk); 825 if (skb->sk) { 826 frag->sk = skb->sk; 827 frag->destructor = sock_wfree; 828 } 829 skb->truesize -= frag->truesize; 830 } 831 832 /* Everything is OK. Generate! */ 833 ip_fraglist_init(skb, iph, hlen, &iter); 834 835 for (;;) { 836 /* Prepare header of the next frame, 837 * before previous one went down. */ 838 if (iter.frag) { 839 bool first_frag = (iter.offset == 0); 840 841 IPCB(iter.frag)->flags = IPCB(skb)->flags; 842 ip_fraglist_prepare(skb, &iter); 843 if (first_frag && IPCB(skb)->opt.optlen) { 844 /* ipcb->opt is not populated for frags 845 * coming from __ip_make_skb(), 846 * ip_options_fragment() needs optlen 847 */ 848 IPCB(iter.frag)->opt.optlen = 849 IPCB(skb)->opt.optlen; 850 ip_options_fragment(iter.frag); 851 ip_send_check(iter.iph); 852 } 853 } 854 855 skb->tstamp = tstamp; 856 err = output(net, sk, skb); 857 858 if (!err) 859 IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES); 860 if (err || !iter.frag) 861 break; 862 863 skb = ip_fraglist_next(&iter); 864 } 865 866 if (err == 0) { 867 IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS); 868 return 0; 869 } 870 871 kfree_skb_list(iter.frag); 872 873 IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS); 874 return err; 875 876 slow_path_clean: 877 skb_walk_frags(skb, frag2) { 878 if (frag2 == frag) 879 break; 880 frag2->sk = NULL; 881 frag2->destructor = NULL; 882 skb->truesize += frag2->truesize; 883 } 884 } 885 886 slow_path: 887 /* 888 * Fragment the datagram. 889 */ 890 891 ip_frag_init(skb, hlen, ll_rs, mtu, IPCB(skb)->flags & IPSKB_FRAG_PMTU, 892 &state); 893 894 /* 895 * Keep copying data until we run out. 896 */ 897 898 while (state.left > 0) { 899 bool first_frag = (state.offset == 0); 900 901 skb2 = ip_frag_next(skb, &state); 902 if (IS_ERR(skb2)) { 903 err = PTR_ERR(skb2); 904 goto fail; 905 } 906 ip_frag_ipcb(skb, skb2, first_frag); 907 908 /* 909 * Put this fragment into the sending queue. 910 */ 911 skb2->tstamp = tstamp; 912 err = output(net, sk, skb2); 913 if (err) 914 goto fail; 915 916 IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES); 917 } 918 consume_skb(skb); 919 IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS); 920 return err; 921 922 fail: 923 kfree_skb(skb); 924 IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS); 925 return err; 926 } 927 EXPORT_SYMBOL(ip_do_fragment); 928 929 int 930 ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb) 931 { 932 struct msghdr *msg = from; 933 934 if (skb->ip_summed == CHECKSUM_PARTIAL) { 935 if (!copy_from_iter_full(to, len, &msg->msg_iter)) 936 return -EFAULT; 937 } else { 938 __wsum csum = 0; 939 if (!csum_and_copy_from_iter_full(to, len, &csum, &msg->msg_iter)) 940 return -EFAULT; 941 skb->csum = csum_block_add(skb->csum, csum, odd); 942 } 943 return 0; 944 } 945 EXPORT_SYMBOL(ip_generic_getfrag); 946 947 static inline __wsum 948 csum_page(struct page *page, int offset, int copy) 949 { 950 char *kaddr; 951 __wsum csum; 952 kaddr = kmap(page); 953 csum = csum_partial(kaddr + offset, copy, 0); 954 kunmap(page); 955 return csum; 956 } 957 958 static int __ip_append_data(struct sock *sk, 959 struct flowi4 *fl4, 960 struct sk_buff_head *queue, 961 struct inet_cork *cork, 962 struct page_frag *pfrag, 963 int getfrag(void *from, char *to, int offset, 964 int len, int odd, struct sk_buff *skb), 965 void *from, int length, int transhdrlen, 966 unsigned int flags) 967 { 968 struct inet_sock *inet = inet_sk(sk); 969 struct ubuf_info *uarg = NULL; 970 struct sk_buff *skb; 971 972 struct ip_options *opt = cork->opt; 973 int hh_len; 974 int exthdrlen; 975 int mtu; 976 int copy; 977 int err; 978 int offset = 0; 979 unsigned int maxfraglen, fragheaderlen, maxnonfragsize; 980 int csummode = CHECKSUM_NONE; 981 struct rtable *rt = (struct rtable *)cork->dst; 982 unsigned int wmem_alloc_delta = 0; 983 bool paged, extra_uref = false; 984 u32 tskey = 0; 985 986 skb = skb_peek_tail(queue); 987 988 exthdrlen = !skb ? rt->dst.header_len : 0; 989 mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize; 990 paged = !!cork->gso_size; 991 992 if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP && 993 sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) 994 tskey = sk->sk_tskey++; 995 996 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 997 998 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0); 999 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen; 1000 maxnonfragsize = ip_sk_ignore_df(sk) ? IP_MAX_MTU : mtu; 1001 1002 if (cork->length + length > maxnonfragsize - fragheaderlen) { 1003 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, 1004 mtu - (opt ? opt->optlen : 0)); 1005 return -EMSGSIZE; 1006 } 1007 1008 /* 1009 * transhdrlen > 0 means that this is the first fragment and we wish 1010 * it won't be fragmented in the future. 1011 */ 1012 if (transhdrlen && 1013 length + fragheaderlen <= mtu && 1014 rt->dst.dev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM) && 1015 (!(flags & MSG_MORE) || cork->gso_size) && 1016 (!exthdrlen || (rt->dst.dev->features & NETIF_F_HW_ESP_TX_CSUM))) 1017 csummode = CHECKSUM_PARTIAL; 1018 1019 if (flags & MSG_ZEROCOPY && length && sock_flag(sk, SOCK_ZEROCOPY)) { 1020 uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); 1021 if (!uarg) 1022 return -ENOBUFS; 1023 extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ 1024 if (rt->dst.dev->features & NETIF_F_SG && 1025 csummode == CHECKSUM_PARTIAL) { 1026 paged = true; 1027 } else { 1028 uarg->zerocopy = 0; 1029 skb_zcopy_set(skb, uarg, &extra_uref); 1030 } 1031 } 1032 1033 cork->length += length; 1034 1035 /* So, what's going on in the loop below? 1036 * 1037 * We use calculated fragment length to generate chained skb, 1038 * each of segments is IP fragment ready for sending to network after 1039 * adding appropriate IP header. 1040 */ 1041 1042 if (!skb) 1043 goto alloc_new_skb; 1044 1045 while (length > 0) { 1046 /* Check if the remaining data fits into current packet. */ 1047 copy = mtu - skb->len; 1048 if (copy < length) 1049 copy = maxfraglen - skb->len; 1050 if (copy <= 0) { 1051 char *data; 1052 unsigned int datalen; 1053 unsigned int fraglen; 1054 unsigned int fraggap; 1055 unsigned int alloclen, alloc_extra; 1056 unsigned int pagedlen; 1057 struct sk_buff *skb_prev; 1058 alloc_new_skb: 1059 skb_prev = skb; 1060 if (skb_prev) 1061 fraggap = skb_prev->len - maxfraglen; 1062 else 1063 fraggap = 0; 1064 1065 /* 1066 * If remaining data exceeds the mtu, 1067 * we know we need more fragment(s). 1068 */ 1069 datalen = length + fraggap; 1070 if (datalen > mtu - fragheaderlen) 1071 datalen = maxfraglen - fragheaderlen; 1072 fraglen = datalen + fragheaderlen; 1073 pagedlen = 0; 1074 1075 alloc_extra = hh_len + 15; 1076 alloc_extra += exthdrlen; 1077 1078 /* The last fragment gets additional space at tail. 1079 * Note, with MSG_MORE we overallocate on fragments, 1080 * because we have no idea what fragment will be 1081 * the last. 1082 */ 1083 if (datalen == length + fraggap) 1084 alloc_extra += rt->dst.trailer_len; 1085 1086 if ((flags & MSG_MORE) && 1087 !(rt->dst.dev->features&NETIF_F_SG)) 1088 alloclen = mtu; 1089 else if (!paged && 1090 (fraglen + alloc_extra < SKB_MAX_ALLOC || 1091 !(rt->dst.dev->features & NETIF_F_SG))) 1092 alloclen = fraglen; 1093 else { 1094 alloclen = min_t(int, fraglen, MAX_HEADER); 1095 pagedlen = fraglen - alloclen; 1096 } 1097 1098 alloclen += alloc_extra; 1099 1100 if (transhdrlen) { 1101 skb = sock_alloc_send_skb(sk, alloclen, 1102 (flags & MSG_DONTWAIT), &err); 1103 } else { 1104 skb = NULL; 1105 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <= 1106 2 * sk->sk_sndbuf) 1107 skb = alloc_skb(alloclen, 1108 sk->sk_allocation); 1109 if (unlikely(!skb)) 1110 err = -ENOBUFS; 1111 } 1112 if (!skb) 1113 goto error; 1114 1115 /* 1116 * Fill in the control structures 1117 */ 1118 skb->ip_summed = csummode; 1119 skb->csum = 0; 1120 skb_reserve(skb, hh_len); 1121 1122 /* 1123 * Find where to start putting bytes. 1124 */ 1125 data = skb_put(skb, fraglen + exthdrlen - pagedlen); 1126 skb_set_network_header(skb, exthdrlen); 1127 skb->transport_header = (skb->network_header + 1128 fragheaderlen); 1129 data += fragheaderlen + exthdrlen; 1130 1131 if (fraggap) { 1132 skb->csum = skb_copy_and_csum_bits( 1133 skb_prev, maxfraglen, 1134 data + transhdrlen, fraggap); 1135 skb_prev->csum = csum_sub(skb_prev->csum, 1136 skb->csum); 1137 data += fraggap; 1138 pskb_trim_unique(skb_prev, maxfraglen); 1139 } 1140 1141 copy = datalen - transhdrlen - fraggap - pagedlen; 1142 if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { 1143 err = -EFAULT; 1144 kfree_skb(skb); 1145 goto error; 1146 } 1147 1148 offset += copy; 1149 length -= copy + transhdrlen; 1150 transhdrlen = 0; 1151 exthdrlen = 0; 1152 csummode = CHECKSUM_NONE; 1153 1154 /* only the initial fragment is time stamped */ 1155 skb_shinfo(skb)->tx_flags = cork->tx_flags; 1156 cork->tx_flags = 0; 1157 skb_shinfo(skb)->tskey = tskey; 1158 tskey = 0; 1159 skb_zcopy_set(skb, uarg, &extra_uref); 1160 1161 if ((flags & MSG_CONFIRM) && !skb_prev) 1162 skb_set_dst_pending_confirm(skb, 1); 1163 1164 /* 1165 * Put the packet on the pending queue. 1166 */ 1167 if (!skb->destructor) { 1168 skb->destructor = sock_wfree; 1169 skb->sk = sk; 1170 wmem_alloc_delta += skb->truesize; 1171 } 1172 __skb_queue_tail(queue, skb); 1173 continue; 1174 } 1175 1176 if (copy > length) 1177 copy = length; 1178 1179 if (!(rt->dst.dev->features&NETIF_F_SG) && 1180 skb_tailroom(skb) >= copy) { 1181 unsigned int off; 1182 1183 off = skb->len; 1184 if (getfrag(from, skb_put(skb, copy), 1185 offset, copy, off, skb) < 0) { 1186 __skb_trim(skb, off); 1187 err = -EFAULT; 1188 goto error; 1189 } 1190 } else if (!uarg || !uarg->zerocopy) { 1191 int i = skb_shinfo(skb)->nr_frags; 1192 1193 err = -ENOMEM; 1194 if (!sk_page_frag_refill(sk, pfrag)) 1195 goto error; 1196 1197 if (!skb_can_coalesce(skb, i, pfrag->page, 1198 pfrag->offset)) { 1199 err = -EMSGSIZE; 1200 if (i == MAX_SKB_FRAGS) 1201 goto error; 1202 1203 __skb_fill_page_desc(skb, i, pfrag->page, 1204 pfrag->offset, 0); 1205 skb_shinfo(skb)->nr_frags = ++i; 1206 get_page(pfrag->page); 1207 } 1208 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1209 if (getfrag(from, 1210 page_address(pfrag->page) + pfrag->offset, 1211 offset, copy, skb->len, skb) < 0) 1212 goto error_efault; 1213 1214 pfrag->offset += copy; 1215 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1216 skb->len += copy; 1217 skb->data_len += copy; 1218 skb->truesize += copy; 1219 wmem_alloc_delta += copy; 1220 } else { 1221 err = skb_zerocopy_iter_dgram(skb, from, copy); 1222 if (err < 0) 1223 goto error; 1224 } 1225 offset += copy; 1226 length -= copy; 1227 } 1228 1229 if (wmem_alloc_delta) 1230 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1231 return 0; 1232 1233 error_efault: 1234 err = -EFAULT; 1235 error: 1236 net_zcopy_put_abort(uarg, extra_uref); 1237 cork->length -= length; 1238 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS); 1239 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1240 return err; 1241 } 1242 1243 static int ip_setup_cork(struct sock *sk, struct inet_cork *cork, 1244 struct ipcm_cookie *ipc, struct rtable **rtp) 1245 { 1246 struct ip_options_rcu *opt; 1247 struct rtable *rt; 1248 1249 rt = *rtp; 1250 if (unlikely(!rt)) 1251 return -EFAULT; 1252 1253 /* 1254 * setup for corking. 1255 */ 1256 opt = ipc->opt; 1257 if (opt) { 1258 if (!cork->opt) { 1259 cork->opt = kmalloc(sizeof(struct ip_options) + 40, 1260 sk->sk_allocation); 1261 if (unlikely(!cork->opt)) 1262 return -ENOBUFS; 1263 } 1264 memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen); 1265 cork->flags |= IPCORK_OPT; 1266 cork->addr = ipc->addr; 1267 } 1268 1269 cork->fragsize = ip_sk_use_pmtu(sk) ? 1270 dst_mtu(&rt->dst) : READ_ONCE(rt->dst.dev->mtu); 1271 1272 if (!inetdev_valid_mtu(cork->fragsize)) 1273 return -ENETUNREACH; 1274 1275 cork->gso_size = ipc->gso_size; 1276 1277 cork->dst = &rt->dst; 1278 /* We stole this route, caller should not release it. */ 1279 *rtp = NULL; 1280 1281 cork->length = 0; 1282 cork->ttl = ipc->ttl; 1283 cork->tos = ipc->tos; 1284 cork->mark = ipc->sockc.mark; 1285 cork->priority = ipc->priority; 1286 cork->transmit_time = ipc->sockc.transmit_time; 1287 cork->tx_flags = 0; 1288 sock_tx_timestamp(sk, ipc->sockc.tsflags, &cork->tx_flags); 1289 1290 return 0; 1291 } 1292 1293 /* 1294 * ip_append_data() and ip_append_page() can make one large IP datagram 1295 * from many pieces of data. Each pieces will be holded on the socket 1296 * until ip_push_pending_frames() is called. Each piece can be a page 1297 * or non-page data. 1298 * 1299 * Not only UDP, other transport protocols - e.g. raw sockets - can use 1300 * this interface potentially. 1301 * 1302 * LATER: length must be adjusted by pad at tail, when it is required. 1303 */ 1304 int ip_append_data(struct sock *sk, struct flowi4 *fl4, 1305 int getfrag(void *from, char *to, int offset, int len, 1306 int odd, struct sk_buff *skb), 1307 void *from, int length, int transhdrlen, 1308 struct ipcm_cookie *ipc, struct rtable **rtp, 1309 unsigned int flags) 1310 { 1311 struct inet_sock *inet = inet_sk(sk); 1312 int err; 1313 1314 if (flags&MSG_PROBE) 1315 return 0; 1316 1317 if (skb_queue_empty(&sk->sk_write_queue)) { 1318 err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp); 1319 if (err) 1320 return err; 1321 } else { 1322 transhdrlen = 0; 1323 } 1324 1325 return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base, 1326 sk_page_frag(sk), getfrag, 1327 from, length, transhdrlen, flags); 1328 } 1329 1330 ssize_t ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page, 1331 int offset, size_t size, int flags) 1332 { 1333 struct inet_sock *inet = inet_sk(sk); 1334 struct sk_buff *skb; 1335 struct rtable *rt; 1336 struct ip_options *opt = NULL; 1337 struct inet_cork *cork; 1338 int hh_len; 1339 int mtu; 1340 int len; 1341 int err; 1342 unsigned int maxfraglen, fragheaderlen, fraggap, maxnonfragsize; 1343 1344 if (inet->hdrincl) 1345 return -EPERM; 1346 1347 if (flags&MSG_PROBE) 1348 return 0; 1349 1350 if (skb_queue_empty(&sk->sk_write_queue)) 1351 return -EINVAL; 1352 1353 cork = &inet->cork.base; 1354 rt = (struct rtable *)cork->dst; 1355 if (cork->flags & IPCORK_OPT) 1356 opt = cork->opt; 1357 1358 if (!(rt->dst.dev->features & NETIF_F_SG)) 1359 return -EOPNOTSUPP; 1360 1361 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1362 mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize; 1363 1364 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0); 1365 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen; 1366 maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu; 1367 1368 if (cork->length + size > maxnonfragsize - fragheaderlen) { 1369 ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, 1370 mtu - (opt ? opt->optlen : 0)); 1371 return -EMSGSIZE; 1372 } 1373 1374 skb = skb_peek_tail(&sk->sk_write_queue); 1375 if (!skb) 1376 return -EINVAL; 1377 1378 cork->length += size; 1379 1380 while (size > 0) { 1381 /* Check if the remaining data fits into current packet. */ 1382 len = mtu - skb->len; 1383 if (len < size) 1384 len = maxfraglen - skb->len; 1385 1386 if (len <= 0) { 1387 struct sk_buff *skb_prev; 1388 int alloclen; 1389 1390 skb_prev = skb; 1391 fraggap = skb_prev->len - maxfraglen; 1392 1393 alloclen = fragheaderlen + hh_len + fraggap + 15; 1394 skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation); 1395 if (unlikely(!skb)) { 1396 err = -ENOBUFS; 1397 goto error; 1398 } 1399 1400 /* 1401 * Fill in the control structures 1402 */ 1403 skb->ip_summed = CHECKSUM_NONE; 1404 skb->csum = 0; 1405 skb_reserve(skb, hh_len); 1406 1407 /* 1408 * Find where to start putting bytes. 1409 */ 1410 skb_put(skb, fragheaderlen + fraggap); 1411 skb_reset_network_header(skb); 1412 skb->transport_header = (skb->network_header + 1413 fragheaderlen); 1414 if (fraggap) { 1415 skb->csum = skb_copy_and_csum_bits(skb_prev, 1416 maxfraglen, 1417 skb_transport_header(skb), 1418 fraggap); 1419 skb_prev->csum = csum_sub(skb_prev->csum, 1420 skb->csum); 1421 pskb_trim_unique(skb_prev, maxfraglen); 1422 } 1423 1424 /* 1425 * Put the packet on the pending queue. 1426 */ 1427 __skb_queue_tail(&sk->sk_write_queue, skb); 1428 continue; 1429 } 1430 1431 if (len > size) 1432 len = size; 1433 1434 if (skb_append_pagefrags(skb, page, offset, len)) { 1435 err = -EMSGSIZE; 1436 goto error; 1437 } 1438 1439 if (skb->ip_summed == CHECKSUM_NONE) { 1440 __wsum csum; 1441 csum = csum_page(page, offset, len); 1442 skb->csum = csum_block_add(skb->csum, csum, skb->len); 1443 } 1444 1445 skb->len += len; 1446 skb->data_len += len; 1447 skb->truesize += len; 1448 refcount_add(len, &sk->sk_wmem_alloc); 1449 offset += len; 1450 size -= len; 1451 } 1452 return 0; 1453 1454 error: 1455 cork->length -= size; 1456 IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS); 1457 return err; 1458 } 1459 1460 static void ip_cork_release(struct inet_cork *cork) 1461 { 1462 cork->flags &= ~IPCORK_OPT; 1463 kfree(cork->opt); 1464 cork->opt = NULL; 1465 dst_release(cork->dst); 1466 cork->dst = NULL; 1467 } 1468 1469 /* 1470 * Combined all pending IP fragments on the socket as one IP datagram 1471 * and push them out. 1472 */ 1473 struct sk_buff *__ip_make_skb(struct sock *sk, 1474 struct flowi4 *fl4, 1475 struct sk_buff_head *queue, 1476 struct inet_cork *cork) 1477 { 1478 struct sk_buff *skb, *tmp_skb; 1479 struct sk_buff **tail_skb; 1480 struct inet_sock *inet = inet_sk(sk); 1481 struct net *net = sock_net(sk); 1482 struct ip_options *opt = NULL; 1483 struct rtable *rt = (struct rtable *)cork->dst; 1484 struct iphdr *iph; 1485 __be16 df = 0; 1486 __u8 ttl; 1487 1488 skb = __skb_dequeue(queue); 1489 if (!skb) 1490 goto out; 1491 tail_skb = &(skb_shinfo(skb)->frag_list); 1492 1493 /* move skb->data to ip header from ext header */ 1494 if (skb->data < skb_network_header(skb)) 1495 __skb_pull(skb, skb_network_offset(skb)); 1496 while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1497 __skb_pull(tmp_skb, skb_network_header_len(skb)); 1498 *tail_skb = tmp_skb; 1499 tail_skb = &(tmp_skb->next); 1500 skb->len += tmp_skb->len; 1501 skb->data_len += tmp_skb->len; 1502 skb->truesize += tmp_skb->truesize; 1503 tmp_skb->destructor = NULL; 1504 tmp_skb->sk = NULL; 1505 } 1506 1507 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow 1508 * to fragment the frame generated here. No matter, what transforms 1509 * how transforms change size of the packet, it will come out. 1510 */ 1511 skb->ignore_df = ip_sk_ignore_df(sk); 1512 1513 /* DF bit is set when we want to see DF on outgoing frames. 1514 * If ignore_df is set too, we still allow to fragment this frame 1515 * locally. */ 1516 if (inet->pmtudisc == IP_PMTUDISC_DO || 1517 inet->pmtudisc == IP_PMTUDISC_PROBE || 1518 (skb->len <= dst_mtu(&rt->dst) && 1519 ip_dont_fragment(sk, &rt->dst))) 1520 df = htons(IP_DF); 1521 1522 if (cork->flags & IPCORK_OPT) 1523 opt = cork->opt; 1524 1525 if (cork->ttl != 0) 1526 ttl = cork->ttl; 1527 else if (rt->rt_type == RTN_MULTICAST) 1528 ttl = inet->mc_ttl; 1529 else 1530 ttl = ip_select_ttl(inet, &rt->dst); 1531 1532 iph = ip_hdr(skb); 1533 iph->version = 4; 1534 iph->ihl = 5; 1535 iph->tos = (cork->tos != -1) ? cork->tos : inet->tos; 1536 iph->frag_off = df; 1537 iph->ttl = ttl; 1538 iph->protocol = sk->sk_protocol; 1539 ip_copy_addrs(iph, fl4); 1540 ip_select_ident(net, skb, sk); 1541 1542 if (opt) { 1543 iph->ihl += opt->optlen >> 2; 1544 ip_options_build(skb, opt, cork->addr, rt, 0); 1545 } 1546 1547 skb->priority = (cork->tos != -1) ? cork->priority: sk->sk_priority; 1548 skb->mark = cork->mark; 1549 skb->tstamp = cork->transmit_time; 1550 /* 1551 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec 1552 * on dst refcount 1553 */ 1554 cork->dst = NULL; 1555 skb_dst_set(skb, &rt->dst); 1556 1557 if (iph->protocol == IPPROTO_ICMP) 1558 icmp_out_count(net, ((struct icmphdr *) 1559 skb_transport_header(skb))->type); 1560 1561 ip_cork_release(cork); 1562 out: 1563 return skb; 1564 } 1565 1566 int ip_send_skb(struct net *net, struct sk_buff *skb) 1567 { 1568 int err; 1569 1570 err = ip_local_out(net, skb->sk, skb); 1571 if (err) { 1572 if (err > 0) 1573 err = net_xmit_errno(err); 1574 if (err) 1575 IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS); 1576 } 1577 1578 return err; 1579 } 1580 1581 int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4) 1582 { 1583 struct sk_buff *skb; 1584 1585 skb = ip_finish_skb(sk, fl4); 1586 if (!skb) 1587 return 0; 1588 1589 /* Netfilter gets whole the not fragmented skb. */ 1590 return ip_send_skb(sock_net(sk), skb); 1591 } 1592 1593 /* 1594 * Throw away all pending data on the socket. 1595 */ 1596 static void __ip_flush_pending_frames(struct sock *sk, 1597 struct sk_buff_head *queue, 1598 struct inet_cork *cork) 1599 { 1600 struct sk_buff *skb; 1601 1602 while ((skb = __skb_dequeue_tail(queue)) != NULL) 1603 kfree_skb(skb); 1604 1605 ip_cork_release(cork); 1606 } 1607 1608 void ip_flush_pending_frames(struct sock *sk) 1609 { 1610 __ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base); 1611 } 1612 1613 struct sk_buff *ip_make_skb(struct sock *sk, 1614 struct flowi4 *fl4, 1615 int getfrag(void *from, char *to, int offset, 1616 int len, int odd, struct sk_buff *skb), 1617 void *from, int length, int transhdrlen, 1618 struct ipcm_cookie *ipc, struct rtable **rtp, 1619 struct inet_cork *cork, unsigned int flags) 1620 { 1621 struct sk_buff_head queue; 1622 int err; 1623 1624 if (flags & MSG_PROBE) 1625 return NULL; 1626 1627 __skb_queue_head_init(&queue); 1628 1629 cork->flags = 0; 1630 cork->addr = 0; 1631 cork->opt = NULL; 1632 err = ip_setup_cork(sk, cork, ipc, rtp); 1633 if (err) 1634 return ERR_PTR(err); 1635 1636 err = __ip_append_data(sk, fl4, &queue, cork, 1637 ¤t->task_frag, getfrag, 1638 from, length, transhdrlen, flags); 1639 if (err) { 1640 __ip_flush_pending_frames(sk, &queue, cork); 1641 return ERR_PTR(err); 1642 } 1643 1644 return __ip_make_skb(sk, fl4, &queue, cork); 1645 } 1646 1647 /* 1648 * Fetch data from kernel space and fill in checksum if needed. 1649 */ 1650 static int ip_reply_glue_bits(void *dptr, char *to, int offset, 1651 int len, int odd, struct sk_buff *skb) 1652 { 1653 __wsum csum; 1654 1655 csum = csum_partial_copy_nocheck(dptr+offset, to, len); 1656 skb->csum = csum_block_add(skb->csum, csum, odd); 1657 return 0; 1658 } 1659 1660 /* 1661 * Generic function to send a packet as reply to another packet. 1662 * Used to send some TCP resets/acks so far. 1663 */ 1664 void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb, 1665 const struct ip_options *sopt, 1666 __be32 daddr, __be32 saddr, 1667 const struct ip_reply_arg *arg, 1668 unsigned int len, u64 transmit_time) 1669 { 1670 struct ip_options_data replyopts; 1671 struct ipcm_cookie ipc; 1672 struct flowi4 fl4; 1673 struct rtable *rt = skb_rtable(skb); 1674 struct net *net = sock_net(sk); 1675 struct sk_buff *nskb; 1676 int err; 1677 int oif; 1678 1679 if (__ip_options_echo(net, &replyopts.opt.opt, skb, sopt)) 1680 return; 1681 1682 ipcm_init(&ipc); 1683 ipc.addr = daddr; 1684 ipc.sockc.transmit_time = transmit_time; 1685 1686 if (replyopts.opt.opt.optlen) { 1687 ipc.opt = &replyopts.opt; 1688 1689 if (replyopts.opt.opt.srr) 1690 daddr = replyopts.opt.opt.faddr; 1691 } 1692 1693 oif = arg->bound_dev_if; 1694 if (!oif && netif_index_is_l3_master(net, skb->skb_iif)) 1695 oif = skb->skb_iif; 1696 1697 flowi4_init_output(&fl4, oif, 1698 IP4_REPLY_MARK(net, skb->mark) ?: sk->sk_mark, 1699 RT_TOS(arg->tos), 1700 RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol, 1701 ip_reply_arg_flowi_flags(arg), 1702 daddr, saddr, 1703 tcp_hdr(skb)->source, tcp_hdr(skb)->dest, 1704 arg->uid); 1705 security_skb_classify_flow(skb, flowi4_to_flowi_common(&fl4)); 1706 rt = ip_route_output_key(net, &fl4); 1707 if (IS_ERR(rt)) 1708 return; 1709 1710 inet_sk(sk)->tos = arg->tos & ~INET_ECN_MASK; 1711 1712 sk->sk_protocol = ip_hdr(skb)->protocol; 1713 sk->sk_bound_dev_if = arg->bound_dev_if; 1714 sk->sk_sndbuf = sysctl_wmem_default; 1715 ipc.sockc.mark = fl4.flowi4_mark; 1716 err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base, 1717 len, 0, &ipc, &rt, MSG_DONTWAIT); 1718 if (unlikely(err)) { 1719 ip_flush_pending_frames(sk); 1720 goto out; 1721 } 1722 1723 nskb = skb_peek(&sk->sk_write_queue); 1724 if (nskb) { 1725 if (arg->csumoffset >= 0) 1726 *((__sum16 *)skb_transport_header(nskb) + 1727 arg->csumoffset) = csum_fold(csum_add(nskb->csum, 1728 arg->csum)); 1729 nskb->ip_summed = CHECKSUM_NONE; 1730 ip_push_pending_frames(sk, &fl4); 1731 } 1732 out: 1733 ip_rt_put(rt); 1734 } 1735 1736 void __init ip_init(void) 1737 { 1738 ip_rt_init(); 1739 inet_initpeers(); 1740 1741 #if defined(CONFIG_IP_MULTICAST) 1742 igmp_mc_init(); 1743 #endif 1744 } 1745