1 /* 2 * IPv6 output functions 3 * Linux INET6 implementation 4 * 5 * Authors: 6 * Pedro Roque <roque@di.fc.ul.pt> 7 * 8 * Based on linux/net/ipv4/ip_output.c 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or (at your option) any later version. 14 * 15 * Changes: 16 * A.N.Kuznetsov : airthmetics in fragmentation. 17 * extension headers are implemented. 18 * route changes now work. 19 * ip6_forward does not confuse sniffers. 20 * etc. 21 * 22 * H. von Brand : Added missing #include <linux/string.h> 23 * Imran Patel : frag id should be in NBO 24 * Kazunori MIYAZAWA @USAGI 25 * : add ip6_append_data and related functions 26 * for datagram xmit 27 */ 28 29 #include <linux/errno.h> 30 #include <linux/kernel.h> 31 #include <linux/string.h> 32 #include <linux/socket.h> 33 #include <linux/net.h> 34 #include <linux/netdevice.h> 35 #include <linux/if_arp.h> 36 #include <linux/in6.h> 37 #include <linux/tcp.h> 38 #include <linux/route.h> 39 #include <linux/module.h> 40 #include <linux/slab.h> 41 42 #include <linux/bpf-cgroup.h> 43 #include <linux/netfilter.h> 44 #include <linux/netfilter_ipv6.h> 45 46 #include <net/sock.h> 47 #include <net/snmp.h> 48 49 #include <net/ipv6.h> 50 #include <net/ndisc.h> 51 #include <net/protocol.h> 52 #include <net/ip6_route.h> 53 #include <net/addrconf.h> 54 #include <net/rawv6.h> 55 #include <net/icmp.h> 56 #include <net/xfrm.h> 57 #include <net/checksum.h> 58 #include <linux/mroute6.h> 59 #include <net/l3mdev.h> 60 #include <net/lwtunnel.h> 61 62 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) 63 { 64 struct dst_entry *dst = skb_dst(skb); 65 struct net_device *dev = dst->dev; 66 struct neighbour *neigh; 67 struct in6_addr *nexthop; 68 int ret; 69 70 skb->protocol = htons(ETH_P_IPV6); 71 skb->dev = dev; 72 73 if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) { 74 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 75 76 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && 77 ((mroute6_socket(net, skb) && 78 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || 79 ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr, 80 &ipv6_hdr(skb)->saddr))) { 81 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 82 83 /* Do not check for IFF_ALLMULTI; multicast routing 84 is not supported in any case. 85 */ 86 if (newskb) 87 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING, 88 net, sk, newskb, NULL, newskb->dev, 89 dev_loopback_xmit); 90 91 if (ipv6_hdr(skb)->hop_limit == 0) { 92 IP6_INC_STATS(net, idev, 93 IPSTATS_MIB_OUTDISCARDS); 94 kfree_skb(skb); 95 return 0; 96 } 97 } 98 99 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len); 100 101 if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <= 102 IPV6_ADDR_SCOPE_NODELOCAL && 103 !(dev->flags & IFF_LOOPBACK)) { 104 kfree_skb(skb); 105 return 0; 106 } 107 } 108 109 if (lwtunnel_xmit_redirect(dst->lwtstate)) { 110 int res = lwtunnel_xmit(skb); 111 112 if (res < 0 || res == LWTUNNEL_XMIT_DONE) 113 return res; 114 } 115 116 rcu_read_lock_bh(); 117 nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr); 118 neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop); 119 if (unlikely(!neigh)) 120 neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false); 121 if (!IS_ERR(neigh)) { 122 ret = dst_neigh_output(dst, neigh, skb); 123 rcu_read_unlock_bh(); 124 return ret; 125 } 126 rcu_read_unlock_bh(); 127 128 IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES); 129 kfree_skb(skb); 130 return -EINVAL; 131 } 132 133 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 134 { 135 int ret; 136 137 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); 138 if (ret) { 139 kfree_skb(skb); 140 return ret; 141 } 142 143 if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) || 144 dst_allfrag(skb_dst(skb)) || 145 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)) 146 return ip6_fragment(net, sk, skb, ip6_finish_output2); 147 else 148 return ip6_finish_output2(net, sk, skb); 149 } 150 151 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb) 152 { 153 struct net_device *dev = skb_dst(skb)->dev; 154 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 155 156 if (unlikely(idev->cnf.disable_ipv6)) { 157 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 158 kfree_skb(skb); 159 return 0; 160 } 161 162 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, 163 net, sk, skb, NULL, dev, 164 ip6_finish_output, 165 !(IP6CB(skb)->flags & IP6SKB_REROUTED)); 166 } 167 168 /* 169 * xmit an sk_buff (used by TCP, SCTP and DCCP) 170 * Note : socket lock is not held for SYNACK packets, but might be modified 171 * by calls to skb_set_owner_w() and ipv6_local_error(), 172 * which are using proper atomic operations or spinlocks. 173 */ 174 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, 175 __u32 mark, struct ipv6_txoptions *opt, int tclass) 176 { 177 struct net *net = sock_net(sk); 178 const struct ipv6_pinfo *np = inet6_sk(sk); 179 struct in6_addr *first_hop = &fl6->daddr; 180 struct dst_entry *dst = skb_dst(skb); 181 struct ipv6hdr *hdr; 182 u8 proto = fl6->flowi6_proto; 183 int seg_len = skb->len; 184 int hlimit = -1; 185 u32 mtu; 186 187 if (opt) { 188 unsigned int head_room; 189 190 /* First: exthdrs may take lots of space (~8K for now) 191 MAX_HEADER is not enough. 192 */ 193 head_room = opt->opt_nflen + opt->opt_flen; 194 seg_len += head_room; 195 head_room += sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev); 196 197 if (skb_headroom(skb) < head_room) { 198 struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room); 199 if (!skb2) { 200 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 201 IPSTATS_MIB_OUTDISCARDS); 202 kfree_skb(skb); 203 return -ENOBUFS; 204 } 205 consume_skb(skb); 206 skb = skb2; 207 /* skb_set_owner_w() changes sk->sk_wmem_alloc atomically, 208 * it is safe to call in our context (socket lock not held) 209 */ 210 skb_set_owner_w(skb, (struct sock *)sk); 211 } 212 if (opt->opt_flen) 213 ipv6_push_frag_opts(skb, opt, &proto); 214 if (opt->opt_nflen) 215 ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop, 216 &fl6->saddr); 217 } 218 219 skb_push(skb, sizeof(struct ipv6hdr)); 220 skb_reset_network_header(skb); 221 hdr = ipv6_hdr(skb); 222 223 /* 224 * Fill in the IPv6 header 225 */ 226 if (np) 227 hlimit = np->hop_limit; 228 if (hlimit < 0) 229 hlimit = ip6_dst_hoplimit(dst); 230 231 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel, 232 np->autoflowlabel, fl6)); 233 234 hdr->payload_len = htons(seg_len); 235 hdr->nexthdr = proto; 236 hdr->hop_limit = hlimit; 237 238 hdr->saddr = fl6->saddr; 239 hdr->daddr = *first_hop; 240 241 skb->protocol = htons(ETH_P_IPV6); 242 skb->priority = sk->sk_priority; 243 skb->mark = mark; 244 245 mtu = dst_mtu(dst); 246 if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) { 247 IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)), 248 IPSTATS_MIB_OUT, skb->len); 249 250 /* if egress device is enslaved to an L3 master device pass the 251 * skb to its handler for processing 252 */ 253 skb = l3mdev_ip6_out((struct sock *)sk, skb); 254 if (unlikely(!skb)) 255 return 0; 256 257 /* hooks should never assume socket lock is held. 258 * we promote our socket to non const 259 */ 260 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, 261 net, (struct sock *)sk, skb, NULL, dst->dev, 262 dst_output); 263 } 264 265 skb->dev = dst->dev; 266 /* ipv6_local_error() does not require socket lock, 267 * we promote our socket to non const 268 */ 269 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu); 270 271 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS); 272 kfree_skb(skb); 273 return -EMSGSIZE; 274 } 275 EXPORT_SYMBOL(ip6_xmit); 276 277 static int ip6_call_ra_chain(struct sk_buff *skb, int sel) 278 { 279 struct ip6_ra_chain *ra; 280 struct sock *last = NULL; 281 282 read_lock(&ip6_ra_lock); 283 for (ra = ip6_ra_chain; ra; ra = ra->next) { 284 struct sock *sk = ra->sk; 285 if (sk && ra->sel == sel && 286 (!sk->sk_bound_dev_if || 287 sk->sk_bound_dev_if == skb->dev->ifindex)) { 288 if (last) { 289 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 290 if (skb2) 291 rawv6_rcv(last, skb2); 292 } 293 last = sk; 294 } 295 } 296 297 if (last) { 298 rawv6_rcv(last, skb); 299 read_unlock(&ip6_ra_lock); 300 return 1; 301 } 302 read_unlock(&ip6_ra_lock); 303 return 0; 304 } 305 306 static int ip6_forward_proxy_check(struct sk_buff *skb) 307 { 308 struct ipv6hdr *hdr = ipv6_hdr(skb); 309 u8 nexthdr = hdr->nexthdr; 310 __be16 frag_off; 311 int offset; 312 313 if (ipv6_ext_hdr(nexthdr)) { 314 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off); 315 if (offset < 0) 316 return 0; 317 } else 318 offset = sizeof(struct ipv6hdr); 319 320 if (nexthdr == IPPROTO_ICMPV6) { 321 struct icmp6hdr *icmp6; 322 323 if (!pskb_may_pull(skb, (skb_network_header(skb) + 324 offset + 1 - skb->data))) 325 return 0; 326 327 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset); 328 329 switch (icmp6->icmp6_type) { 330 case NDISC_ROUTER_SOLICITATION: 331 case NDISC_ROUTER_ADVERTISEMENT: 332 case NDISC_NEIGHBOUR_SOLICITATION: 333 case NDISC_NEIGHBOUR_ADVERTISEMENT: 334 case NDISC_REDIRECT: 335 /* For reaction involving unicast neighbor discovery 336 * message destined to the proxied address, pass it to 337 * input function. 338 */ 339 return 1; 340 default: 341 break; 342 } 343 } 344 345 /* 346 * The proxying router can't forward traffic sent to a link-local 347 * address, so signal the sender and discard the packet. This 348 * behavior is clarified by the MIPv6 specification. 349 */ 350 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) { 351 dst_link_failure(skb); 352 return -1; 353 } 354 355 return 0; 356 } 357 358 static inline int ip6_forward_finish(struct net *net, struct sock *sk, 359 struct sk_buff *skb) 360 { 361 return dst_output(net, sk, skb); 362 } 363 364 static unsigned int ip6_dst_mtu_forward(const struct dst_entry *dst) 365 { 366 unsigned int mtu; 367 struct inet6_dev *idev; 368 369 if (dst_metric_locked(dst, RTAX_MTU)) { 370 mtu = dst_metric_raw(dst, RTAX_MTU); 371 if (mtu) 372 return mtu; 373 } 374 375 mtu = IPV6_MIN_MTU; 376 rcu_read_lock(); 377 idev = __in6_dev_get(dst->dev); 378 if (idev) 379 mtu = idev->cnf.mtu6; 380 rcu_read_unlock(); 381 382 return mtu; 383 } 384 385 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu) 386 { 387 if (skb->len <= mtu) 388 return false; 389 390 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */ 391 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu) 392 return true; 393 394 if (skb->ignore_df) 395 return false; 396 397 if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu)) 398 return false; 399 400 return true; 401 } 402 403 int ip6_forward(struct sk_buff *skb) 404 { 405 struct dst_entry *dst = skb_dst(skb); 406 struct ipv6hdr *hdr = ipv6_hdr(skb); 407 struct inet6_skb_parm *opt = IP6CB(skb); 408 struct net *net = dev_net(dst->dev); 409 u32 mtu; 410 411 if (net->ipv6.devconf_all->forwarding == 0) 412 goto error; 413 414 if (skb->pkt_type != PACKET_HOST) 415 goto drop; 416 417 if (unlikely(skb->sk)) 418 goto drop; 419 420 if (skb_warn_if_lro(skb)) 421 goto drop; 422 423 if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) { 424 __IP6_INC_STATS(net, ip6_dst_idev(dst), 425 IPSTATS_MIB_INDISCARDS); 426 goto drop; 427 } 428 429 skb_forward_csum(skb); 430 431 /* 432 * We DO NOT make any processing on 433 * RA packets, pushing them to user level AS IS 434 * without ane WARRANTY that application will be able 435 * to interpret them. The reason is that we 436 * cannot make anything clever here. 437 * 438 * We are not end-node, so that if packet contains 439 * AH/ESP, we cannot make anything. 440 * Defragmentation also would be mistake, RA packets 441 * cannot be fragmented, because there is no warranty 442 * that different fragments will go along one path. --ANK 443 */ 444 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) { 445 if (ip6_call_ra_chain(skb, ntohs(opt->ra))) 446 return 0; 447 } 448 449 /* 450 * check and decrement ttl 451 */ 452 if (hdr->hop_limit <= 1) { 453 /* Force OUTPUT device used as source address */ 454 skb->dev = dst->dev; 455 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0); 456 __IP6_INC_STATS(net, ip6_dst_idev(dst), 457 IPSTATS_MIB_INHDRERRORS); 458 459 kfree_skb(skb); 460 return -ETIMEDOUT; 461 } 462 463 /* XXX: idev->cnf.proxy_ndp? */ 464 if (net->ipv6.devconf_all->proxy_ndp && 465 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) { 466 int proxied = ip6_forward_proxy_check(skb); 467 if (proxied > 0) 468 return ip6_input(skb); 469 else if (proxied < 0) { 470 __IP6_INC_STATS(net, ip6_dst_idev(dst), 471 IPSTATS_MIB_INDISCARDS); 472 goto drop; 473 } 474 } 475 476 if (!xfrm6_route_forward(skb)) { 477 __IP6_INC_STATS(net, ip6_dst_idev(dst), 478 IPSTATS_MIB_INDISCARDS); 479 goto drop; 480 } 481 dst = skb_dst(skb); 482 483 /* IPv6 specs say nothing about it, but it is clear that we cannot 484 send redirects to source routed frames. 485 We don't send redirects to frames decapsulated from IPsec. 486 */ 487 if (skb->dev == dst->dev && opt->srcrt == 0 && !skb_sec_path(skb)) { 488 struct in6_addr *target = NULL; 489 struct inet_peer *peer; 490 struct rt6_info *rt; 491 492 /* 493 * incoming and outgoing devices are the same 494 * send a redirect. 495 */ 496 497 rt = (struct rt6_info *) dst; 498 if (rt->rt6i_flags & RTF_GATEWAY) 499 target = &rt->rt6i_gateway; 500 else 501 target = &hdr->daddr; 502 503 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1); 504 505 /* Limit redirects both by destination (here) 506 and by source (inside ndisc_send_redirect) 507 */ 508 if (inet_peer_xrlim_allow(peer, 1*HZ)) 509 ndisc_send_redirect(skb, target); 510 if (peer) 511 inet_putpeer(peer); 512 } else { 513 int addrtype = ipv6_addr_type(&hdr->saddr); 514 515 /* This check is security critical. */ 516 if (addrtype == IPV6_ADDR_ANY || 517 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK)) 518 goto error; 519 if (addrtype & IPV6_ADDR_LINKLOCAL) { 520 icmpv6_send(skb, ICMPV6_DEST_UNREACH, 521 ICMPV6_NOT_NEIGHBOUR, 0); 522 goto error; 523 } 524 } 525 526 mtu = ip6_dst_mtu_forward(dst); 527 if (mtu < IPV6_MIN_MTU) 528 mtu = IPV6_MIN_MTU; 529 530 if (ip6_pkt_too_big(skb, mtu)) { 531 /* Again, force OUTPUT device used as source address */ 532 skb->dev = dst->dev; 533 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 534 __IP6_INC_STATS(net, ip6_dst_idev(dst), 535 IPSTATS_MIB_INTOOBIGERRORS); 536 __IP6_INC_STATS(net, ip6_dst_idev(dst), 537 IPSTATS_MIB_FRAGFAILS); 538 kfree_skb(skb); 539 return -EMSGSIZE; 540 } 541 542 if (skb_cow(skb, dst->dev->hard_header_len)) { 543 __IP6_INC_STATS(net, ip6_dst_idev(dst), 544 IPSTATS_MIB_OUTDISCARDS); 545 goto drop; 546 } 547 548 hdr = ipv6_hdr(skb); 549 550 /* Mangling hops number delayed to point after skb COW */ 551 552 hdr->hop_limit--; 553 554 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS); 555 __IP6_ADD_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len); 556 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, 557 net, NULL, skb, skb->dev, dst->dev, 558 ip6_forward_finish); 559 560 error: 561 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_INADDRERRORS); 562 drop: 563 kfree_skb(skb); 564 return -EINVAL; 565 } 566 567 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from) 568 { 569 to->pkt_type = from->pkt_type; 570 to->priority = from->priority; 571 to->protocol = from->protocol; 572 skb_dst_drop(to); 573 skb_dst_set(to, dst_clone(skb_dst(from))); 574 to->dev = from->dev; 575 to->mark = from->mark; 576 577 #ifdef CONFIG_NET_SCHED 578 to->tc_index = from->tc_index; 579 #endif 580 nf_copy(to, from); 581 skb_copy_secmark(to, from); 582 } 583 584 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 585 int (*output)(struct net *, struct sock *, struct sk_buff *)) 586 { 587 struct sk_buff *frag; 588 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 589 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? 590 inet6_sk(skb->sk) : NULL; 591 struct ipv6hdr *tmp_hdr; 592 struct frag_hdr *fh; 593 unsigned int mtu, hlen, left, len; 594 int hroom, troom; 595 __be32 frag_id; 596 int ptr, offset = 0, err = 0; 597 u8 *prevhdr, nexthdr = 0; 598 599 hlen = ip6_find_1stfragopt(skb, &prevhdr); 600 nexthdr = *prevhdr; 601 602 mtu = ip6_skb_dst_mtu(skb); 603 604 /* We must not fragment if the socket is set to force MTU discovery 605 * or if the skb it not generated by a local socket. 606 */ 607 if (unlikely(!skb->ignore_df && skb->len > mtu)) 608 goto fail_toobig; 609 610 if (IP6CB(skb)->frag_max_size) { 611 if (IP6CB(skb)->frag_max_size > mtu) 612 goto fail_toobig; 613 614 /* don't send fragments larger than what we received */ 615 mtu = IP6CB(skb)->frag_max_size; 616 if (mtu < IPV6_MIN_MTU) 617 mtu = IPV6_MIN_MTU; 618 } 619 620 if (np && np->frag_size < mtu) { 621 if (np->frag_size) 622 mtu = np->frag_size; 623 } 624 if (mtu < hlen + sizeof(struct frag_hdr) + 8) 625 goto fail_toobig; 626 mtu -= hlen + sizeof(struct frag_hdr); 627 628 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr, 629 &ipv6_hdr(skb)->saddr); 630 631 if (skb->ip_summed == CHECKSUM_PARTIAL && 632 (err = skb_checksum_help(skb))) 633 goto fail; 634 635 hroom = LL_RESERVED_SPACE(rt->dst.dev); 636 if (skb_has_frag_list(skb)) { 637 unsigned int first_len = skb_pagelen(skb); 638 struct sk_buff *frag2; 639 640 if (first_len - hlen > mtu || 641 ((first_len - hlen) & 7) || 642 skb_cloned(skb) || 643 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) 644 goto slow_path; 645 646 skb_walk_frags(skb, frag) { 647 /* Correct geometry. */ 648 if (frag->len > mtu || 649 ((frag->len & 7) && frag->next) || 650 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr))) 651 goto slow_path_clean; 652 653 /* Partially cloned skb? */ 654 if (skb_shared(frag)) 655 goto slow_path_clean; 656 657 BUG_ON(frag->sk); 658 if (skb->sk) { 659 frag->sk = skb->sk; 660 frag->destructor = sock_wfree; 661 } 662 skb->truesize -= frag->truesize; 663 } 664 665 err = 0; 666 offset = 0; 667 /* BUILD HEADER */ 668 669 *prevhdr = NEXTHDR_FRAGMENT; 670 tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC); 671 if (!tmp_hdr) { 672 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 673 IPSTATS_MIB_FRAGFAILS); 674 err = -ENOMEM; 675 goto fail; 676 } 677 frag = skb_shinfo(skb)->frag_list; 678 skb_frag_list_init(skb); 679 680 __skb_pull(skb, hlen); 681 fh = (struct frag_hdr *)__skb_push(skb, sizeof(struct frag_hdr)); 682 __skb_push(skb, hlen); 683 skb_reset_network_header(skb); 684 memcpy(skb_network_header(skb), tmp_hdr, hlen); 685 686 fh->nexthdr = nexthdr; 687 fh->reserved = 0; 688 fh->frag_off = htons(IP6_MF); 689 fh->identification = frag_id; 690 691 first_len = skb_pagelen(skb); 692 skb->data_len = first_len - skb_headlen(skb); 693 skb->len = first_len; 694 ipv6_hdr(skb)->payload_len = htons(first_len - 695 sizeof(struct ipv6hdr)); 696 697 dst_hold(&rt->dst); 698 699 for (;;) { 700 /* Prepare header of the next frame, 701 * before previous one went down. */ 702 if (frag) { 703 frag->ip_summed = CHECKSUM_NONE; 704 skb_reset_transport_header(frag); 705 fh = (struct frag_hdr *)__skb_push(frag, sizeof(struct frag_hdr)); 706 __skb_push(frag, hlen); 707 skb_reset_network_header(frag); 708 memcpy(skb_network_header(frag), tmp_hdr, 709 hlen); 710 offset += skb->len - hlen - sizeof(struct frag_hdr); 711 fh->nexthdr = nexthdr; 712 fh->reserved = 0; 713 fh->frag_off = htons(offset); 714 if (frag->next) 715 fh->frag_off |= htons(IP6_MF); 716 fh->identification = frag_id; 717 ipv6_hdr(frag)->payload_len = 718 htons(frag->len - 719 sizeof(struct ipv6hdr)); 720 ip6_copy_metadata(frag, skb); 721 } 722 723 err = output(net, sk, skb); 724 if (!err) 725 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 726 IPSTATS_MIB_FRAGCREATES); 727 728 if (err || !frag) 729 break; 730 731 skb = frag; 732 frag = skb->next; 733 skb->next = NULL; 734 } 735 736 kfree(tmp_hdr); 737 738 if (err == 0) { 739 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 740 IPSTATS_MIB_FRAGOKS); 741 ip6_rt_put(rt); 742 return 0; 743 } 744 745 kfree_skb_list(frag); 746 747 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 748 IPSTATS_MIB_FRAGFAILS); 749 ip6_rt_put(rt); 750 return err; 751 752 slow_path_clean: 753 skb_walk_frags(skb, frag2) { 754 if (frag2 == frag) 755 break; 756 frag2->sk = NULL; 757 frag2->destructor = NULL; 758 skb->truesize += frag2->truesize; 759 } 760 } 761 762 slow_path: 763 left = skb->len - hlen; /* Space per frame */ 764 ptr = hlen; /* Where to start from */ 765 766 /* 767 * Fragment the datagram. 768 */ 769 770 *prevhdr = NEXTHDR_FRAGMENT; 771 troom = rt->dst.dev->needed_tailroom; 772 773 /* 774 * Keep copying data until we run out. 775 */ 776 while (left > 0) { 777 len = left; 778 /* IF: it doesn't fit, use 'mtu' - the data space left */ 779 if (len > mtu) 780 len = mtu; 781 /* IF: we are not sending up to and including the packet end 782 then align the next start on an eight byte boundary */ 783 if (len < left) { 784 len &= ~7; 785 } 786 787 /* Allocate buffer */ 788 frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) + 789 hroom + troom, GFP_ATOMIC); 790 if (!frag) { 791 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 792 IPSTATS_MIB_FRAGFAILS); 793 err = -ENOMEM; 794 goto fail; 795 } 796 797 /* 798 * Set up data on packet 799 */ 800 801 ip6_copy_metadata(frag, skb); 802 skb_reserve(frag, hroom); 803 skb_put(frag, len + hlen + sizeof(struct frag_hdr)); 804 skb_reset_network_header(frag); 805 fh = (struct frag_hdr *)(skb_network_header(frag) + hlen); 806 frag->transport_header = (frag->network_header + hlen + 807 sizeof(struct frag_hdr)); 808 809 /* 810 * Charge the memory for the fragment to any owner 811 * it might possess 812 */ 813 if (skb->sk) 814 skb_set_owner_w(frag, skb->sk); 815 816 /* 817 * Copy the packet header into the new buffer. 818 */ 819 skb_copy_from_linear_data(skb, skb_network_header(frag), hlen); 820 821 /* 822 * Build fragment header. 823 */ 824 fh->nexthdr = nexthdr; 825 fh->reserved = 0; 826 fh->identification = frag_id; 827 828 /* 829 * Copy a block of the IP datagram. 830 */ 831 BUG_ON(skb_copy_bits(skb, ptr, skb_transport_header(frag), 832 len)); 833 left -= len; 834 835 fh->frag_off = htons(offset); 836 if (left > 0) 837 fh->frag_off |= htons(IP6_MF); 838 ipv6_hdr(frag)->payload_len = htons(frag->len - 839 sizeof(struct ipv6hdr)); 840 841 ptr += len; 842 offset += len; 843 844 /* 845 * Put this fragment into the sending queue. 846 */ 847 err = output(net, sk, frag); 848 if (err) 849 goto fail; 850 851 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 852 IPSTATS_MIB_FRAGCREATES); 853 } 854 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 855 IPSTATS_MIB_FRAGOKS); 856 consume_skb(skb); 857 return err; 858 859 fail_toobig: 860 if (skb->sk && dst_allfrag(skb_dst(skb))) 861 sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK); 862 863 skb->dev = skb_dst(skb)->dev; 864 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 865 err = -EMSGSIZE; 866 867 fail: 868 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 869 IPSTATS_MIB_FRAGFAILS); 870 kfree_skb(skb); 871 return err; 872 } 873 874 static inline int ip6_rt_check(const struct rt6key *rt_key, 875 const struct in6_addr *fl_addr, 876 const struct in6_addr *addr_cache) 877 { 878 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) && 879 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache)); 880 } 881 882 static struct dst_entry *ip6_sk_dst_check(struct sock *sk, 883 struct dst_entry *dst, 884 const struct flowi6 *fl6) 885 { 886 struct ipv6_pinfo *np = inet6_sk(sk); 887 struct rt6_info *rt; 888 889 if (!dst) 890 goto out; 891 892 if (dst->ops->family != AF_INET6) { 893 dst_release(dst); 894 return NULL; 895 } 896 897 rt = (struct rt6_info *)dst; 898 /* Yes, checking route validity in not connected 899 * case is not very simple. Take into account, 900 * that we do not support routing by source, TOS, 901 * and MSG_DONTROUTE --ANK (980726) 902 * 903 * 1. ip6_rt_check(): If route was host route, 904 * check that cached destination is current. 905 * If it is network route, we still may 906 * check its validity using saved pointer 907 * to the last used address: daddr_cache. 908 * We do not want to save whole address now, 909 * (because main consumer of this service 910 * is tcp, which has not this problem), 911 * so that the last trick works only on connected 912 * sockets. 913 * 2. oif also should be the same. 914 */ 915 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) || 916 #ifdef CONFIG_IPV6_SUBTREES 917 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) || 918 #endif 919 (!(fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) && 920 (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex))) { 921 dst_release(dst); 922 dst = NULL; 923 } 924 925 out: 926 return dst; 927 } 928 929 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk, 930 struct dst_entry **dst, struct flowi6 *fl6) 931 { 932 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 933 struct neighbour *n; 934 struct rt6_info *rt; 935 #endif 936 int err; 937 int flags = 0; 938 939 /* The correct way to handle this would be to do 940 * ip6_route_get_saddr, and then ip6_route_output; however, 941 * the route-specific preferred source forces the 942 * ip6_route_output call _before_ ip6_route_get_saddr. 943 * 944 * In source specific routing (no src=any default route), 945 * ip6_route_output will fail given src=any saddr, though, so 946 * that's why we try it again later. 947 */ 948 if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) { 949 struct rt6_info *rt; 950 bool had_dst = *dst != NULL; 951 952 if (!had_dst) 953 *dst = ip6_route_output(net, sk, fl6); 954 rt = (*dst)->error ? NULL : (struct rt6_info *)*dst; 955 err = ip6_route_get_saddr(net, rt, &fl6->daddr, 956 sk ? inet6_sk(sk)->srcprefs : 0, 957 &fl6->saddr); 958 if (err) 959 goto out_err_release; 960 961 /* If we had an erroneous initial result, pretend it 962 * never existed and let the SA-enabled version take 963 * over. 964 */ 965 if (!had_dst && (*dst)->error) { 966 dst_release(*dst); 967 *dst = NULL; 968 } 969 970 if (fl6->flowi6_oif) 971 flags |= RT6_LOOKUP_F_IFACE; 972 } 973 974 if (!*dst) 975 *dst = ip6_route_output_flags(net, sk, fl6, flags); 976 977 err = (*dst)->error; 978 if (err) 979 goto out_err_release; 980 981 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 982 /* 983 * Here if the dst entry we've looked up 984 * has a neighbour entry that is in the INCOMPLETE 985 * state and the src address from the flow is 986 * marked as OPTIMISTIC, we release the found 987 * dst entry and replace it instead with the 988 * dst entry of the nexthop router 989 */ 990 rt = (struct rt6_info *) *dst; 991 rcu_read_lock_bh(); 992 n = __ipv6_neigh_lookup_noref(rt->dst.dev, 993 rt6_nexthop(rt, &fl6->daddr)); 994 err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0; 995 rcu_read_unlock_bh(); 996 997 if (err) { 998 struct inet6_ifaddr *ifp; 999 struct flowi6 fl_gw6; 1000 int redirect; 1001 1002 ifp = ipv6_get_ifaddr(net, &fl6->saddr, 1003 (*dst)->dev, 1); 1004 1005 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC); 1006 if (ifp) 1007 in6_ifa_put(ifp); 1008 1009 if (redirect) { 1010 /* 1011 * We need to get the dst entry for the 1012 * default router instead 1013 */ 1014 dst_release(*dst); 1015 memcpy(&fl_gw6, fl6, sizeof(struct flowi6)); 1016 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr)); 1017 *dst = ip6_route_output(net, sk, &fl_gw6); 1018 err = (*dst)->error; 1019 if (err) 1020 goto out_err_release; 1021 } 1022 } 1023 #endif 1024 if (ipv6_addr_v4mapped(&fl6->saddr) && 1025 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) { 1026 err = -EAFNOSUPPORT; 1027 goto out_err_release; 1028 } 1029 1030 return 0; 1031 1032 out_err_release: 1033 dst_release(*dst); 1034 *dst = NULL; 1035 1036 if (err == -ENETUNREACH) 1037 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES); 1038 return err; 1039 } 1040 1041 /** 1042 * ip6_dst_lookup - perform route lookup on flow 1043 * @sk: socket which provides route info 1044 * @dst: pointer to dst_entry * for result 1045 * @fl6: flow to lookup 1046 * 1047 * This function performs a route lookup on the given flow. 1048 * 1049 * It returns zero on success, or a standard errno code on error. 1050 */ 1051 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, 1052 struct flowi6 *fl6) 1053 { 1054 *dst = NULL; 1055 return ip6_dst_lookup_tail(net, sk, dst, fl6); 1056 } 1057 EXPORT_SYMBOL_GPL(ip6_dst_lookup); 1058 1059 /** 1060 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec 1061 * @sk: socket which provides route info 1062 * @fl6: flow to lookup 1063 * @final_dst: final destination address for ipsec lookup 1064 * 1065 * This function performs a route lookup on the given flow. 1066 * 1067 * It returns a valid dst pointer on success, or a pointer encoded 1068 * error code. 1069 */ 1070 struct dst_entry *ip6_dst_lookup_flow(const struct sock *sk, struct flowi6 *fl6, 1071 const struct in6_addr *final_dst) 1072 { 1073 struct dst_entry *dst = NULL; 1074 int err; 1075 1076 err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6); 1077 if (err) 1078 return ERR_PTR(err); 1079 if (final_dst) 1080 fl6->daddr = *final_dst; 1081 1082 return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0); 1083 } 1084 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow); 1085 1086 /** 1087 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow 1088 * @sk: socket which provides the dst cache and route info 1089 * @fl6: flow to lookup 1090 * @final_dst: final destination address for ipsec lookup 1091 * 1092 * This function performs a route lookup on the given flow with the 1093 * possibility of using the cached route in the socket if it is valid. 1094 * It will take the socket dst lock when operating on the dst cache. 1095 * As a result, this function can only be used in process context. 1096 * 1097 * It returns a valid dst pointer on success, or a pointer encoded 1098 * error code. 1099 */ 1100 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6, 1101 const struct in6_addr *final_dst) 1102 { 1103 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie); 1104 1105 dst = ip6_sk_dst_check(sk, dst, fl6); 1106 if (!dst) 1107 dst = ip6_dst_lookup_flow(sk, fl6, final_dst); 1108 1109 return dst; 1110 } 1111 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow); 1112 1113 static inline int ip6_ufo_append_data(struct sock *sk, 1114 struct sk_buff_head *queue, 1115 int getfrag(void *from, char *to, int offset, int len, 1116 int odd, struct sk_buff *skb), 1117 void *from, int length, int hh_len, int fragheaderlen, 1118 int exthdrlen, int transhdrlen, int mtu, 1119 unsigned int flags, const struct flowi6 *fl6) 1120 1121 { 1122 struct sk_buff *skb; 1123 int err; 1124 1125 /* There is support for UDP large send offload by network 1126 * device, so create one single skb packet containing complete 1127 * udp datagram 1128 */ 1129 skb = skb_peek_tail(queue); 1130 if (!skb) { 1131 skb = sock_alloc_send_skb(sk, 1132 hh_len + fragheaderlen + transhdrlen + 20, 1133 (flags & MSG_DONTWAIT), &err); 1134 if (!skb) 1135 return err; 1136 1137 /* reserve space for Hardware header */ 1138 skb_reserve(skb, hh_len); 1139 1140 /* create space for UDP/IP header */ 1141 skb_put(skb, fragheaderlen + transhdrlen); 1142 1143 /* initialize network header pointer */ 1144 skb_set_network_header(skb, exthdrlen); 1145 1146 /* initialize protocol header pointer */ 1147 skb->transport_header = skb->network_header + fragheaderlen; 1148 1149 skb->protocol = htons(ETH_P_IPV6); 1150 skb->csum = 0; 1151 1152 __skb_queue_tail(queue, skb); 1153 } else if (skb_is_gso(skb)) { 1154 goto append; 1155 } 1156 1157 skb->ip_summed = CHECKSUM_PARTIAL; 1158 /* Specify the length of each IPv6 datagram fragment. 1159 * It has to be a multiple of 8. 1160 */ 1161 skb_shinfo(skb)->gso_size = (mtu - fragheaderlen - 1162 sizeof(struct frag_hdr)) & ~7; 1163 skb_shinfo(skb)->gso_type = SKB_GSO_UDP; 1164 skb_shinfo(skb)->ip6_frag_id = ipv6_select_ident(sock_net(sk), 1165 &fl6->daddr, 1166 &fl6->saddr); 1167 1168 append: 1169 return skb_append_datato_frags(sk, skb, getfrag, from, 1170 (length - transhdrlen)); 1171 } 1172 1173 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src, 1174 gfp_t gfp) 1175 { 1176 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1177 } 1178 1179 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src, 1180 gfp_t gfp) 1181 { 1182 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1183 } 1184 1185 static void ip6_append_data_mtu(unsigned int *mtu, 1186 int *maxfraglen, 1187 unsigned int fragheaderlen, 1188 struct sk_buff *skb, 1189 struct rt6_info *rt, 1190 unsigned int orig_mtu) 1191 { 1192 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) { 1193 if (!skb) { 1194 /* first fragment, reserve header_len */ 1195 *mtu = orig_mtu - rt->dst.header_len; 1196 1197 } else { 1198 /* 1199 * this fragment is not first, the headers 1200 * space is regarded as data space. 1201 */ 1202 *mtu = orig_mtu; 1203 } 1204 *maxfraglen = ((*mtu - fragheaderlen) & ~7) 1205 + fragheaderlen - sizeof(struct frag_hdr); 1206 } 1207 } 1208 1209 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, 1210 struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6, 1211 struct rt6_info *rt, struct flowi6 *fl6) 1212 { 1213 struct ipv6_pinfo *np = inet6_sk(sk); 1214 unsigned int mtu; 1215 struct ipv6_txoptions *opt = ipc6->opt; 1216 1217 /* 1218 * setup for corking 1219 */ 1220 if (opt) { 1221 if (WARN_ON(v6_cork->opt)) 1222 return -EINVAL; 1223 1224 v6_cork->opt = kzalloc(opt->tot_len, sk->sk_allocation); 1225 if (unlikely(!v6_cork->opt)) 1226 return -ENOBUFS; 1227 1228 v6_cork->opt->tot_len = opt->tot_len; 1229 v6_cork->opt->opt_flen = opt->opt_flen; 1230 v6_cork->opt->opt_nflen = opt->opt_nflen; 1231 1232 v6_cork->opt->dst0opt = ip6_opt_dup(opt->dst0opt, 1233 sk->sk_allocation); 1234 if (opt->dst0opt && !v6_cork->opt->dst0opt) 1235 return -ENOBUFS; 1236 1237 v6_cork->opt->dst1opt = ip6_opt_dup(opt->dst1opt, 1238 sk->sk_allocation); 1239 if (opt->dst1opt && !v6_cork->opt->dst1opt) 1240 return -ENOBUFS; 1241 1242 v6_cork->opt->hopopt = ip6_opt_dup(opt->hopopt, 1243 sk->sk_allocation); 1244 if (opt->hopopt && !v6_cork->opt->hopopt) 1245 return -ENOBUFS; 1246 1247 v6_cork->opt->srcrt = ip6_rthdr_dup(opt->srcrt, 1248 sk->sk_allocation); 1249 if (opt->srcrt && !v6_cork->opt->srcrt) 1250 return -ENOBUFS; 1251 1252 /* need source address above miyazawa*/ 1253 } 1254 dst_hold(&rt->dst); 1255 cork->base.dst = &rt->dst; 1256 cork->fl.u.ip6 = *fl6; 1257 v6_cork->hop_limit = ipc6->hlimit; 1258 v6_cork->tclass = ipc6->tclass; 1259 if (rt->dst.flags & DST_XFRM_TUNNEL) 1260 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1261 rt->dst.dev->mtu : dst_mtu(&rt->dst); 1262 else 1263 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1264 rt->dst.dev->mtu : dst_mtu(rt->dst.path); 1265 if (np->frag_size < mtu) { 1266 if (np->frag_size) 1267 mtu = np->frag_size; 1268 } 1269 cork->base.fragsize = mtu; 1270 if (dst_allfrag(rt->dst.path)) 1271 cork->base.flags |= IPCORK_ALLFRAG; 1272 cork->base.length = 0; 1273 1274 return 0; 1275 } 1276 1277 static int __ip6_append_data(struct sock *sk, 1278 struct flowi6 *fl6, 1279 struct sk_buff_head *queue, 1280 struct inet_cork *cork, 1281 struct inet6_cork *v6_cork, 1282 struct page_frag *pfrag, 1283 int getfrag(void *from, char *to, int offset, 1284 int len, int odd, struct sk_buff *skb), 1285 void *from, int length, int transhdrlen, 1286 unsigned int flags, struct ipcm6_cookie *ipc6, 1287 const struct sockcm_cookie *sockc) 1288 { 1289 struct sk_buff *skb, *skb_prev = NULL; 1290 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu; 1291 int exthdrlen = 0; 1292 int dst_exthdrlen = 0; 1293 int hh_len; 1294 int copy; 1295 int err; 1296 int offset = 0; 1297 __u8 tx_flags = 0; 1298 u32 tskey = 0; 1299 struct rt6_info *rt = (struct rt6_info *)cork->dst; 1300 struct ipv6_txoptions *opt = v6_cork->opt; 1301 int csummode = CHECKSUM_NONE; 1302 unsigned int maxnonfragsize, headersize; 1303 1304 skb = skb_peek_tail(queue); 1305 if (!skb) { 1306 exthdrlen = opt ? opt->opt_flen : 0; 1307 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; 1308 } 1309 1310 mtu = cork->fragsize; 1311 orig_mtu = mtu; 1312 1313 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1314 1315 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + 1316 (opt ? opt->opt_nflen : 0); 1317 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - 1318 sizeof(struct frag_hdr); 1319 1320 headersize = sizeof(struct ipv6hdr) + 1321 (opt ? opt->opt_flen + opt->opt_nflen : 0) + 1322 (dst_allfrag(&rt->dst) ? 1323 sizeof(struct frag_hdr) : 0) + 1324 rt->rt6i_nfheader_len; 1325 1326 if (cork->length + length > mtu - headersize && ipc6->dontfrag && 1327 (sk->sk_protocol == IPPROTO_UDP || 1328 sk->sk_protocol == IPPROTO_RAW)) { 1329 ipv6_local_rxpmtu(sk, fl6, mtu - headersize + 1330 sizeof(struct ipv6hdr)); 1331 goto emsgsize; 1332 } 1333 1334 if (ip6_sk_ignore_df(sk)) 1335 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN; 1336 else 1337 maxnonfragsize = mtu; 1338 1339 if (cork->length + length > maxnonfragsize - headersize) { 1340 emsgsize: 1341 ipv6_local_error(sk, EMSGSIZE, fl6, 1342 mtu - headersize + 1343 sizeof(struct ipv6hdr)); 1344 return -EMSGSIZE; 1345 } 1346 1347 /* CHECKSUM_PARTIAL only with no extension headers and when 1348 * we are not going to fragment 1349 */ 1350 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP && 1351 headersize == sizeof(struct ipv6hdr) && 1352 length <= mtu - headersize && 1353 !(flags & MSG_MORE) && 1354 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 1355 csummode = CHECKSUM_PARTIAL; 1356 1357 if (sk->sk_type == SOCK_DGRAM || sk->sk_type == SOCK_RAW) { 1358 sock_tx_timestamp(sk, sockc->tsflags, &tx_flags); 1359 if (tx_flags & SKBTX_ANY_SW_TSTAMP && 1360 sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) 1361 tskey = sk->sk_tskey++; 1362 } 1363 1364 /* 1365 * Let's try using as much space as possible. 1366 * Use MTU if total length of the message fits into the MTU. 1367 * Otherwise, we need to reserve fragment header and 1368 * fragment alignment (= 8-15 octects, in total). 1369 * 1370 * Note that we may need to "move" the data from the tail of 1371 * of the buffer to the new fragment when we split 1372 * the message. 1373 * 1374 * FIXME: It may be fragmented into multiple chunks 1375 * at once if non-fragmentable extension headers 1376 * are too large. 1377 * --yoshfuji 1378 */ 1379 1380 cork->length += length; 1381 if ((((length + fragheaderlen) > mtu) || 1382 (skb && skb_is_gso(skb))) && 1383 (sk->sk_protocol == IPPROTO_UDP) && 1384 (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len && 1385 (sk->sk_type == SOCK_DGRAM) && !udp_get_no_check6_tx(sk)) { 1386 err = ip6_ufo_append_data(sk, queue, getfrag, from, length, 1387 hh_len, fragheaderlen, exthdrlen, 1388 transhdrlen, mtu, flags, fl6); 1389 if (err) 1390 goto error; 1391 return 0; 1392 } 1393 1394 if (!skb) 1395 goto alloc_new_skb; 1396 1397 while (length > 0) { 1398 /* Check if the remaining data fits into current packet. */ 1399 copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len; 1400 if (copy < length) 1401 copy = maxfraglen - skb->len; 1402 1403 if (copy <= 0) { 1404 char *data; 1405 unsigned int datalen; 1406 unsigned int fraglen; 1407 unsigned int fraggap; 1408 unsigned int alloclen; 1409 alloc_new_skb: 1410 /* There's no room in the current skb */ 1411 if (skb) 1412 fraggap = skb->len - maxfraglen; 1413 else 1414 fraggap = 0; 1415 /* update mtu and maxfraglen if necessary */ 1416 if (!skb || !skb_prev) 1417 ip6_append_data_mtu(&mtu, &maxfraglen, 1418 fragheaderlen, skb, rt, 1419 orig_mtu); 1420 1421 skb_prev = skb; 1422 1423 /* 1424 * If remaining data exceeds the mtu, 1425 * we know we need more fragment(s). 1426 */ 1427 datalen = length + fraggap; 1428 1429 if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) 1430 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; 1431 if ((flags & MSG_MORE) && 1432 !(rt->dst.dev->features&NETIF_F_SG)) 1433 alloclen = mtu; 1434 else 1435 alloclen = datalen + fragheaderlen; 1436 1437 alloclen += dst_exthdrlen; 1438 1439 if (datalen != length + fraggap) { 1440 /* 1441 * this is not the last fragment, the trailer 1442 * space is regarded as data space. 1443 */ 1444 datalen += rt->dst.trailer_len; 1445 } 1446 1447 alloclen += rt->dst.trailer_len; 1448 fraglen = datalen + fragheaderlen; 1449 1450 /* 1451 * We just reserve space for fragment header. 1452 * Note: this may be overallocation if the message 1453 * (without MSG_MORE) fits into the MTU. 1454 */ 1455 alloclen += sizeof(struct frag_hdr); 1456 1457 if (transhdrlen) { 1458 skb = sock_alloc_send_skb(sk, 1459 alloclen + hh_len, 1460 (flags & MSG_DONTWAIT), &err); 1461 } else { 1462 skb = NULL; 1463 if (atomic_read(&sk->sk_wmem_alloc) <= 1464 2 * sk->sk_sndbuf) 1465 skb = sock_wmalloc(sk, 1466 alloclen + hh_len, 1, 1467 sk->sk_allocation); 1468 if (unlikely(!skb)) 1469 err = -ENOBUFS; 1470 } 1471 if (!skb) 1472 goto error; 1473 /* 1474 * Fill in the control structures 1475 */ 1476 skb->protocol = htons(ETH_P_IPV6); 1477 skb->ip_summed = csummode; 1478 skb->csum = 0; 1479 /* reserve for fragmentation and ipsec header */ 1480 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + 1481 dst_exthdrlen); 1482 1483 /* Only the initial fragment is time stamped */ 1484 skb_shinfo(skb)->tx_flags = tx_flags; 1485 tx_flags = 0; 1486 skb_shinfo(skb)->tskey = tskey; 1487 tskey = 0; 1488 1489 /* 1490 * Find where to start putting bytes 1491 */ 1492 data = skb_put(skb, fraglen); 1493 skb_set_network_header(skb, exthdrlen); 1494 data += fragheaderlen; 1495 skb->transport_header = (skb->network_header + 1496 fragheaderlen); 1497 if (fraggap) { 1498 skb->csum = skb_copy_and_csum_bits( 1499 skb_prev, maxfraglen, 1500 data + transhdrlen, fraggap, 0); 1501 skb_prev->csum = csum_sub(skb_prev->csum, 1502 skb->csum); 1503 data += fraggap; 1504 pskb_trim_unique(skb_prev, maxfraglen); 1505 } 1506 copy = datalen - transhdrlen - fraggap; 1507 1508 if (copy < 0) { 1509 err = -EINVAL; 1510 kfree_skb(skb); 1511 goto error; 1512 } else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { 1513 err = -EFAULT; 1514 kfree_skb(skb); 1515 goto error; 1516 } 1517 1518 offset += copy; 1519 length -= datalen - fraggap; 1520 transhdrlen = 0; 1521 exthdrlen = 0; 1522 dst_exthdrlen = 0; 1523 1524 /* 1525 * Put the packet on the pending queue 1526 */ 1527 __skb_queue_tail(queue, skb); 1528 continue; 1529 } 1530 1531 if (copy > length) 1532 copy = length; 1533 1534 if (!(rt->dst.dev->features&NETIF_F_SG)) { 1535 unsigned int off; 1536 1537 off = skb->len; 1538 if (getfrag(from, skb_put(skb, copy), 1539 offset, copy, off, skb) < 0) { 1540 __skb_trim(skb, off); 1541 err = -EFAULT; 1542 goto error; 1543 } 1544 } else { 1545 int i = skb_shinfo(skb)->nr_frags; 1546 1547 err = -ENOMEM; 1548 if (!sk_page_frag_refill(sk, pfrag)) 1549 goto error; 1550 1551 if (!skb_can_coalesce(skb, i, pfrag->page, 1552 pfrag->offset)) { 1553 err = -EMSGSIZE; 1554 if (i == MAX_SKB_FRAGS) 1555 goto error; 1556 1557 __skb_fill_page_desc(skb, i, pfrag->page, 1558 pfrag->offset, 0); 1559 skb_shinfo(skb)->nr_frags = ++i; 1560 get_page(pfrag->page); 1561 } 1562 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1563 if (getfrag(from, 1564 page_address(pfrag->page) + pfrag->offset, 1565 offset, copy, skb->len, skb) < 0) 1566 goto error_efault; 1567 1568 pfrag->offset += copy; 1569 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1570 skb->len += copy; 1571 skb->data_len += copy; 1572 skb->truesize += copy; 1573 atomic_add(copy, &sk->sk_wmem_alloc); 1574 } 1575 offset += copy; 1576 length -= copy; 1577 } 1578 1579 return 0; 1580 1581 error_efault: 1582 err = -EFAULT; 1583 error: 1584 cork->length -= length; 1585 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); 1586 return err; 1587 } 1588 1589 int ip6_append_data(struct sock *sk, 1590 int getfrag(void *from, char *to, int offset, int len, 1591 int odd, struct sk_buff *skb), 1592 void *from, int length, int transhdrlen, 1593 struct ipcm6_cookie *ipc6, struct flowi6 *fl6, 1594 struct rt6_info *rt, unsigned int flags, 1595 const struct sockcm_cookie *sockc) 1596 { 1597 struct inet_sock *inet = inet_sk(sk); 1598 struct ipv6_pinfo *np = inet6_sk(sk); 1599 int exthdrlen; 1600 int err; 1601 1602 if (flags&MSG_PROBE) 1603 return 0; 1604 if (skb_queue_empty(&sk->sk_write_queue)) { 1605 /* 1606 * setup for corking 1607 */ 1608 err = ip6_setup_cork(sk, &inet->cork, &np->cork, 1609 ipc6, rt, fl6); 1610 if (err) 1611 return err; 1612 1613 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 1614 length += exthdrlen; 1615 transhdrlen += exthdrlen; 1616 } else { 1617 fl6 = &inet->cork.fl.u.ip6; 1618 transhdrlen = 0; 1619 } 1620 1621 return __ip6_append_data(sk, fl6, &sk->sk_write_queue, &inet->cork.base, 1622 &np->cork, sk_page_frag(sk), getfrag, 1623 from, length, transhdrlen, flags, ipc6, sockc); 1624 } 1625 EXPORT_SYMBOL_GPL(ip6_append_data); 1626 1627 static void ip6_cork_release(struct inet_cork_full *cork, 1628 struct inet6_cork *v6_cork) 1629 { 1630 if (v6_cork->opt) { 1631 kfree(v6_cork->opt->dst0opt); 1632 kfree(v6_cork->opt->dst1opt); 1633 kfree(v6_cork->opt->hopopt); 1634 kfree(v6_cork->opt->srcrt); 1635 kfree(v6_cork->opt); 1636 v6_cork->opt = NULL; 1637 } 1638 1639 if (cork->base.dst) { 1640 dst_release(cork->base.dst); 1641 cork->base.dst = NULL; 1642 cork->base.flags &= ~IPCORK_ALLFRAG; 1643 } 1644 memset(&cork->fl, 0, sizeof(cork->fl)); 1645 } 1646 1647 struct sk_buff *__ip6_make_skb(struct sock *sk, 1648 struct sk_buff_head *queue, 1649 struct inet_cork_full *cork, 1650 struct inet6_cork *v6_cork) 1651 { 1652 struct sk_buff *skb, *tmp_skb; 1653 struct sk_buff **tail_skb; 1654 struct in6_addr final_dst_buf, *final_dst = &final_dst_buf; 1655 struct ipv6_pinfo *np = inet6_sk(sk); 1656 struct net *net = sock_net(sk); 1657 struct ipv6hdr *hdr; 1658 struct ipv6_txoptions *opt = v6_cork->opt; 1659 struct rt6_info *rt = (struct rt6_info *)cork->base.dst; 1660 struct flowi6 *fl6 = &cork->fl.u.ip6; 1661 unsigned char proto = fl6->flowi6_proto; 1662 1663 skb = __skb_dequeue(queue); 1664 if (!skb) 1665 goto out; 1666 tail_skb = &(skb_shinfo(skb)->frag_list); 1667 1668 /* move skb->data to ip header from ext header */ 1669 if (skb->data < skb_network_header(skb)) 1670 __skb_pull(skb, skb_network_offset(skb)); 1671 while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1672 __skb_pull(tmp_skb, skb_network_header_len(skb)); 1673 *tail_skb = tmp_skb; 1674 tail_skb = &(tmp_skb->next); 1675 skb->len += tmp_skb->len; 1676 skb->data_len += tmp_skb->len; 1677 skb->truesize += tmp_skb->truesize; 1678 tmp_skb->destructor = NULL; 1679 tmp_skb->sk = NULL; 1680 } 1681 1682 /* Allow local fragmentation. */ 1683 skb->ignore_df = ip6_sk_ignore_df(sk); 1684 1685 *final_dst = fl6->daddr; 1686 __skb_pull(skb, skb_network_header_len(skb)); 1687 if (opt && opt->opt_flen) 1688 ipv6_push_frag_opts(skb, opt, &proto); 1689 if (opt && opt->opt_nflen) 1690 ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr); 1691 1692 skb_push(skb, sizeof(struct ipv6hdr)); 1693 skb_reset_network_header(skb); 1694 hdr = ipv6_hdr(skb); 1695 1696 ip6_flow_hdr(hdr, v6_cork->tclass, 1697 ip6_make_flowlabel(net, skb, fl6->flowlabel, 1698 np->autoflowlabel, fl6)); 1699 hdr->hop_limit = v6_cork->hop_limit; 1700 hdr->nexthdr = proto; 1701 hdr->saddr = fl6->saddr; 1702 hdr->daddr = *final_dst; 1703 1704 skb->priority = sk->sk_priority; 1705 skb->mark = sk->sk_mark; 1706 1707 skb_dst_set(skb, dst_clone(&rt->dst)); 1708 IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len); 1709 if (proto == IPPROTO_ICMPV6) { 1710 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 1711 1712 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type); 1713 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS); 1714 } 1715 1716 ip6_cork_release(cork, v6_cork); 1717 out: 1718 return skb; 1719 } 1720 1721 int ip6_send_skb(struct sk_buff *skb) 1722 { 1723 struct net *net = sock_net(skb->sk); 1724 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 1725 int err; 1726 1727 err = ip6_local_out(net, skb->sk, skb); 1728 if (err) { 1729 if (err > 0) 1730 err = net_xmit_errno(err); 1731 if (err) 1732 IP6_INC_STATS(net, rt->rt6i_idev, 1733 IPSTATS_MIB_OUTDISCARDS); 1734 } 1735 1736 return err; 1737 } 1738 1739 int ip6_push_pending_frames(struct sock *sk) 1740 { 1741 struct sk_buff *skb; 1742 1743 skb = ip6_finish_skb(sk); 1744 if (!skb) 1745 return 0; 1746 1747 return ip6_send_skb(skb); 1748 } 1749 EXPORT_SYMBOL_GPL(ip6_push_pending_frames); 1750 1751 static void __ip6_flush_pending_frames(struct sock *sk, 1752 struct sk_buff_head *queue, 1753 struct inet_cork_full *cork, 1754 struct inet6_cork *v6_cork) 1755 { 1756 struct sk_buff *skb; 1757 1758 while ((skb = __skb_dequeue_tail(queue)) != NULL) { 1759 if (skb_dst(skb)) 1760 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)), 1761 IPSTATS_MIB_OUTDISCARDS); 1762 kfree_skb(skb); 1763 } 1764 1765 ip6_cork_release(cork, v6_cork); 1766 } 1767 1768 void ip6_flush_pending_frames(struct sock *sk) 1769 { 1770 __ip6_flush_pending_frames(sk, &sk->sk_write_queue, 1771 &inet_sk(sk)->cork, &inet6_sk(sk)->cork); 1772 } 1773 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames); 1774 1775 struct sk_buff *ip6_make_skb(struct sock *sk, 1776 int getfrag(void *from, char *to, int offset, 1777 int len, int odd, struct sk_buff *skb), 1778 void *from, int length, int transhdrlen, 1779 struct ipcm6_cookie *ipc6, struct flowi6 *fl6, 1780 struct rt6_info *rt, unsigned int flags, 1781 const struct sockcm_cookie *sockc) 1782 { 1783 struct inet_cork_full cork; 1784 struct inet6_cork v6_cork; 1785 struct sk_buff_head queue; 1786 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 1787 int err; 1788 1789 if (flags & MSG_PROBE) 1790 return NULL; 1791 1792 __skb_queue_head_init(&queue); 1793 1794 cork.base.flags = 0; 1795 cork.base.addr = 0; 1796 cork.base.opt = NULL; 1797 v6_cork.opt = NULL; 1798 err = ip6_setup_cork(sk, &cork, &v6_cork, ipc6, rt, fl6); 1799 if (err) 1800 return ERR_PTR(err); 1801 1802 if (ipc6->dontfrag < 0) 1803 ipc6->dontfrag = inet6_sk(sk)->dontfrag; 1804 1805 err = __ip6_append_data(sk, fl6, &queue, &cork.base, &v6_cork, 1806 ¤t->task_frag, getfrag, from, 1807 length + exthdrlen, transhdrlen + exthdrlen, 1808 flags, ipc6, sockc); 1809 if (err) { 1810 __ip6_flush_pending_frames(sk, &queue, &cork, &v6_cork); 1811 return ERR_PTR(err); 1812 } 1813 1814 return __ip6_make_skb(sk, &queue, &cork, &v6_cork); 1815 } 1816