1 /* 2 * IPv6 output functions 3 * Linux INET6 implementation 4 * 5 * Authors: 6 * Pedro Roque <roque@di.fc.ul.pt> 7 * 8 * Based on linux/net/ipv4/ip_output.c 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or (at your option) any later version. 14 * 15 * Changes: 16 * A.N.Kuznetsov : airthmetics in fragmentation. 17 * extension headers are implemented. 18 * route changes now work. 19 * ip6_forward does not confuse sniffers. 20 * etc. 21 * 22 * H. von Brand : Added missing #include <linux/string.h> 23 * Imran Patel : frag id should be in NBO 24 * Kazunori MIYAZAWA @USAGI 25 * : add ip6_append_data and related functions 26 * for datagram xmit 27 */ 28 29 #include <linux/errno.h> 30 #include <linux/kernel.h> 31 #include <linux/string.h> 32 #include <linux/socket.h> 33 #include <linux/net.h> 34 #include <linux/netdevice.h> 35 #include <linux/if_arp.h> 36 #include <linux/in6.h> 37 #include <linux/tcp.h> 38 #include <linux/route.h> 39 #include <linux/module.h> 40 #include <linux/slab.h> 41 42 #include <linux/netfilter.h> 43 #include <linux/netfilter_ipv6.h> 44 45 #include <net/sock.h> 46 #include <net/snmp.h> 47 48 #include <net/ipv6.h> 49 #include <net/ndisc.h> 50 #include <net/protocol.h> 51 #include <net/ip6_route.h> 52 #include <net/addrconf.h> 53 #include <net/rawv6.h> 54 #include <net/icmp.h> 55 #include <net/xfrm.h> 56 #include <net/checksum.h> 57 #include <linux/mroute6.h> 58 #include <net/l3mdev.h> 59 60 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) 61 { 62 struct dst_entry *dst = skb_dst(skb); 63 struct net_device *dev = dst->dev; 64 struct neighbour *neigh; 65 struct in6_addr *nexthop; 66 int ret; 67 68 skb->protocol = htons(ETH_P_IPV6); 69 skb->dev = dev; 70 71 if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) { 72 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 73 74 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && 75 ((mroute6_socket(net, skb) && 76 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || 77 ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr, 78 &ipv6_hdr(skb)->saddr))) { 79 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 80 81 /* Do not check for IFF_ALLMULTI; multicast routing 82 is not supported in any case. 83 */ 84 if (newskb) 85 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING, 86 net, sk, newskb, NULL, newskb->dev, 87 dev_loopback_xmit); 88 89 if (ipv6_hdr(skb)->hop_limit == 0) { 90 IP6_INC_STATS(net, idev, 91 IPSTATS_MIB_OUTDISCARDS); 92 kfree_skb(skb); 93 return 0; 94 } 95 } 96 97 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len); 98 99 if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <= 100 IPV6_ADDR_SCOPE_NODELOCAL && 101 !(dev->flags & IFF_LOOPBACK)) { 102 kfree_skb(skb); 103 return 0; 104 } 105 } 106 107 rcu_read_lock_bh(); 108 nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr); 109 neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop); 110 if (unlikely(!neigh)) 111 neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false); 112 if (!IS_ERR(neigh)) { 113 ret = dst_neigh_output(dst, neigh, skb); 114 rcu_read_unlock_bh(); 115 return ret; 116 } 117 rcu_read_unlock_bh(); 118 119 IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES); 120 kfree_skb(skb); 121 return -EINVAL; 122 } 123 124 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 125 { 126 if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) || 127 dst_allfrag(skb_dst(skb)) || 128 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)) 129 return ip6_fragment(net, sk, skb, ip6_finish_output2); 130 else 131 return ip6_finish_output2(net, sk, skb); 132 } 133 134 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb) 135 { 136 struct net_device *dev = skb_dst(skb)->dev; 137 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 138 139 if (unlikely(idev->cnf.disable_ipv6)) { 140 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 141 kfree_skb(skb); 142 return 0; 143 } 144 145 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, 146 net, sk, skb, NULL, dev, 147 ip6_finish_output, 148 !(IP6CB(skb)->flags & IP6SKB_REROUTED)); 149 } 150 151 /* 152 * xmit an sk_buff (used by TCP, SCTP and DCCP) 153 * Note : socket lock is not held for SYNACK packets, but might be modified 154 * by calls to skb_set_owner_w() and ipv6_local_error(), 155 * which are using proper atomic operations or spinlocks. 156 */ 157 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, 158 struct ipv6_txoptions *opt, int tclass) 159 { 160 struct net *net = sock_net(sk); 161 const struct ipv6_pinfo *np = inet6_sk(sk); 162 struct in6_addr *first_hop = &fl6->daddr; 163 struct dst_entry *dst = skb_dst(skb); 164 struct ipv6hdr *hdr; 165 u8 proto = fl6->flowi6_proto; 166 int seg_len = skb->len; 167 int hlimit = -1; 168 u32 mtu; 169 170 if (opt) { 171 unsigned int head_room; 172 173 /* First: exthdrs may take lots of space (~8K for now) 174 MAX_HEADER is not enough. 175 */ 176 head_room = opt->opt_nflen + opt->opt_flen; 177 seg_len += head_room; 178 head_room += sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev); 179 180 if (skb_headroom(skb) < head_room) { 181 struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room); 182 if (!skb2) { 183 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 184 IPSTATS_MIB_OUTDISCARDS); 185 kfree_skb(skb); 186 return -ENOBUFS; 187 } 188 consume_skb(skb); 189 skb = skb2; 190 /* skb_set_owner_w() changes sk->sk_wmem_alloc atomically, 191 * it is safe to call in our context (socket lock not held) 192 */ 193 skb_set_owner_w(skb, (struct sock *)sk); 194 } 195 if (opt->opt_flen) 196 ipv6_push_frag_opts(skb, opt, &proto); 197 if (opt->opt_nflen) 198 ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop); 199 } 200 201 skb_push(skb, sizeof(struct ipv6hdr)); 202 skb_reset_network_header(skb); 203 hdr = ipv6_hdr(skb); 204 205 /* 206 * Fill in the IPv6 header 207 */ 208 if (np) 209 hlimit = np->hop_limit; 210 if (hlimit < 0) 211 hlimit = ip6_dst_hoplimit(dst); 212 213 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel, 214 np->autoflowlabel, fl6)); 215 216 hdr->payload_len = htons(seg_len); 217 hdr->nexthdr = proto; 218 hdr->hop_limit = hlimit; 219 220 hdr->saddr = fl6->saddr; 221 hdr->daddr = *first_hop; 222 223 skb->protocol = htons(ETH_P_IPV6); 224 skb->priority = sk->sk_priority; 225 skb->mark = sk->sk_mark; 226 227 mtu = dst_mtu(dst); 228 if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) { 229 IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)), 230 IPSTATS_MIB_OUT, skb->len); 231 /* hooks should never assume socket lock is held. 232 * we promote our socket to non const 233 */ 234 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, 235 net, (struct sock *)sk, skb, NULL, dst->dev, 236 dst_output); 237 } 238 239 skb->dev = dst->dev; 240 /* ipv6_local_error() does not require socket lock, 241 * we promote our socket to non const 242 */ 243 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu); 244 245 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS); 246 kfree_skb(skb); 247 return -EMSGSIZE; 248 } 249 EXPORT_SYMBOL(ip6_xmit); 250 251 static int ip6_call_ra_chain(struct sk_buff *skb, int sel) 252 { 253 struct ip6_ra_chain *ra; 254 struct sock *last = NULL; 255 256 read_lock(&ip6_ra_lock); 257 for (ra = ip6_ra_chain; ra; ra = ra->next) { 258 struct sock *sk = ra->sk; 259 if (sk && ra->sel == sel && 260 (!sk->sk_bound_dev_if || 261 sk->sk_bound_dev_if == skb->dev->ifindex)) { 262 if (last) { 263 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 264 if (skb2) 265 rawv6_rcv(last, skb2); 266 } 267 last = sk; 268 } 269 } 270 271 if (last) { 272 rawv6_rcv(last, skb); 273 read_unlock(&ip6_ra_lock); 274 return 1; 275 } 276 read_unlock(&ip6_ra_lock); 277 return 0; 278 } 279 280 static int ip6_forward_proxy_check(struct sk_buff *skb) 281 { 282 struct ipv6hdr *hdr = ipv6_hdr(skb); 283 u8 nexthdr = hdr->nexthdr; 284 __be16 frag_off; 285 int offset; 286 287 if (ipv6_ext_hdr(nexthdr)) { 288 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off); 289 if (offset < 0) 290 return 0; 291 } else 292 offset = sizeof(struct ipv6hdr); 293 294 if (nexthdr == IPPROTO_ICMPV6) { 295 struct icmp6hdr *icmp6; 296 297 if (!pskb_may_pull(skb, (skb_network_header(skb) + 298 offset + 1 - skb->data))) 299 return 0; 300 301 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset); 302 303 switch (icmp6->icmp6_type) { 304 case NDISC_ROUTER_SOLICITATION: 305 case NDISC_ROUTER_ADVERTISEMENT: 306 case NDISC_NEIGHBOUR_SOLICITATION: 307 case NDISC_NEIGHBOUR_ADVERTISEMENT: 308 case NDISC_REDIRECT: 309 /* For reaction involving unicast neighbor discovery 310 * message destined to the proxied address, pass it to 311 * input function. 312 */ 313 return 1; 314 default: 315 break; 316 } 317 } 318 319 /* 320 * The proxying router can't forward traffic sent to a link-local 321 * address, so signal the sender and discard the packet. This 322 * behavior is clarified by the MIPv6 specification. 323 */ 324 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) { 325 dst_link_failure(skb); 326 return -1; 327 } 328 329 return 0; 330 } 331 332 static inline int ip6_forward_finish(struct net *net, struct sock *sk, 333 struct sk_buff *skb) 334 { 335 skb_sender_cpu_clear(skb); 336 return dst_output(net, sk, skb); 337 } 338 339 static unsigned int ip6_dst_mtu_forward(const struct dst_entry *dst) 340 { 341 unsigned int mtu; 342 struct inet6_dev *idev; 343 344 if (dst_metric_locked(dst, RTAX_MTU)) { 345 mtu = dst_metric_raw(dst, RTAX_MTU); 346 if (mtu) 347 return mtu; 348 } 349 350 mtu = IPV6_MIN_MTU; 351 rcu_read_lock(); 352 idev = __in6_dev_get(dst->dev); 353 if (idev) 354 mtu = idev->cnf.mtu6; 355 rcu_read_unlock(); 356 357 return mtu; 358 } 359 360 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu) 361 { 362 if (skb->len <= mtu) 363 return false; 364 365 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */ 366 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu) 367 return true; 368 369 if (skb->ignore_df) 370 return false; 371 372 if (skb_is_gso(skb) && skb_gso_network_seglen(skb) <= mtu) 373 return false; 374 375 return true; 376 } 377 378 int ip6_forward(struct sk_buff *skb) 379 { 380 struct dst_entry *dst = skb_dst(skb); 381 struct ipv6hdr *hdr = ipv6_hdr(skb); 382 struct inet6_skb_parm *opt = IP6CB(skb); 383 struct net *net = dev_net(dst->dev); 384 u32 mtu; 385 386 if (net->ipv6.devconf_all->forwarding == 0) 387 goto error; 388 389 if (skb->pkt_type != PACKET_HOST) 390 goto drop; 391 392 if (unlikely(skb->sk)) 393 goto drop; 394 395 if (skb_warn_if_lro(skb)) 396 goto drop; 397 398 if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) { 399 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), 400 IPSTATS_MIB_INDISCARDS); 401 goto drop; 402 } 403 404 skb_forward_csum(skb); 405 406 /* 407 * We DO NOT make any processing on 408 * RA packets, pushing them to user level AS IS 409 * without ane WARRANTY that application will be able 410 * to interpret them. The reason is that we 411 * cannot make anything clever here. 412 * 413 * We are not end-node, so that if packet contains 414 * AH/ESP, we cannot make anything. 415 * Defragmentation also would be mistake, RA packets 416 * cannot be fragmented, because there is no warranty 417 * that different fragments will go along one path. --ANK 418 */ 419 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) { 420 if (ip6_call_ra_chain(skb, ntohs(opt->ra))) 421 return 0; 422 } 423 424 /* 425 * check and decrement ttl 426 */ 427 if (hdr->hop_limit <= 1) { 428 /* Force OUTPUT device used as source address */ 429 skb->dev = dst->dev; 430 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0); 431 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), 432 IPSTATS_MIB_INHDRERRORS); 433 434 kfree_skb(skb); 435 return -ETIMEDOUT; 436 } 437 438 /* XXX: idev->cnf.proxy_ndp? */ 439 if (net->ipv6.devconf_all->proxy_ndp && 440 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) { 441 int proxied = ip6_forward_proxy_check(skb); 442 if (proxied > 0) 443 return ip6_input(skb); 444 else if (proxied < 0) { 445 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), 446 IPSTATS_MIB_INDISCARDS); 447 goto drop; 448 } 449 } 450 451 if (!xfrm6_route_forward(skb)) { 452 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), 453 IPSTATS_MIB_INDISCARDS); 454 goto drop; 455 } 456 dst = skb_dst(skb); 457 458 /* IPv6 specs say nothing about it, but it is clear that we cannot 459 send redirects to source routed frames. 460 We don't send redirects to frames decapsulated from IPsec. 461 */ 462 if (skb->dev == dst->dev && opt->srcrt == 0 && !skb_sec_path(skb)) { 463 struct in6_addr *target = NULL; 464 struct inet_peer *peer; 465 struct rt6_info *rt; 466 467 /* 468 * incoming and outgoing devices are the same 469 * send a redirect. 470 */ 471 472 rt = (struct rt6_info *) dst; 473 if (rt->rt6i_flags & RTF_GATEWAY) 474 target = &rt->rt6i_gateway; 475 else 476 target = &hdr->daddr; 477 478 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1); 479 480 /* Limit redirects both by destination (here) 481 and by source (inside ndisc_send_redirect) 482 */ 483 if (inet_peer_xrlim_allow(peer, 1*HZ)) 484 ndisc_send_redirect(skb, target); 485 if (peer) 486 inet_putpeer(peer); 487 } else { 488 int addrtype = ipv6_addr_type(&hdr->saddr); 489 490 /* This check is security critical. */ 491 if (addrtype == IPV6_ADDR_ANY || 492 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK)) 493 goto error; 494 if (addrtype & IPV6_ADDR_LINKLOCAL) { 495 icmpv6_send(skb, ICMPV6_DEST_UNREACH, 496 ICMPV6_NOT_NEIGHBOUR, 0); 497 goto error; 498 } 499 } 500 501 mtu = ip6_dst_mtu_forward(dst); 502 if (mtu < IPV6_MIN_MTU) 503 mtu = IPV6_MIN_MTU; 504 505 if (ip6_pkt_too_big(skb, mtu)) { 506 /* Again, force OUTPUT device used as source address */ 507 skb->dev = dst->dev; 508 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 509 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), 510 IPSTATS_MIB_INTOOBIGERRORS); 511 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), 512 IPSTATS_MIB_FRAGFAILS); 513 kfree_skb(skb); 514 return -EMSGSIZE; 515 } 516 517 if (skb_cow(skb, dst->dev->hard_header_len)) { 518 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), 519 IPSTATS_MIB_OUTDISCARDS); 520 goto drop; 521 } 522 523 hdr = ipv6_hdr(skb); 524 525 /* Mangling hops number delayed to point after skb COW */ 526 527 hdr->hop_limit--; 528 529 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS); 530 IP6_ADD_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len); 531 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, 532 net, NULL, skb, skb->dev, dst->dev, 533 ip6_forward_finish); 534 535 error: 536 IP6_INC_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_INADDRERRORS); 537 drop: 538 kfree_skb(skb); 539 return -EINVAL; 540 } 541 542 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from) 543 { 544 to->pkt_type = from->pkt_type; 545 to->priority = from->priority; 546 to->protocol = from->protocol; 547 skb_dst_drop(to); 548 skb_dst_set(to, dst_clone(skb_dst(from))); 549 to->dev = from->dev; 550 to->mark = from->mark; 551 552 #ifdef CONFIG_NET_SCHED 553 to->tc_index = from->tc_index; 554 #endif 555 nf_copy(to, from); 556 skb_copy_secmark(to, from); 557 } 558 559 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 560 int (*output)(struct net *, struct sock *, struct sk_buff *)) 561 { 562 struct sk_buff *frag; 563 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 564 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? 565 inet6_sk(skb->sk) : NULL; 566 struct ipv6hdr *tmp_hdr; 567 struct frag_hdr *fh; 568 unsigned int mtu, hlen, left, len; 569 int hroom, troom; 570 __be32 frag_id; 571 int ptr, offset = 0, err = 0; 572 u8 *prevhdr, nexthdr = 0; 573 574 hlen = ip6_find_1stfragopt(skb, &prevhdr); 575 nexthdr = *prevhdr; 576 577 mtu = ip6_skb_dst_mtu(skb); 578 579 /* We must not fragment if the socket is set to force MTU discovery 580 * or if the skb it not generated by a local socket. 581 */ 582 if (unlikely(!skb->ignore_df && skb->len > mtu)) 583 goto fail_toobig; 584 585 if (IP6CB(skb)->frag_max_size) { 586 if (IP6CB(skb)->frag_max_size > mtu) 587 goto fail_toobig; 588 589 /* don't send fragments larger than what we received */ 590 mtu = IP6CB(skb)->frag_max_size; 591 if (mtu < IPV6_MIN_MTU) 592 mtu = IPV6_MIN_MTU; 593 } 594 595 if (np && np->frag_size < mtu) { 596 if (np->frag_size) 597 mtu = np->frag_size; 598 } 599 if (mtu < hlen + sizeof(struct frag_hdr) + 8) 600 goto fail_toobig; 601 mtu -= hlen + sizeof(struct frag_hdr); 602 603 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr, 604 &ipv6_hdr(skb)->saddr); 605 606 if (skb->ip_summed == CHECKSUM_PARTIAL && 607 (err = skb_checksum_help(skb))) 608 goto fail; 609 610 hroom = LL_RESERVED_SPACE(rt->dst.dev); 611 if (skb_has_frag_list(skb)) { 612 int first_len = skb_pagelen(skb); 613 struct sk_buff *frag2; 614 615 if (first_len - hlen > mtu || 616 ((first_len - hlen) & 7) || 617 skb_cloned(skb) || 618 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) 619 goto slow_path; 620 621 skb_walk_frags(skb, frag) { 622 /* Correct geometry. */ 623 if (frag->len > mtu || 624 ((frag->len & 7) && frag->next) || 625 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr))) 626 goto slow_path_clean; 627 628 /* Partially cloned skb? */ 629 if (skb_shared(frag)) 630 goto slow_path_clean; 631 632 BUG_ON(frag->sk); 633 if (skb->sk) { 634 frag->sk = skb->sk; 635 frag->destructor = sock_wfree; 636 } 637 skb->truesize -= frag->truesize; 638 } 639 640 err = 0; 641 offset = 0; 642 /* BUILD HEADER */ 643 644 *prevhdr = NEXTHDR_FRAGMENT; 645 tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC); 646 if (!tmp_hdr) { 647 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 648 IPSTATS_MIB_FRAGFAILS); 649 err = -ENOMEM; 650 goto fail; 651 } 652 frag = skb_shinfo(skb)->frag_list; 653 skb_frag_list_init(skb); 654 655 __skb_pull(skb, hlen); 656 fh = (struct frag_hdr *)__skb_push(skb, sizeof(struct frag_hdr)); 657 __skb_push(skb, hlen); 658 skb_reset_network_header(skb); 659 memcpy(skb_network_header(skb), tmp_hdr, hlen); 660 661 fh->nexthdr = nexthdr; 662 fh->reserved = 0; 663 fh->frag_off = htons(IP6_MF); 664 fh->identification = frag_id; 665 666 first_len = skb_pagelen(skb); 667 skb->data_len = first_len - skb_headlen(skb); 668 skb->len = first_len; 669 ipv6_hdr(skb)->payload_len = htons(first_len - 670 sizeof(struct ipv6hdr)); 671 672 dst_hold(&rt->dst); 673 674 for (;;) { 675 /* Prepare header of the next frame, 676 * before previous one went down. */ 677 if (frag) { 678 frag->ip_summed = CHECKSUM_NONE; 679 skb_reset_transport_header(frag); 680 fh = (struct frag_hdr *)__skb_push(frag, sizeof(struct frag_hdr)); 681 __skb_push(frag, hlen); 682 skb_reset_network_header(frag); 683 memcpy(skb_network_header(frag), tmp_hdr, 684 hlen); 685 offset += skb->len - hlen - sizeof(struct frag_hdr); 686 fh->nexthdr = nexthdr; 687 fh->reserved = 0; 688 fh->frag_off = htons(offset); 689 if (frag->next) 690 fh->frag_off |= htons(IP6_MF); 691 fh->identification = frag_id; 692 ipv6_hdr(frag)->payload_len = 693 htons(frag->len - 694 sizeof(struct ipv6hdr)); 695 ip6_copy_metadata(frag, skb); 696 } 697 698 err = output(net, sk, skb); 699 if (!err) 700 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 701 IPSTATS_MIB_FRAGCREATES); 702 703 if (err || !frag) 704 break; 705 706 skb = frag; 707 frag = skb->next; 708 skb->next = NULL; 709 } 710 711 kfree(tmp_hdr); 712 713 if (err == 0) { 714 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 715 IPSTATS_MIB_FRAGOKS); 716 ip6_rt_put(rt); 717 return 0; 718 } 719 720 kfree_skb_list(frag); 721 722 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 723 IPSTATS_MIB_FRAGFAILS); 724 ip6_rt_put(rt); 725 return err; 726 727 slow_path_clean: 728 skb_walk_frags(skb, frag2) { 729 if (frag2 == frag) 730 break; 731 frag2->sk = NULL; 732 frag2->destructor = NULL; 733 skb->truesize += frag2->truesize; 734 } 735 } 736 737 slow_path: 738 left = skb->len - hlen; /* Space per frame */ 739 ptr = hlen; /* Where to start from */ 740 741 /* 742 * Fragment the datagram. 743 */ 744 745 *prevhdr = NEXTHDR_FRAGMENT; 746 troom = rt->dst.dev->needed_tailroom; 747 748 /* 749 * Keep copying data until we run out. 750 */ 751 while (left > 0) { 752 len = left; 753 /* IF: it doesn't fit, use 'mtu' - the data space left */ 754 if (len > mtu) 755 len = mtu; 756 /* IF: we are not sending up to and including the packet end 757 then align the next start on an eight byte boundary */ 758 if (len < left) { 759 len &= ~7; 760 } 761 762 /* Allocate buffer */ 763 frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) + 764 hroom + troom, GFP_ATOMIC); 765 if (!frag) { 766 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 767 IPSTATS_MIB_FRAGFAILS); 768 err = -ENOMEM; 769 goto fail; 770 } 771 772 /* 773 * Set up data on packet 774 */ 775 776 ip6_copy_metadata(frag, skb); 777 skb_reserve(frag, hroom); 778 skb_put(frag, len + hlen + sizeof(struct frag_hdr)); 779 skb_reset_network_header(frag); 780 fh = (struct frag_hdr *)(skb_network_header(frag) + hlen); 781 frag->transport_header = (frag->network_header + hlen + 782 sizeof(struct frag_hdr)); 783 784 /* 785 * Charge the memory for the fragment to any owner 786 * it might possess 787 */ 788 if (skb->sk) 789 skb_set_owner_w(frag, skb->sk); 790 791 /* 792 * Copy the packet header into the new buffer. 793 */ 794 skb_copy_from_linear_data(skb, skb_network_header(frag), hlen); 795 796 /* 797 * Build fragment header. 798 */ 799 fh->nexthdr = nexthdr; 800 fh->reserved = 0; 801 fh->identification = frag_id; 802 803 /* 804 * Copy a block of the IP datagram. 805 */ 806 BUG_ON(skb_copy_bits(skb, ptr, skb_transport_header(frag), 807 len)); 808 left -= len; 809 810 fh->frag_off = htons(offset); 811 if (left > 0) 812 fh->frag_off |= htons(IP6_MF); 813 ipv6_hdr(frag)->payload_len = htons(frag->len - 814 sizeof(struct ipv6hdr)); 815 816 ptr += len; 817 offset += len; 818 819 /* 820 * Put this fragment into the sending queue. 821 */ 822 err = output(net, sk, frag); 823 if (err) 824 goto fail; 825 826 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 827 IPSTATS_MIB_FRAGCREATES); 828 } 829 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 830 IPSTATS_MIB_FRAGOKS); 831 consume_skb(skb); 832 return err; 833 834 fail_toobig: 835 if (skb->sk && dst_allfrag(skb_dst(skb))) 836 sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK); 837 838 skb->dev = skb_dst(skb)->dev; 839 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 840 err = -EMSGSIZE; 841 842 fail: 843 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 844 IPSTATS_MIB_FRAGFAILS); 845 kfree_skb(skb); 846 return err; 847 } 848 849 static inline int ip6_rt_check(const struct rt6key *rt_key, 850 const struct in6_addr *fl_addr, 851 const struct in6_addr *addr_cache) 852 { 853 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) && 854 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache)); 855 } 856 857 static struct dst_entry *ip6_sk_dst_check(struct sock *sk, 858 struct dst_entry *dst, 859 const struct flowi6 *fl6) 860 { 861 struct ipv6_pinfo *np = inet6_sk(sk); 862 struct rt6_info *rt; 863 864 if (!dst) 865 goto out; 866 867 if (dst->ops->family != AF_INET6) { 868 dst_release(dst); 869 return NULL; 870 } 871 872 rt = (struct rt6_info *)dst; 873 /* Yes, checking route validity in not connected 874 * case is not very simple. Take into account, 875 * that we do not support routing by source, TOS, 876 * and MSG_DONTROUTE --ANK (980726) 877 * 878 * 1. ip6_rt_check(): If route was host route, 879 * check that cached destination is current. 880 * If it is network route, we still may 881 * check its validity using saved pointer 882 * to the last used address: daddr_cache. 883 * We do not want to save whole address now, 884 * (because main consumer of this service 885 * is tcp, which has not this problem), 886 * so that the last trick works only on connected 887 * sockets. 888 * 2. oif also should be the same. 889 */ 890 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) || 891 #ifdef CONFIG_IPV6_SUBTREES 892 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) || 893 #endif 894 (!(fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) && 895 (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex))) { 896 dst_release(dst); 897 dst = NULL; 898 } 899 900 out: 901 return dst; 902 } 903 904 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk, 905 struct dst_entry **dst, struct flowi6 *fl6) 906 { 907 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 908 struct neighbour *n; 909 struct rt6_info *rt; 910 #endif 911 int err; 912 int flags = 0; 913 914 /* The correct way to handle this would be to do 915 * ip6_route_get_saddr, and then ip6_route_output; however, 916 * the route-specific preferred source forces the 917 * ip6_route_output call _before_ ip6_route_get_saddr. 918 * 919 * In source specific routing (no src=any default route), 920 * ip6_route_output will fail given src=any saddr, though, so 921 * that's why we try it again later. 922 */ 923 if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) { 924 struct rt6_info *rt; 925 bool had_dst = *dst != NULL; 926 927 if (!had_dst) 928 *dst = ip6_route_output(net, sk, fl6); 929 rt = (*dst)->error ? NULL : (struct rt6_info *)*dst; 930 err = ip6_route_get_saddr(net, rt, &fl6->daddr, 931 sk ? inet6_sk(sk)->srcprefs : 0, 932 &fl6->saddr); 933 if (err) 934 goto out_err_release; 935 936 /* If we had an erroneous initial result, pretend it 937 * never existed and let the SA-enabled version take 938 * over. 939 */ 940 if (!had_dst && (*dst)->error) { 941 dst_release(*dst); 942 *dst = NULL; 943 } 944 945 if (fl6->flowi6_oif) 946 flags |= RT6_LOOKUP_F_IFACE; 947 } 948 949 if (!*dst) 950 *dst = ip6_route_output_flags(net, sk, fl6, flags); 951 952 err = (*dst)->error; 953 if (err) 954 goto out_err_release; 955 956 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 957 /* 958 * Here if the dst entry we've looked up 959 * has a neighbour entry that is in the INCOMPLETE 960 * state and the src address from the flow is 961 * marked as OPTIMISTIC, we release the found 962 * dst entry and replace it instead with the 963 * dst entry of the nexthop router 964 */ 965 rt = (struct rt6_info *) *dst; 966 rcu_read_lock_bh(); 967 n = __ipv6_neigh_lookup_noref(rt->dst.dev, 968 rt6_nexthop(rt, &fl6->daddr)); 969 err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0; 970 rcu_read_unlock_bh(); 971 972 if (err) { 973 struct inet6_ifaddr *ifp; 974 struct flowi6 fl_gw6; 975 int redirect; 976 977 ifp = ipv6_get_ifaddr(net, &fl6->saddr, 978 (*dst)->dev, 1); 979 980 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC); 981 if (ifp) 982 in6_ifa_put(ifp); 983 984 if (redirect) { 985 /* 986 * We need to get the dst entry for the 987 * default router instead 988 */ 989 dst_release(*dst); 990 memcpy(&fl_gw6, fl6, sizeof(struct flowi6)); 991 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr)); 992 *dst = ip6_route_output(net, sk, &fl_gw6); 993 err = (*dst)->error; 994 if (err) 995 goto out_err_release; 996 } 997 } 998 #endif 999 1000 return 0; 1001 1002 out_err_release: 1003 if (err == -ENETUNREACH) 1004 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES); 1005 dst_release(*dst); 1006 *dst = NULL; 1007 return err; 1008 } 1009 1010 /** 1011 * ip6_dst_lookup - perform route lookup on flow 1012 * @sk: socket which provides route info 1013 * @dst: pointer to dst_entry * for result 1014 * @fl6: flow to lookup 1015 * 1016 * This function performs a route lookup on the given flow. 1017 * 1018 * It returns zero on success, or a standard errno code on error. 1019 */ 1020 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, 1021 struct flowi6 *fl6) 1022 { 1023 *dst = NULL; 1024 return ip6_dst_lookup_tail(net, sk, dst, fl6); 1025 } 1026 EXPORT_SYMBOL_GPL(ip6_dst_lookup); 1027 1028 /** 1029 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec 1030 * @sk: socket which provides route info 1031 * @fl6: flow to lookup 1032 * @final_dst: final destination address for ipsec lookup 1033 * 1034 * This function performs a route lookup on the given flow. 1035 * 1036 * It returns a valid dst pointer on success, or a pointer encoded 1037 * error code. 1038 */ 1039 struct dst_entry *ip6_dst_lookup_flow(const struct sock *sk, struct flowi6 *fl6, 1040 const struct in6_addr *final_dst) 1041 { 1042 struct dst_entry *dst = NULL; 1043 int err; 1044 1045 err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6); 1046 if (err) 1047 return ERR_PTR(err); 1048 if (final_dst) 1049 fl6->daddr = *final_dst; 1050 if (!fl6->flowi6_oif) 1051 fl6->flowi6_oif = l3mdev_fib_oif(dst->dev); 1052 1053 return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0); 1054 } 1055 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow); 1056 1057 /** 1058 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow 1059 * @sk: socket which provides the dst cache and route info 1060 * @fl6: flow to lookup 1061 * @final_dst: final destination address for ipsec lookup 1062 * 1063 * This function performs a route lookup on the given flow with the 1064 * possibility of using the cached route in the socket if it is valid. 1065 * It will take the socket dst lock when operating on the dst cache. 1066 * As a result, this function can only be used in process context. 1067 * 1068 * It returns a valid dst pointer on success, or a pointer encoded 1069 * error code. 1070 */ 1071 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6, 1072 const struct in6_addr *final_dst) 1073 { 1074 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie); 1075 int err; 1076 1077 dst = ip6_sk_dst_check(sk, dst, fl6); 1078 1079 err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6); 1080 if (err) 1081 return ERR_PTR(err); 1082 if (final_dst) 1083 fl6->daddr = *final_dst; 1084 1085 return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0); 1086 } 1087 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow); 1088 1089 static inline int ip6_ufo_append_data(struct sock *sk, 1090 struct sk_buff_head *queue, 1091 int getfrag(void *from, char *to, int offset, int len, 1092 int odd, struct sk_buff *skb), 1093 void *from, int length, int hh_len, int fragheaderlen, 1094 int transhdrlen, int mtu, unsigned int flags, 1095 const struct flowi6 *fl6) 1096 1097 { 1098 struct sk_buff *skb; 1099 int err; 1100 1101 /* There is support for UDP large send offload by network 1102 * device, so create one single skb packet containing complete 1103 * udp datagram 1104 */ 1105 skb = skb_peek_tail(queue); 1106 if (!skb) { 1107 skb = sock_alloc_send_skb(sk, 1108 hh_len + fragheaderlen + transhdrlen + 20, 1109 (flags & MSG_DONTWAIT), &err); 1110 if (!skb) 1111 return err; 1112 1113 /* reserve space for Hardware header */ 1114 skb_reserve(skb, hh_len); 1115 1116 /* create space for UDP/IP header */ 1117 skb_put(skb, fragheaderlen + transhdrlen); 1118 1119 /* initialize network header pointer */ 1120 skb_reset_network_header(skb); 1121 1122 /* initialize protocol header pointer */ 1123 skb->transport_header = skb->network_header + fragheaderlen; 1124 1125 skb->protocol = htons(ETH_P_IPV6); 1126 skb->csum = 0; 1127 1128 __skb_queue_tail(queue, skb); 1129 } else if (skb_is_gso(skb)) { 1130 goto append; 1131 } 1132 1133 skb->ip_summed = CHECKSUM_PARTIAL; 1134 /* Specify the length of each IPv6 datagram fragment. 1135 * It has to be a multiple of 8. 1136 */ 1137 skb_shinfo(skb)->gso_size = (mtu - fragheaderlen - 1138 sizeof(struct frag_hdr)) & ~7; 1139 skb_shinfo(skb)->gso_type = SKB_GSO_UDP; 1140 skb_shinfo(skb)->ip6_frag_id = ipv6_select_ident(sock_net(sk), 1141 &fl6->daddr, 1142 &fl6->saddr); 1143 1144 append: 1145 return skb_append_datato_frags(sk, skb, getfrag, from, 1146 (length - transhdrlen)); 1147 } 1148 1149 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src, 1150 gfp_t gfp) 1151 { 1152 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1153 } 1154 1155 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src, 1156 gfp_t gfp) 1157 { 1158 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1159 } 1160 1161 static void ip6_append_data_mtu(unsigned int *mtu, 1162 int *maxfraglen, 1163 unsigned int fragheaderlen, 1164 struct sk_buff *skb, 1165 struct rt6_info *rt, 1166 unsigned int orig_mtu) 1167 { 1168 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) { 1169 if (!skb) { 1170 /* first fragment, reserve header_len */ 1171 *mtu = orig_mtu - rt->dst.header_len; 1172 1173 } else { 1174 /* 1175 * this fragment is not first, the headers 1176 * space is regarded as data space. 1177 */ 1178 *mtu = orig_mtu; 1179 } 1180 *maxfraglen = ((*mtu - fragheaderlen) & ~7) 1181 + fragheaderlen - sizeof(struct frag_hdr); 1182 } 1183 } 1184 1185 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, 1186 struct inet6_cork *v6_cork, 1187 int hlimit, int tclass, struct ipv6_txoptions *opt, 1188 struct rt6_info *rt, struct flowi6 *fl6) 1189 { 1190 struct ipv6_pinfo *np = inet6_sk(sk); 1191 unsigned int mtu; 1192 1193 /* 1194 * setup for corking 1195 */ 1196 if (opt) { 1197 if (WARN_ON(v6_cork->opt)) 1198 return -EINVAL; 1199 1200 v6_cork->opt = kzalloc(opt->tot_len, sk->sk_allocation); 1201 if (unlikely(!v6_cork->opt)) 1202 return -ENOBUFS; 1203 1204 v6_cork->opt->tot_len = opt->tot_len; 1205 v6_cork->opt->opt_flen = opt->opt_flen; 1206 v6_cork->opt->opt_nflen = opt->opt_nflen; 1207 1208 v6_cork->opt->dst0opt = ip6_opt_dup(opt->dst0opt, 1209 sk->sk_allocation); 1210 if (opt->dst0opt && !v6_cork->opt->dst0opt) 1211 return -ENOBUFS; 1212 1213 v6_cork->opt->dst1opt = ip6_opt_dup(opt->dst1opt, 1214 sk->sk_allocation); 1215 if (opt->dst1opt && !v6_cork->opt->dst1opt) 1216 return -ENOBUFS; 1217 1218 v6_cork->opt->hopopt = ip6_opt_dup(opt->hopopt, 1219 sk->sk_allocation); 1220 if (opt->hopopt && !v6_cork->opt->hopopt) 1221 return -ENOBUFS; 1222 1223 v6_cork->opt->srcrt = ip6_rthdr_dup(opt->srcrt, 1224 sk->sk_allocation); 1225 if (opt->srcrt && !v6_cork->opt->srcrt) 1226 return -ENOBUFS; 1227 1228 /* need source address above miyazawa*/ 1229 } 1230 dst_hold(&rt->dst); 1231 cork->base.dst = &rt->dst; 1232 cork->fl.u.ip6 = *fl6; 1233 v6_cork->hop_limit = hlimit; 1234 v6_cork->tclass = tclass; 1235 if (rt->dst.flags & DST_XFRM_TUNNEL) 1236 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1237 rt->dst.dev->mtu : dst_mtu(&rt->dst); 1238 else 1239 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1240 rt->dst.dev->mtu : dst_mtu(rt->dst.path); 1241 if (np->frag_size < mtu) { 1242 if (np->frag_size) 1243 mtu = np->frag_size; 1244 } 1245 cork->base.fragsize = mtu; 1246 if (dst_allfrag(rt->dst.path)) 1247 cork->base.flags |= IPCORK_ALLFRAG; 1248 cork->base.length = 0; 1249 1250 return 0; 1251 } 1252 1253 static int __ip6_append_data(struct sock *sk, 1254 struct flowi6 *fl6, 1255 struct sk_buff_head *queue, 1256 struct inet_cork *cork, 1257 struct inet6_cork *v6_cork, 1258 struct page_frag *pfrag, 1259 int getfrag(void *from, char *to, int offset, 1260 int len, int odd, struct sk_buff *skb), 1261 void *from, int length, int transhdrlen, 1262 unsigned int flags, int dontfrag) 1263 { 1264 struct sk_buff *skb, *skb_prev = NULL; 1265 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu; 1266 int exthdrlen = 0; 1267 int dst_exthdrlen = 0; 1268 int hh_len; 1269 int copy; 1270 int err; 1271 int offset = 0; 1272 __u8 tx_flags = 0; 1273 u32 tskey = 0; 1274 struct rt6_info *rt = (struct rt6_info *)cork->dst; 1275 struct ipv6_txoptions *opt = v6_cork->opt; 1276 int csummode = CHECKSUM_NONE; 1277 unsigned int maxnonfragsize, headersize; 1278 1279 skb = skb_peek_tail(queue); 1280 if (!skb) { 1281 exthdrlen = opt ? opt->opt_flen : 0; 1282 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; 1283 } 1284 1285 mtu = cork->fragsize; 1286 orig_mtu = mtu; 1287 1288 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1289 1290 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + 1291 (opt ? opt->opt_nflen : 0); 1292 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - 1293 sizeof(struct frag_hdr); 1294 1295 headersize = sizeof(struct ipv6hdr) + 1296 (opt ? opt->opt_flen + opt->opt_nflen : 0) + 1297 (dst_allfrag(&rt->dst) ? 1298 sizeof(struct frag_hdr) : 0) + 1299 rt->rt6i_nfheader_len; 1300 1301 if (cork->length + length > mtu - headersize && dontfrag && 1302 (sk->sk_protocol == IPPROTO_UDP || 1303 sk->sk_protocol == IPPROTO_RAW)) { 1304 ipv6_local_rxpmtu(sk, fl6, mtu - headersize + 1305 sizeof(struct ipv6hdr)); 1306 goto emsgsize; 1307 } 1308 1309 if (ip6_sk_ignore_df(sk)) 1310 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN; 1311 else 1312 maxnonfragsize = mtu; 1313 1314 if (cork->length + length > maxnonfragsize - headersize) { 1315 emsgsize: 1316 ipv6_local_error(sk, EMSGSIZE, fl6, 1317 mtu - headersize + 1318 sizeof(struct ipv6hdr)); 1319 return -EMSGSIZE; 1320 } 1321 1322 /* CHECKSUM_PARTIAL only with no extension headers and when 1323 * we are not going to fragment 1324 */ 1325 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP && 1326 headersize == sizeof(struct ipv6hdr) && 1327 length < mtu - headersize && 1328 !(flags & MSG_MORE) && 1329 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 1330 csummode = CHECKSUM_PARTIAL; 1331 1332 if (sk->sk_type == SOCK_DGRAM || sk->sk_type == SOCK_RAW) { 1333 sock_tx_timestamp(sk, &tx_flags); 1334 if (tx_flags & SKBTX_ANY_SW_TSTAMP && 1335 sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) 1336 tskey = sk->sk_tskey++; 1337 } 1338 1339 /* 1340 * Let's try using as much space as possible. 1341 * Use MTU if total length of the message fits into the MTU. 1342 * Otherwise, we need to reserve fragment header and 1343 * fragment alignment (= 8-15 octects, in total). 1344 * 1345 * Note that we may need to "move" the data from the tail of 1346 * of the buffer to the new fragment when we split 1347 * the message. 1348 * 1349 * FIXME: It may be fragmented into multiple chunks 1350 * at once if non-fragmentable extension headers 1351 * are too large. 1352 * --yoshfuji 1353 */ 1354 1355 cork->length += length; 1356 if (((length > mtu) || 1357 (skb && skb_is_gso(skb))) && 1358 (sk->sk_protocol == IPPROTO_UDP) && 1359 (rt->dst.dev->features & NETIF_F_UFO) && 1360 (sk->sk_type == SOCK_DGRAM) && !udp_get_no_check6_tx(sk)) { 1361 err = ip6_ufo_append_data(sk, queue, getfrag, from, length, 1362 hh_len, fragheaderlen, 1363 transhdrlen, mtu, flags, fl6); 1364 if (err) 1365 goto error; 1366 return 0; 1367 } 1368 1369 if (!skb) 1370 goto alloc_new_skb; 1371 1372 while (length > 0) { 1373 /* Check if the remaining data fits into current packet. */ 1374 copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len; 1375 if (copy < length) 1376 copy = maxfraglen - skb->len; 1377 1378 if (copy <= 0) { 1379 char *data; 1380 unsigned int datalen; 1381 unsigned int fraglen; 1382 unsigned int fraggap; 1383 unsigned int alloclen; 1384 alloc_new_skb: 1385 /* There's no room in the current skb */ 1386 if (skb) 1387 fraggap = skb->len - maxfraglen; 1388 else 1389 fraggap = 0; 1390 /* update mtu and maxfraglen if necessary */ 1391 if (!skb || !skb_prev) 1392 ip6_append_data_mtu(&mtu, &maxfraglen, 1393 fragheaderlen, skb, rt, 1394 orig_mtu); 1395 1396 skb_prev = skb; 1397 1398 /* 1399 * If remaining data exceeds the mtu, 1400 * we know we need more fragment(s). 1401 */ 1402 datalen = length + fraggap; 1403 1404 if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) 1405 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; 1406 if ((flags & MSG_MORE) && 1407 !(rt->dst.dev->features&NETIF_F_SG)) 1408 alloclen = mtu; 1409 else 1410 alloclen = datalen + fragheaderlen; 1411 1412 alloclen += dst_exthdrlen; 1413 1414 if (datalen != length + fraggap) { 1415 /* 1416 * this is not the last fragment, the trailer 1417 * space is regarded as data space. 1418 */ 1419 datalen += rt->dst.trailer_len; 1420 } 1421 1422 alloclen += rt->dst.trailer_len; 1423 fraglen = datalen + fragheaderlen; 1424 1425 /* 1426 * We just reserve space for fragment header. 1427 * Note: this may be overallocation if the message 1428 * (without MSG_MORE) fits into the MTU. 1429 */ 1430 alloclen += sizeof(struct frag_hdr); 1431 1432 if (transhdrlen) { 1433 skb = sock_alloc_send_skb(sk, 1434 alloclen + hh_len, 1435 (flags & MSG_DONTWAIT), &err); 1436 } else { 1437 skb = NULL; 1438 if (atomic_read(&sk->sk_wmem_alloc) <= 1439 2 * sk->sk_sndbuf) 1440 skb = sock_wmalloc(sk, 1441 alloclen + hh_len, 1, 1442 sk->sk_allocation); 1443 if (unlikely(!skb)) 1444 err = -ENOBUFS; 1445 } 1446 if (!skb) 1447 goto error; 1448 /* 1449 * Fill in the control structures 1450 */ 1451 skb->protocol = htons(ETH_P_IPV6); 1452 skb->ip_summed = csummode; 1453 skb->csum = 0; 1454 /* reserve for fragmentation and ipsec header */ 1455 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + 1456 dst_exthdrlen); 1457 1458 /* Only the initial fragment is time stamped */ 1459 skb_shinfo(skb)->tx_flags = tx_flags; 1460 tx_flags = 0; 1461 skb_shinfo(skb)->tskey = tskey; 1462 tskey = 0; 1463 1464 /* 1465 * Find where to start putting bytes 1466 */ 1467 data = skb_put(skb, fraglen); 1468 skb_set_network_header(skb, exthdrlen); 1469 data += fragheaderlen; 1470 skb->transport_header = (skb->network_header + 1471 fragheaderlen); 1472 if (fraggap) { 1473 skb->csum = skb_copy_and_csum_bits( 1474 skb_prev, maxfraglen, 1475 data + transhdrlen, fraggap, 0); 1476 skb_prev->csum = csum_sub(skb_prev->csum, 1477 skb->csum); 1478 data += fraggap; 1479 pskb_trim_unique(skb_prev, maxfraglen); 1480 } 1481 copy = datalen - transhdrlen - fraggap; 1482 1483 if (copy < 0) { 1484 err = -EINVAL; 1485 kfree_skb(skb); 1486 goto error; 1487 } else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { 1488 err = -EFAULT; 1489 kfree_skb(skb); 1490 goto error; 1491 } 1492 1493 offset += copy; 1494 length -= datalen - fraggap; 1495 transhdrlen = 0; 1496 exthdrlen = 0; 1497 dst_exthdrlen = 0; 1498 1499 /* 1500 * Put the packet on the pending queue 1501 */ 1502 __skb_queue_tail(queue, skb); 1503 continue; 1504 } 1505 1506 if (copy > length) 1507 copy = length; 1508 1509 if (!(rt->dst.dev->features&NETIF_F_SG)) { 1510 unsigned int off; 1511 1512 off = skb->len; 1513 if (getfrag(from, skb_put(skb, copy), 1514 offset, copy, off, skb) < 0) { 1515 __skb_trim(skb, off); 1516 err = -EFAULT; 1517 goto error; 1518 } 1519 } else { 1520 int i = skb_shinfo(skb)->nr_frags; 1521 1522 err = -ENOMEM; 1523 if (!sk_page_frag_refill(sk, pfrag)) 1524 goto error; 1525 1526 if (!skb_can_coalesce(skb, i, pfrag->page, 1527 pfrag->offset)) { 1528 err = -EMSGSIZE; 1529 if (i == MAX_SKB_FRAGS) 1530 goto error; 1531 1532 __skb_fill_page_desc(skb, i, pfrag->page, 1533 pfrag->offset, 0); 1534 skb_shinfo(skb)->nr_frags = ++i; 1535 get_page(pfrag->page); 1536 } 1537 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1538 if (getfrag(from, 1539 page_address(pfrag->page) + pfrag->offset, 1540 offset, copy, skb->len, skb) < 0) 1541 goto error_efault; 1542 1543 pfrag->offset += copy; 1544 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1545 skb->len += copy; 1546 skb->data_len += copy; 1547 skb->truesize += copy; 1548 atomic_add(copy, &sk->sk_wmem_alloc); 1549 } 1550 offset += copy; 1551 length -= copy; 1552 } 1553 1554 return 0; 1555 1556 error_efault: 1557 err = -EFAULT; 1558 error: 1559 cork->length -= length; 1560 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); 1561 return err; 1562 } 1563 1564 int ip6_append_data(struct sock *sk, 1565 int getfrag(void *from, char *to, int offset, int len, 1566 int odd, struct sk_buff *skb), 1567 void *from, int length, int transhdrlen, int hlimit, 1568 int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6, 1569 struct rt6_info *rt, unsigned int flags, int dontfrag) 1570 { 1571 struct inet_sock *inet = inet_sk(sk); 1572 struct ipv6_pinfo *np = inet6_sk(sk); 1573 int exthdrlen; 1574 int err; 1575 1576 if (flags&MSG_PROBE) 1577 return 0; 1578 if (skb_queue_empty(&sk->sk_write_queue)) { 1579 /* 1580 * setup for corking 1581 */ 1582 err = ip6_setup_cork(sk, &inet->cork, &np->cork, hlimit, 1583 tclass, opt, rt, fl6); 1584 if (err) 1585 return err; 1586 1587 exthdrlen = (opt ? opt->opt_flen : 0); 1588 length += exthdrlen; 1589 transhdrlen += exthdrlen; 1590 } else { 1591 fl6 = &inet->cork.fl.u.ip6; 1592 transhdrlen = 0; 1593 } 1594 1595 return __ip6_append_data(sk, fl6, &sk->sk_write_queue, &inet->cork.base, 1596 &np->cork, sk_page_frag(sk), getfrag, 1597 from, length, transhdrlen, flags, dontfrag); 1598 } 1599 EXPORT_SYMBOL_GPL(ip6_append_data); 1600 1601 static void ip6_cork_release(struct inet_cork_full *cork, 1602 struct inet6_cork *v6_cork) 1603 { 1604 if (v6_cork->opt) { 1605 kfree(v6_cork->opt->dst0opt); 1606 kfree(v6_cork->opt->dst1opt); 1607 kfree(v6_cork->opt->hopopt); 1608 kfree(v6_cork->opt->srcrt); 1609 kfree(v6_cork->opt); 1610 v6_cork->opt = NULL; 1611 } 1612 1613 if (cork->base.dst) { 1614 dst_release(cork->base.dst); 1615 cork->base.dst = NULL; 1616 cork->base.flags &= ~IPCORK_ALLFRAG; 1617 } 1618 memset(&cork->fl, 0, sizeof(cork->fl)); 1619 } 1620 1621 struct sk_buff *__ip6_make_skb(struct sock *sk, 1622 struct sk_buff_head *queue, 1623 struct inet_cork_full *cork, 1624 struct inet6_cork *v6_cork) 1625 { 1626 struct sk_buff *skb, *tmp_skb; 1627 struct sk_buff **tail_skb; 1628 struct in6_addr final_dst_buf, *final_dst = &final_dst_buf; 1629 struct ipv6_pinfo *np = inet6_sk(sk); 1630 struct net *net = sock_net(sk); 1631 struct ipv6hdr *hdr; 1632 struct ipv6_txoptions *opt = v6_cork->opt; 1633 struct rt6_info *rt = (struct rt6_info *)cork->base.dst; 1634 struct flowi6 *fl6 = &cork->fl.u.ip6; 1635 unsigned char proto = fl6->flowi6_proto; 1636 1637 skb = __skb_dequeue(queue); 1638 if (!skb) 1639 goto out; 1640 tail_skb = &(skb_shinfo(skb)->frag_list); 1641 1642 /* move skb->data to ip header from ext header */ 1643 if (skb->data < skb_network_header(skb)) 1644 __skb_pull(skb, skb_network_offset(skb)); 1645 while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1646 __skb_pull(tmp_skb, skb_network_header_len(skb)); 1647 *tail_skb = tmp_skb; 1648 tail_skb = &(tmp_skb->next); 1649 skb->len += tmp_skb->len; 1650 skb->data_len += tmp_skb->len; 1651 skb->truesize += tmp_skb->truesize; 1652 tmp_skb->destructor = NULL; 1653 tmp_skb->sk = NULL; 1654 } 1655 1656 /* Allow local fragmentation. */ 1657 skb->ignore_df = ip6_sk_ignore_df(sk); 1658 1659 *final_dst = fl6->daddr; 1660 __skb_pull(skb, skb_network_header_len(skb)); 1661 if (opt && opt->opt_flen) 1662 ipv6_push_frag_opts(skb, opt, &proto); 1663 if (opt && opt->opt_nflen) 1664 ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst); 1665 1666 skb_push(skb, sizeof(struct ipv6hdr)); 1667 skb_reset_network_header(skb); 1668 hdr = ipv6_hdr(skb); 1669 1670 ip6_flow_hdr(hdr, v6_cork->tclass, 1671 ip6_make_flowlabel(net, skb, fl6->flowlabel, 1672 np->autoflowlabel, fl6)); 1673 hdr->hop_limit = v6_cork->hop_limit; 1674 hdr->nexthdr = proto; 1675 hdr->saddr = fl6->saddr; 1676 hdr->daddr = *final_dst; 1677 1678 skb->priority = sk->sk_priority; 1679 skb->mark = sk->sk_mark; 1680 1681 skb_dst_set(skb, dst_clone(&rt->dst)); 1682 IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len); 1683 if (proto == IPPROTO_ICMPV6) { 1684 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 1685 1686 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type); 1687 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS); 1688 } 1689 1690 ip6_cork_release(cork, v6_cork); 1691 out: 1692 return skb; 1693 } 1694 1695 int ip6_send_skb(struct sk_buff *skb) 1696 { 1697 struct net *net = sock_net(skb->sk); 1698 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 1699 int err; 1700 1701 err = ip6_local_out(net, skb->sk, skb); 1702 if (err) { 1703 if (err > 0) 1704 err = net_xmit_errno(err); 1705 if (err) 1706 IP6_INC_STATS(net, rt->rt6i_idev, 1707 IPSTATS_MIB_OUTDISCARDS); 1708 } 1709 1710 return err; 1711 } 1712 1713 int ip6_push_pending_frames(struct sock *sk) 1714 { 1715 struct sk_buff *skb; 1716 1717 skb = ip6_finish_skb(sk); 1718 if (!skb) 1719 return 0; 1720 1721 return ip6_send_skb(skb); 1722 } 1723 EXPORT_SYMBOL_GPL(ip6_push_pending_frames); 1724 1725 static void __ip6_flush_pending_frames(struct sock *sk, 1726 struct sk_buff_head *queue, 1727 struct inet_cork_full *cork, 1728 struct inet6_cork *v6_cork) 1729 { 1730 struct sk_buff *skb; 1731 1732 while ((skb = __skb_dequeue_tail(queue)) != NULL) { 1733 if (skb_dst(skb)) 1734 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)), 1735 IPSTATS_MIB_OUTDISCARDS); 1736 kfree_skb(skb); 1737 } 1738 1739 ip6_cork_release(cork, v6_cork); 1740 } 1741 1742 void ip6_flush_pending_frames(struct sock *sk) 1743 { 1744 __ip6_flush_pending_frames(sk, &sk->sk_write_queue, 1745 &inet_sk(sk)->cork, &inet6_sk(sk)->cork); 1746 } 1747 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames); 1748 1749 struct sk_buff *ip6_make_skb(struct sock *sk, 1750 int getfrag(void *from, char *to, int offset, 1751 int len, int odd, struct sk_buff *skb), 1752 void *from, int length, int transhdrlen, 1753 int hlimit, int tclass, 1754 struct ipv6_txoptions *opt, struct flowi6 *fl6, 1755 struct rt6_info *rt, unsigned int flags, 1756 int dontfrag) 1757 { 1758 struct inet_cork_full cork; 1759 struct inet6_cork v6_cork; 1760 struct sk_buff_head queue; 1761 int exthdrlen = (opt ? opt->opt_flen : 0); 1762 int err; 1763 1764 if (flags & MSG_PROBE) 1765 return NULL; 1766 1767 __skb_queue_head_init(&queue); 1768 1769 cork.base.flags = 0; 1770 cork.base.addr = 0; 1771 cork.base.opt = NULL; 1772 v6_cork.opt = NULL; 1773 err = ip6_setup_cork(sk, &cork, &v6_cork, hlimit, tclass, opt, rt, fl6); 1774 if (err) 1775 return ERR_PTR(err); 1776 1777 if (dontfrag < 0) 1778 dontfrag = inet6_sk(sk)->dontfrag; 1779 1780 err = __ip6_append_data(sk, fl6, &queue, &cork.base, &v6_cork, 1781 ¤t->task_frag, getfrag, from, 1782 length + exthdrlen, transhdrlen + exthdrlen, 1783 flags, dontfrag); 1784 if (err) { 1785 __ip6_flush_pending_frames(sk, &queue, &cork, &v6_cork); 1786 return ERR_PTR(err); 1787 } 1788 1789 return __ip6_make_skb(sk, &queue, &cork, &v6_cork); 1790 } 1791