1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * IPv6 output functions 4 * Linux INET6 implementation 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 * 9 * Based on linux/net/ipv4/ip_output.c 10 * 11 * Changes: 12 * A.N.Kuznetsov : airthmetics in fragmentation. 13 * extension headers are implemented. 14 * route changes now work. 15 * ip6_forward does not confuse sniffers. 16 * etc. 17 * 18 * H. von Brand : Added missing #include <linux/string.h> 19 * Imran Patel : frag id should be in NBO 20 * Kazunori MIYAZAWA @USAGI 21 * : add ip6_append_data and related functions 22 * for datagram xmit 23 */ 24 25 #include <linux/errno.h> 26 #include <linux/kernel.h> 27 #include <linux/string.h> 28 #include <linux/socket.h> 29 #include <linux/net.h> 30 #include <linux/netdevice.h> 31 #include <linux/if_arp.h> 32 #include <linux/in6.h> 33 #include <linux/tcp.h> 34 #include <linux/route.h> 35 #include <linux/module.h> 36 #include <linux/slab.h> 37 38 #include <linux/bpf-cgroup.h> 39 #include <linux/netfilter.h> 40 #include <linux/netfilter_ipv6.h> 41 42 #include <net/sock.h> 43 #include <net/snmp.h> 44 45 #include <net/gso.h> 46 #include <net/ipv6.h> 47 #include <net/ndisc.h> 48 #include <net/protocol.h> 49 #include <net/ip6_route.h> 50 #include <net/addrconf.h> 51 #include <net/rawv6.h> 52 #include <net/icmp.h> 53 #include <net/xfrm.h> 54 #include <net/checksum.h> 55 #include <linux/mroute6.h> 56 #include <net/l3mdev.h> 57 #include <net/lwtunnel.h> 58 #include <net/ip_tunnels.h> 59 60 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) 61 { 62 struct dst_entry *dst = skb_dst(skb); 63 struct net_device *dev = dst_dev_rcu(dst); 64 struct inet6_dev *idev = ip6_dst_idev(dst); 65 unsigned int hh_len = LL_RESERVED_SPACE(dev); 66 const struct in6_addr *daddr, *nexthop; 67 struct ipv6hdr *hdr; 68 struct neighbour *neigh; 69 int ret; 70 71 /* Be paranoid, rather than too clever. */ 72 if (unlikely(hh_len > skb_headroom(skb)) && dev->header_ops) { 73 /* idev stays alive because we hold rcu_read_lock(). */ 74 skb = skb_expand_head(skb, hh_len); 75 if (!skb) { 76 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 77 return -ENOMEM; 78 } 79 } 80 81 hdr = ipv6_hdr(skb); 82 daddr = &hdr->daddr; 83 if (unlikely(ipv6_addr_is_multicast(daddr))) { 84 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && 85 ((mroute6_is_socket(net, skb) && 86 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || 87 ipv6_chk_mcast_addr(dev, daddr, &hdr->saddr))) { 88 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 89 90 /* Do not check for IFF_ALLMULTI; multicast routing 91 is not supported in any case. 92 */ 93 if (newskb) 94 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING, 95 net, sk, newskb, NULL, newskb->dev, 96 dev_loopback_xmit); 97 98 if (hdr->hop_limit == 0) { 99 IP6_INC_STATS(net, idev, 100 IPSTATS_MIB_OUTDISCARDS); 101 kfree_skb(skb); 102 return 0; 103 } 104 } 105 106 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len); 107 if (IPV6_ADDR_MC_SCOPE(daddr) <= IPV6_ADDR_SCOPE_NODELOCAL && 108 !(dev->flags & IFF_LOOPBACK)) { 109 kfree_skb(skb); 110 return 0; 111 } 112 } 113 114 if (lwtunnel_xmit_redirect(dst->lwtstate)) { 115 int res = lwtunnel_xmit(skb); 116 117 if (res != LWTUNNEL_XMIT_CONTINUE) 118 return res; 119 hdr = ipv6_hdr(skb); 120 daddr = &hdr->daddr; 121 } 122 123 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len); 124 125 nexthop = rt6_nexthop(dst_rt6_info(dst), daddr); 126 neigh = __ipv6_neigh_lookup_noref(dev, nexthop); 127 128 if (IS_ERR_OR_NULL(neigh)) { 129 if (unlikely(!neigh)) 130 neigh = __neigh_create(&nd_tbl, nexthop, dev, false); 131 if (IS_ERR(neigh)) { 132 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTNOROUTES); 133 kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL); 134 return -EINVAL; 135 } 136 } 137 sock_confirm_neigh(skb, neigh); 138 ret = neigh_output(neigh, skb, false); 139 return ret; 140 } 141 142 static int 143 ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk, 144 struct sk_buff *skb, unsigned int mtu) 145 { 146 struct sk_buff *segs, *nskb; 147 netdev_features_t features; 148 int ret = 0; 149 150 /* Please see corresponding comment in ip_finish_output_gso 151 * describing the cases where GSO segment length exceeds the 152 * egress MTU. 153 */ 154 features = netif_skb_features(skb); 155 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 156 if (IS_ERR_OR_NULL(segs)) { 157 kfree_skb(skb); 158 return -ENOMEM; 159 } 160 161 consume_skb(skb); 162 163 skb_list_walk_safe(segs, segs, nskb) { 164 int err; 165 166 skb_mark_not_on_list(segs); 167 /* Last GSO segment can be smaller than gso_size (and MTU). 168 * Adding a fragment header would produce an "atomic fragment", 169 * which is considered harmful (RFC-8021). Avoid that. 170 */ 171 err = segs->len > mtu ? 172 ip6_fragment(net, sk, segs, ip6_finish_output2) : 173 ip6_finish_output2(net, sk, segs); 174 if (err && ret == 0) 175 ret = err; 176 } 177 178 return ret; 179 } 180 181 static int ip6_finish_output_gso(struct net *net, struct sock *sk, 182 struct sk_buff *skb, unsigned int mtu) 183 { 184 if (unlikely(!skb_gso_validate_network_len(skb, mtu))) 185 return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu); 186 187 return ip6_finish_output2(net, sk, skb); 188 } 189 190 static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 191 { 192 unsigned int mtu; 193 194 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM) 195 /* Policy lookup after SNAT yielded a new policy */ 196 if (skb_dst(skb)->xfrm) { 197 IP6CB(skb)->flags |= IP6SKB_REROUTED; 198 return dst_output(net, sk, skb); 199 } 200 #endif 201 202 mtu = ip6_skb_dst_mtu(skb); 203 if (skb_is_gso(skb)) 204 return ip6_finish_output_gso(net, sk, skb, mtu); 205 206 if (unlikely(skb->len > mtu || 207 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))) 208 return ip6_fragment(net, sk, skb, ip6_finish_output2); 209 210 return ip6_finish_output2(net, sk, skb); 211 } 212 213 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 214 { 215 int ret; 216 217 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); 218 switch (ret) { 219 case NET_XMIT_SUCCESS: 220 case NET_XMIT_CN: 221 return __ip6_finish_output(net, sk, skb) ? : ret; 222 default: 223 kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS); 224 return ret; 225 } 226 } 227 228 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb) 229 { 230 struct dst_entry *dst = skb_dst(skb); 231 struct net_device *dev, *indev = skb->dev; 232 struct inet6_dev *idev; 233 int ret; 234 235 skb->protocol = htons(ETH_P_IPV6); 236 rcu_read_lock(); 237 dev = dst_dev_rcu(dst); 238 idev = ip6_dst_idev(dst); 239 skb->dev = dev; 240 241 if (unlikely(!idev || READ_ONCE(idev->cnf.disable_ipv6))) { 242 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 243 rcu_read_unlock(); 244 kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED); 245 return 0; 246 } 247 248 ret = NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, 249 net, sk, skb, indev, dev, 250 ip6_finish_output, 251 !(IP6CB(skb)->flags & IP6SKB_REROUTED)); 252 rcu_read_unlock(); 253 return ret; 254 } 255 EXPORT_SYMBOL(ip6_output); 256 257 bool ip6_autoflowlabel(struct net *net, const struct sock *sk) 258 { 259 if (!inet6_test_bit(AUTOFLOWLABEL_SET, sk)) 260 return ip6_default_np_autolabel(net); 261 return inet6_test_bit(AUTOFLOWLABEL, sk); 262 } 263 264 int ip6_dst_hoplimit(struct dst_entry *dst) 265 { 266 int hoplimit = dst_metric_raw(dst, RTAX_HOPLIMIT); 267 268 rcu_read_lock(); 269 if (hoplimit == 0) { 270 struct net_device *dev = dst_dev_rcu(dst); 271 struct inet6_dev *idev; 272 273 idev = __in6_dev_get(dev); 274 if (idev) 275 hoplimit = READ_ONCE(idev->cnf.hop_limit); 276 else 277 hoplimit = READ_ONCE(dev_net(dev)->ipv6.devconf_all->hop_limit); 278 } 279 rcu_read_unlock(); 280 281 return hoplimit; 282 } 283 EXPORT_SYMBOL(ip6_dst_hoplimit); 284 285 /* 286 * xmit an sk_buff (used by TCP and SCTP) 287 * Note : socket lock is not held for SYNACK packets, but might be modified 288 * by calls to skb_set_owner_w() and ipv6_local_error(), 289 * which are using proper atomic operations or spinlocks. 290 */ 291 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, 292 __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority) 293 { 294 const struct ipv6_pinfo *np = inet6_sk(sk); 295 struct in6_addr *first_hop = &fl6->daddr; 296 struct dst_entry *dst = skb_dst(skb); 297 struct inet6_dev *idev = ip6_dst_idev(dst); 298 struct net *net = sock_net(sk); 299 unsigned int head_room; 300 struct net_device *dev; 301 struct ipv6hdr *hdr; 302 u8 proto = fl6->flowi6_proto; 303 int seg_len = skb->len; 304 int ret, hlimit = -1; 305 u32 mtu; 306 307 rcu_read_lock(); 308 309 dev = dst_dev_rcu(dst); 310 head_room = sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dev); 311 if (opt) 312 head_room += opt->opt_nflen + opt->opt_flen; 313 314 if (unlikely(head_room > skb_headroom(skb))) { 315 /* idev stays alive while we hold rcu_read_lock(). */ 316 skb = skb_expand_head(skb, head_room); 317 if (!skb) { 318 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 319 ret = -ENOBUFS; 320 goto unlock; 321 } 322 } 323 324 if (unlikely(opt)) { 325 seg_len += opt->opt_nflen + opt->opt_flen; 326 327 if (opt->opt_flen) 328 proto = ipv6_push_frag_opts(skb, opt, proto); 329 330 if (opt->opt_nflen) 331 proto = ipv6_push_nfrag_opts(skb, opt, proto, 332 &first_hop, 333 &fl6->saddr); 334 } 335 336 if (unlikely(seg_len > IPV6_MAXPLEN)) 337 seg_len = 0; 338 339 __skb_push(skb, sizeof(struct ipv6hdr)); 340 skb_reset_network_header(skb); 341 hdr = ipv6_hdr(skb); 342 343 /* 344 * Fill in the IPv6 header 345 */ 346 if (np) 347 hlimit = READ_ONCE(np->hop_limit); 348 if (hlimit < 0) 349 hlimit = ip6_dst_hoplimit(dst); 350 351 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel, 352 ip6_autoflowlabel(net, sk), fl6)); 353 354 hdr->payload_len = htons(seg_len); 355 hdr->nexthdr = proto; 356 hdr->hop_limit = hlimit; 357 358 hdr->saddr = fl6->saddr; 359 hdr->daddr = *first_hop; 360 361 skb->protocol = htons(ETH_P_IPV6); 362 skb->priority = priority; 363 skb->mark = mark; 364 365 mtu = dst6_mtu(dst); 366 if (likely((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb))) { 367 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS); 368 369 /* if egress device is enslaved to an L3 master device pass the 370 * skb to its handler for processing 371 */ 372 skb = l3mdev_ip6_out((struct sock *)sk, skb); 373 if (unlikely(!skb)) { 374 ret = 0; 375 goto unlock; 376 } 377 378 /* hooks should never assume socket lock is held. 379 * we promote our socket to non const 380 */ 381 ret = NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, 382 net, (struct sock *)sk, skb, NULL, dev, 383 dst_output); 384 goto unlock; 385 } 386 387 ret = -EMSGSIZE; 388 skb->dev = dev; 389 /* ipv6_local_error() does not require socket lock, 390 * we promote our socket to non const 391 */ 392 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu); 393 394 IP6_INC_STATS(net, idev, IPSTATS_MIB_FRAGFAILS); 395 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 396 unlock: 397 rcu_read_unlock(); 398 return ret; 399 } 400 EXPORT_SYMBOL(ip6_xmit); 401 402 static int ip6_call_ra_chain(struct sk_buff *skb, int sel) 403 { 404 struct ip6_ra_chain *ra; 405 struct sock *last = NULL; 406 407 read_lock(&ip6_ra_lock); 408 for (ra = ip6_ra_chain; ra; ra = ra->next) { 409 struct sock *sk = ra->sk; 410 if (sk && ra->sel == sel && 411 (!sk->sk_bound_dev_if || 412 sk->sk_bound_dev_if == skb->dev->ifindex)) { 413 414 if (inet6_test_bit(RTALERT_ISOLATE, sk) && 415 !net_eq(sock_net(sk), dev_net(skb->dev))) { 416 continue; 417 } 418 if (last) { 419 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 420 if (skb2) 421 rawv6_rcv(last, skb2); 422 } 423 last = sk; 424 } 425 } 426 427 if (last) { 428 rawv6_rcv(last, skb); 429 read_unlock(&ip6_ra_lock); 430 return 1; 431 } 432 read_unlock(&ip6_ra_lock); 433 return 0; 434 } 435 436 static int ip6_forward_proxy_check(struct sk_buff *skb) 437 { 438 struct ipv6hdr *hdr = ipv6_hdr(skb); 439 u8 nexthdr = hdr->nexthdr; 440 __be16 frag_off; 441 int offset; 442 443 if (ipv6_ext_hdr(nexthdr)) { 444 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off); 445 if (offset < 0) 446 return 0; 447 } else 448 offset = sizeof(struct ipv6hdr); 449 450 if (nexthdr == IPPROTO_ICMPV6) { 451 struct icmp6hdr *icmp6; 452 453 if (!pskb_may_pull(skb, (skb_network_header(skb) + 454 offset + 1 - skb->data))) 455 return 0; 456 457 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset); 458 459 switch (icmp6->icmp6_type) { 460 case NDISC_ROUTER_SOLICITATION: 461 case NDISC_ROUTER_ADVERTISEMENT: 462 case NDISC_NEIGHBOUR_SOLICITATION: 463 case NDISC_NEIGHBOUR_ADVERTISEMENT: 464 case NDISC_REDIRECT: 465 /* For reaction involving unicast neighbor discovery 466 * message destined to the proxied address, pass it to 467 * input function. 468 */ 469 return 1; 470 default: 471 break; 472 } 473 hdr = ipv6_hdr(skb); 474 } 475 476 /* 477 * The proxying router can't forward traffic sent to a link-local 478 * address, so signal the sender and discard the packet. This 479 * behavior is clarified by the MIPv6 specification. 480 */ 481 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) { 482 dst_link_failure(skb); 483 return -1; 484 } 485 486 return 0; 487 } 488 489 static inline int ip6_forward_finish(struct net *net, struct sock *sk, 490 struct sk_buff *skb) 491 { 492 #ifdef CONFIG_NET_SWITCHDEV 493 if (skb->offload_l3_fwd_mark) { 494 consume_skb(skb); 495 return 0; 496 } 497 #endif 498 499 skb_clear_tstamp(skb); 500 return dst_output(net, sk, skb); 501 } 502 503 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu) 504 { 505 if (skb->len <= mtu) 506 return false; 507 508 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */ 509 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu) 510 return true; 511 512 if (skb->ignore_df) 513 return false; 514 515 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 516 return false; 517 518 return true; 519 } 520 521 int ip6_forward(struct sk_buff *skb) 522 { 523 struct dst_entry *dst = skb_dst(skb); 524 struct ipv6hdr *hdr = ipv6_hdr(skb); 525 struct inet6_skb_parm *opt = IP6CB(skb); 526 struct net *net = dev_net(dst_dev(dst)); 527 struct net_device *dev; 528 struct inet6_dev *idev; 529 SKB_DR(reason); 530 u32 mtu; 531 532 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 533 if (!READ_ONCE(net->ipv6.devconf_all->forwarding) && 534 (!idev || !READ_ONCE(idev->cnf.force_forwarding))) 535 goto error; 536 537 if (skb->pkt_type != PACKET_HOST) 538 goto drop; 539 540 if (unlikely(skb->sk)) 541 goto drop; 542 543 if (skb_warn_if_lro(skb)) 544 goto drop; 545 546 if (!READ_ONCE(net->ipv6.devconf_all->disable_policy) && 547 (!idev || !READ_ONCE(idev->cnf.disable_policy)) && 548 !xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) { 549 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 550 goto drop; 551 } 552 553 skb_forward_csum(skb); 554 555 /* 556 * We DO NOT make any processing on 557 * RA packets, pushing them to user level AS IS 558 * without ane WARRANTY that application will be able 559 * to interpret them. The reason is that we 560 * cannot make anything clever here. 561 * 562 * We are not end-node, so that if packet contains 563 * AH/ESP, we cannot make anything. 564 * Defragmentation also would be mistake, RA packets 565 * cannot be fragmented, because there is no warranty 566 * that different fragments will go along one path. --ANK 567 */ 568 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) { 569 if (ip6_call_ra_chain(skb, ntohs(opt->ra))) 570 return 0; 571 } 572 573 /* 574 * check and decrement ttl 575 */ 576 if (hdr->hop_limit <= 1) { 577 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0); 578 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS); 579 580 kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR); 581 return -ETIMEDOUT; 582 } 583 584 /* XXX: idev->cnf.proxy_ndp? */ 585 if (READ_ONCE(net->ipv6.devconf_all->proxy_ndp) && 586 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev)) { 587 int proxied = ip6_forward_proxy_check(skb); 588 589 hdr = ipv6_hdr(skb); 590 if (proxied > 0) { 591 /* It's tempting to decrease the hop limit 592 * here by 1, as we do at the end of the 593 * function too. 594 * 595 * But that would be incorrect, as proxying is 596 * not forwarding. The ip6_input function 597 * will handle this packet locally, and it 598 * depends on the hop limit being unchanged. 599 * 600 * One example is the NDP hop limit, that 601 * always has to stay 255, but other would be 602 * similar checks around RA packets, where the 603 * user can even change the desired limit. 604 */ 605 return ip6_input(skb); 606 } else if (proxied < 0) { 607 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 608 goto drop; 609 } 610 } 611 612 if (!xfrm6_route_forward(skb)) { 613 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 614 SKB_DR_SET(reason, XFRM_POLICY); 615 goto drop; 616 } 617 dst = skb_dst(skb); 618 dev = dst_dev(dst); 619 /* IPv6 specs say nothing about it, but it is clear that we cannot 620 send redirects to source routed frames. 621 We don't send redirects to frames decapsulated from IPsec. 622 */ 623 if (IP6CB(skb)->iif == dev->ifindex && 624 opt->srcrt == 0 && !skb_sec_path(skb)) { 625 struct in6_addr *target = NULL; 626 struct inet_peer *peer; 627 struct rt6_info *rt; 628 629 /* 630 * incoming and outgoing devices are the same 631 * send a redirect. 632 */ 633 634 rt = dst_rt6_info(dst); 635 if (rt->rt6i_flags & RTF_GATEWAY) 636 target = &rt->rt6i_gateway; 637 else 638 target = &hdr->daddr; 639 640 rcu_read_lock(); 641 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr); 642 643 /* Limit redirects both by destination (here) 644 and by source (inside ndisc_send_redirect) 645 */ 646 if (peer && inet_peer_xrlim_allow(peer, 1*HZ)) 647 ndisc_send_redirect(skb, target); 648 rcu_read_unlock(); 649 } else { 650 int addrtype = ipv6_addr_type(&hdr->saddr); 651 652 /* This check is security critical. */ 653 if (addrtype == IPV6_ADDR_ANY || 654 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK)) 655 goto error; 656 if (addrtype & IPV6_ADDR_LINKLOCAL) { 657 icmpv6_send(skb, ICMPV6_DEST_UNREACH, 658 ICMPV6_NOT_NEIGHBOUR, 0); 659 goto error; 660 } 661 } 662 663 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS); 664 665 mtu = ip6_dst_mtu_maybe_forward(dst, true); 666 if (mtu < IPV6_MIN_MTU) 667 mtu = IPV6_MIN_MTU; 668 669 if (unlikely(ip6_pkt_too_big(skb, mtu))) { 670 /* Again, force OUTPUT device used as source address */ 671 skb->dev = dev; 672 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 673 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS); 674 __IP6_INC_STATS(net, ip6_dst_idev(dst), 675 IPSTATS_MIB_FRAGFAILS); 676 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 677 return -EMSGSIZE; 678 } 679 680 if (skb_cow(skb, dev->hard_header_len)) { 681 __IP6_INC_STATS(net, ip6_dst_idev(dst), 682 IPSTATS_MIB_OUTDISCARDS); 683 goto drop; 684 } 685 686 hdr = ipv6_hdr(skb); 687 688 /* Mangling hops number delayed to point after skb COW */ 689 690 hdr->hop_limit--; 691 692 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, 693 net, NULL, skb, skb->dev, dev, 694 ip6_forward_finish); 695 696 error: 697 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 698 SKB_DR_SET(reason, IP_INADDRERRORS); 699 drop: 700 kfree_skb_reason(skb, reason); 701 return -EINVAL; 702 } 703 704 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from) 705 { 706 to->pkt_type = from->pkt_type; 707 to->priority = from->priority; 708 to->protocol = from->protocol; 709 skb_dst_drop(to); 710 skb_dst_set(to, dst_clone(skb_dst(from))); 711 to->dev = from->dev; 712 to->mark = from->mark; 713 714 skb_copy_hash(to, from); 715 716 #ifdef CONFIG_NET_SCHED 717 to->tc_index = from->tc_index; 718 #endif 719 nf_copy(to, from); 720 skb_ext_copy(to, from); 721 skb_copy_secmark(to, from); 722 } 723 724 int ip6_fraglist_init(struct sk_buff *skb, unsigned int hlen, u8 *prevhdr, 725 u8 nexthdr, __be32 frag_id, 726 struct ip6_fraglist_iter *iter) 727 { 728 unsigned int first_len; 729 struct frag_hdr *fh; 730 731 /* BUILD HEADER */ 732 *prevhdr = NEXTHDR_FRAGMENT; 733 iter->tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC); 734 if (!iter->tmp_hdr) 735 return -ENOMEM; 736 737 iter->frag = skb_shinfo(skb)->frag_list; 738 skb_frag_list_init(skb); 739 740 iter->offset = 0; 741 iter->hlen = hlen; 742 iter->frag_id = frag_id; 743 iter->nexthdr = nexthdr; 744 745 __skb_pull(skb, hlen); 746 fh = __skb_push(skb, sizeof(struct frag_hdr)); 747 __skb_push(skb, hlen); 748 skb_reset_network_header(skb); 749 memcpy(skb_network_header(skb), iter->tmp_hdr, hlen); 750 751 fh->nexthdr = nexthdr; 752 fh->reserved = 0; 753 fh->frag_off = htons(IP6_MF); 754 fh->identification = frag_id; 755 756 first_len = skb_pagelen(skb); 757 skb->data_len = first_len - skb_headlen(skb); 758 skb->len = first_len; 759 ipv6_hdr(skb)->payload_len = htons(first_len - sizeof(struct ipv6hdr)); 760 761 return 0; 762 } 763 EXPORT_SYMBOL(ip6_fraglist_init); 764 765 void ip6_fraglist_prepare(struct sk_buff *skb, 766 struct ip6_fraglist_iter *iter) 767 { 768 struct sk_buff *frag = iter->frag; 769 unsigned int hlen = iter->hlen; 770 struct frag_hdr *fh; 771 772 frag->ip_summed = CHECKSUM_NONE; 773 skb_reset_transport_header(frag); 774 fh = __skb_push(frag, sizeof(struct frag_hdr)); 775 __skb_push(frag, hlen); 776 skb_reset_network_header(frag); 777 memcpy(skb_network_header(frag), iter->tmp_hdr, hlen); 778 iter->offset += skb->len - hlen - sizeof(struct frag_hdr); 779 fh->nexthdr = iter->nexthdr; 780 fh->reserved = 0; 781 fh->frag_off = htons(iter->offset); 782 if (frag->next) 783 fh->frag_off |= htons(IP6_MF); 784 fh->identification = iter->frag_id; 785 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); 786 ip6_copy_metadata(frag, skb); 787 } 788 EXPORT_SYMBOL(ip6_fraglist_prepare); 789 790 void ip6_frag_init(struct sk_buff *skb, unsigned int hlen, unsigned int mtu, 791 unsigned short needed_tailroom, int hdr_room, u8 *prevhdr, 792 u8 nexthdr, __be32 frag_id, struct ip6_frag_state *state) 793 { 794 state->prevhdr = prevhdr; 795 state->nexthdr = nexthdr; 796 state->frag_id = frag_id; 797 798 state->hlen = hlen; 799 state->mtu = mtu; 800 801 state->left = skb->len - hlen; /* Space per frame */ 802 state->ptr = hlen; /* Where to start from */ 803 804 state->hroom = hdr_room; 805 state->troom = needed_tailroom; 806 807 state->offset = 0; 808 } 809 EXPORT_SYMBOL(ip6_frag_init); 810 811 struct sk_buff *ip6_frag_next(struct sk_buff *skb, struct ip6_frag_state *state) 812 { 813 u8 *prevhdr = state->prevhdr, *fragnexthdr_offset; 814 struct sk_buff *frag; 815 struct frag_hdr *fh; 816 unsigned int len; 817 818 len = state->left; 819 /* IF: it doesn't fit, use 'mtu' - the data space left */ 820 if (len > state->mtu) 821 len = state->mtu; 822 /* IF: we are not sending up to and including the packet end 823 then align the next start on an eight byte boundary */ 824 if (len < state->left) 825 len &= ~7; 826 827 /* Allocate buffer */ 828 frag = alloc_skb(len + state->hlen + sizeof(struct frag_hdr) + 829 state->hroom + state->troom, GFP_ATOMIC); 830 if (!frag) 831 return ERR_PTR(-ENOMEM); 832 833 /* 834 * Set up data on packet 835 */ 836 837 ip6_copy_metadata(frag, skb); 838 skb_reserve(frag, state->hroom); 839 skb_put(frag, len + state->hlen + sizeof(struct frag_hdr)); 840 skb_reset_network_header(frag); 841 fh = (struct frag_hdr *)(skb_network_header(frag) + state->hlen); 842 frag->transport_header = (frag->network_header + state->hlen + 843 sizeof(struct frag_hdr)); 844 845 /* 846 * Charge the memory for the fragment to any owner 847 * it might possess 848 */ 849 if (skb->sk) 850 skb_set_owner_w(frag, skb->sk); 851 852 /* 853 * Copy the packet header into the new buffer. 854 */ 855 skb_copy_from_linear_data(skb, skb_network_header(frag), state->hlen); 856 857 fragnexthdr_offset = skb_network_header(frag); 858 fragnexthdr_offset += prevhdr - skb_network_header(skb); 859 *fragnexthdr_offset = NEXTHDR_FRAGMENT; 860 861 /* 862 * Build fragment header. 863 */ 864 fh->nexthdr = state->nexthdr; 865 fh->reserved = 0; 866 fh->identification = state->frag_id; 867 868 /* 869 * Copy a block of the IP datagram. 870 */ 871 BUG_ON(skb_copy_bits(skb, state->ptr, skb_transport_header(frag), 872 len)); 873 state->left -= len; 874 875 fh->frag_off = htons(state->offset); 876 if (state->left > 0) 877 fh->frag_off |= htons(IP6_MF); 878 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); 879 880 state->ptr += len; 881 state->offset += len; 882 883 return frag; 884 } 885 EXPORT_SYMBOL(ip6_frag_next); 886 887 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 888 int (*output)(struct net *, struct sock *, struct sk_buff *)) 889 { 890 struct sk_buff *frag; 891 struct rt6_info *rt = dst_rt6_info(skb_dst(skb)); 892 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? 893 inet6_sk(skb->sk) : NULL; 894 u8 tstamp_type = skb->tstamp_type; 895 struct ip6_frag_state state; 896 unsigned int mtu, hlen, nexthdr_offset; 897 ktime_t tstamp = skb->tstamp; 898 int hroom, err = 0; 899 __be32 frag_id; 900 u8 *prevhdr, nexthdr = 0; 901 902 if (!ipv6_mod_enabled()) { 903 kfree_skb(skb); 904 return -EAFNOSUPPORT; 905 } 906 907 err = ip6_find_1stfragopt(skb, &prevhdr); 908 if (err < 0) 909 goto fail; 910 hlen = err; 911 nexthdr = *prevhdr; 912 nexthdr_offset = prevhdr - skb_network_header(skb); 913 914 mtu = ip6_skb_dst_mtu(skb); 915 916 /* We must not fragment if the socket is set to force MTU discovery 917 * or if the skb it not generated by a local socket. 918 */ 919 if (unlikely(!skb->ignore_df && skb->len > mtu)) 920 goto fail_toobig; 921 922 if (IP6CB(skb)->frag_max_size) { 923 if (IP6CB(skb)->frag_max_size > mtu) 924 goto fail_toobig; 925 926 /* don't send fragments larger than what we received */ 927 mtu = IP6CB(skb)->frag_max_size; 928 if (mtu < IPV6_MIN_MTU) 929 mtu = IPV6_MIN_MTU; 930 } 931 932 if (np) { 933 u32 frag_size = READ_ONCE(np->frag_size); 934 935 if (frag_size && frag_size < mtu) 936 mtu = frag_size; 937 } 938 if (mtu < hlen + sizeof(struct frag_hdr) + 8) 939 goto fail_toobig; 940 mtu -= hlen + sizeof(struct frag_hdr); 941 942 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr, 943 &ipv6_hdr(skb)->saddr); 944 945 if (skb->ip_summed == CHECKSUM_PARTIAL && 946 (err = skb_checksum_help(skb))) 947 goto fail; 948 949 prevhdr = skb_network_header(skb) + nexthdr_offset; 950 hroom = LL_RESERVED_SPACE(rt->dst.dev); 951 if (skb_has_frag_list(skb)) { 952 unsigned int first_len = skb_pagelen(skb); 953 struct ip6_fraglist_iter iter; 954 struct sk_buff *frag2; 955 956 if (first_len - hlen > mtu || 957 ((first_len - hlen) & 7) || 958 skb_cloned(skb) || 959 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) 960 goto slow_path; 961 962 skb_walk_frags(skb, frag) { 963 /* Correct geometry. */ 964 if (frag->len > mtu || 965 ((frag->len & 7) && frag->next) || 966 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr))) 967 goto slow_path_clean; 968 969 /* Partially cloned skb? */ 970 if (skb_shared(frag)) 971 goto slow_path_clean; 972 973 BUG_ON(frag->sk); 974 if (skb->sk) { 975 frag->sk = skb->sk; 976 frag->destructor = sock_wfree; 977 } 978 skb->truesize -= frag->truesize; 979 } 980 981 err = ip6_fraglist_init(skb, hlen, prevhdr, nexthdr, frag_id, 982 &iter); 983 if (err < 0) 984 goto fail; 985 986 /* We prevent @rt from being freed. */ 987 rcu_read_lock(); 988 989 for (;;) { 990 /* Prepare header of the next frame, 991 * before previous one went down. */ 992 if (iter.frag) 993 ip6_fraglist_prepare(skb, &iter); 994 995 skb_set_delivery_time(skb, tstamp, tstamp_type); 996 err = output(net, sk, skb); 997 if (!err) 998 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 999 IPSTATS_MIB_FRAGCREATES); 1000 1001 if (err || !iter.frag) 1002 break; 1003 1004 skb = ip6_fraglist_next(&iter); 1005 } 1006 1007 kfree(iter.tmp_hdr); 1008 1009 if (err == 0) { 1010 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 1011 IPSTATS_MIB_FRAGOKS); 1012 rcu_read_unlock(); 1013 return 0; 1014 } 1015 1016 kfree_skb_list(iter.frag); 1017 1018 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 1019 IPSTATS_MIB_FRAGFAILS); 1020 rcu_read_unlock(); 1021 return err; 1022 1023 slow_path_clean: 1024 skb_walk_frags(skb, frag2) { 1025 if (frag2 == frag) 1026 break; 1027 frag2->sk = NULL; 1028 frag2->destructor = NULL; 1029 skb->truesize += frag2->truesize; 1030 } 1031 } 1032 1033 slow_path: 1034 /* 1035 * Fragment the datagram. 1036 */ 1037 1038 ip6_frag_init(skb, hlen, mtu, rt->dst.dev->needed_tailroom, 1039 LL_RESERVED_SPACE(rt->dst.dev), prevhdr, nexthdr, frag_id, 1040 &state); 1041 1042 /* 1043 * Keep copying data until we run out. 1044 */ 1045 1046 while (state.left > 0) { 1047 frag = ip6_frag_next(skb, &state); 1048 if (IS_ERR(frag)) { 1049 err = PTR_ERR(frag); 1050 goto fail; 1051 } 1052 1053 /* 1054 * Put this fragment into the sending queue. 1055 */ 1056 skb_set_delivery_time(frag, tstamp, tstamp_type); 1057 err = output(net, sk, frag); 1058 if (err) 1059 goto fail; 1060 1061 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1062 IPSTATS_MIB_FRAGCREATES); 1063 } 1064 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1065 IPSTATS_MIB_FRAGOKS); 1066 consume_skb(skb); 1067 return err; 1068 1069 fail_toobig: 1070 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 1071 err = -EMSGSIZE; 1072 1073 fail: 1074 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1075 IPSTATS_MIB_FRAGFAILS); 1076 kfree_skb(skb); 1077 return err; 1078 } 1079 EXPORT_SYMBOL_GPL(ip6_fragment); 1080 1081 static inline int ip6_rt_check(const struct rt6key *rt_key, 1082 const struct in6_addr *fl_addr, 1083 const struct in6_addr *addr_cache) 1084 { 1085 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) && 1086 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache)); 1087 } 1088 1089 static struct dst_entry *ip6_sk_dst_check(struct sock *sk, 1090 struct dst_entry *dst, 1091 const struct flowi6 *fl6) 1092 { 1093 struct ipv6_pinfo *np = inet6_sk(sk); 1094 struct rt6_info *rt; 1095 1096 if (!dst) 1097 goto out; 1098 1099 if (dst->ops->family != AF_INET6) { 1100 dst_release(dst); 1101 return NULL; 1102 } 1103 1104 rt = dst_rt6_info(dst); 1105 /* Yes, checking route validity in not connected 1106 * case is not very simple. Take into account, 1107 * that we do not support routing by source, TOS, 1108 * and MSG_DONTROUTE --ANK (980726) 1109 * 1110 * 1. ip6_rt_check(): If route was host route, 1111 * check that cached destination is current. 1112 * If it is network route, we still may 1113 * check its validity using saved pointer 1114 * to the last used address: daddr_cache. 1115 * We do not want to save whole address now, 1116 * (because main consumer of this service 1117 * is tcp, which has not this problem), 1118 * so that the last trick works only on connected 1119 * sockets. 1120 * 2. oif also should be the same. 1121 */ 1122 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, 1123 np->daddr_cache ? &sk->sk_v6_daddr : NULL) || 1124 #ifdef CONFIG_IPV6_SUBTREES 1125 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, 1126 np->saddr_cache ? &np->saddr : NULL) || 1127 #endif 1128 (fl6->flowi6_oif && fl6->flowi6_oif != dst_dev(dst)->ifindex)) { 1129 dst_release(dst); 1130 dst = NULL; 1131 } 1132 1133 out: 1134 return dst; 1135 } 1136 1137 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk, 1138 struct dst_entry **dst, struct flowi6 *fl6) 1139 { 1140 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 1141 struct neighbour *n; 1142 struct rt6_info *rt; 1143 #endif 1144 int err; 1145 int flags = 0; 1146 1147 /* The correct way to handle this would be to do 1148 * ip6_route_get_saddr, and then ip6_route_output; however, 1149 * the route-specific preferred source forces the 1150 * ip6_route_output call _before_ ip6_route_get_saddr. 1151 * 1152 * In source specific routing (no src=any default route), 1153 * ip6_route_output will fail given src=any saddr, though, so 1154 * that's why we try it again later. 1155 */ 1156 if (ipv6_addr_any(&fl6->saddr)) { 1157 struct fib6_info *from; 1158 struct rt6_info *rt; 1159 1160 *dst = ip6_route_output(net, sk, fl6); 1161 rt = (*dst)->error ? NULL : dst_rt6_info(*dst); 1162 1163 rcu_read_lock(); 1164 from = rt ? rcu_dereference(rt->from) : NULL; 1165 err = ip6_route_get_saddr(net, from, &fl6->daddr, 1166 sk ? READ_ONCE(inet6_sk(sk)->srcprefs) : 0, 1167 fl6->flowi6_l3mdev, 1168 &fl6->saddr); 1169 rcu_read_unlock(); 1170 1171 if (err) 1172 goto out_err_release; 1173 1174 /* If we had an erroneous initial result, pretend it 1175 * never existed and let the SA-enabled version take 1176 * over. 1177 */ 1178 if ((*dst)->error) { 1179 dst_release(*dst); 1180 *dst = NULL; 1181 } 1182 1183 if (fl6->flowi6_oif) 1184 flags |= RT6_LOOKUP_F_IFACE; 1185 } 1186 1187 if (!*dst) 1188 *dst = ip6_route_output_flags(net, sk, fl6, flags); 1189 1190 err = (*dst)->error; 1191 if (err) 1192 goto out_err_release; 1193 1194 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 1195 /* 1196 * Here if the dst entry we've looked up 1197 * has a neighbour entry that is in the INCOMPLETE 1198 * state and the src address from the flow is 1199 * marked as OPTIMISTIC, we release the found 1200 * dst entry and replace it instead with the 1201 * dst entry of the nexthop router 1202 */ 1203 rt = dst_rt6_info(*dst); 1204 rcu_read_lock(); 1205 n = __ipv6_neigh_lookup_noref(rt->dst.dev, 1206 rt6_nexthop(rt, &fl6->daddr)); 1207 err = n && !(READ_ONCE(n->nud_state) & NUD_VALID) ? -EINVAL : 0; 1208 rcu_read_unlock(); 1209 1210 if (err) { 1211 struct inet6_ifaddr *ifp; 1212 struct flowi6 fl_gw6; 1213 int redirect; 1214 1215 ifp = ipv6_get_ifaddr(net, &fl6->saddr, 1216 (*dst)->dev, 1); 1217 1218 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC); 1219 if (ifp) 1220 in6_ifa_put(ifp); 1221 1222 if (redirect) { 1223 /* 1224 * We need to get the dst entry for the 1225 * default router instead 1226 */ 1227 dst_release(*dst); 1228 memcpy(&fl_gw6, fl6, sizeof(struct flowi6)); 1229 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr)); 1230 *dst = ip6_route_output(net, sk, &fl_gw6); 1231 err = (*dst)->error; 1232 if (err) 1233 goto out_err_release; 1234 } 1235 } 1236 #endif 1237 if (ipv6_addr_v4mapped(&fl6->saddr) && 1238 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) { 1239 err = -EAFNOSUPPORT; 1240 goto out_err_release; 1241 } 1242 1243 return 0; 1244 1245 out_err_release: 1246 dst_release(*dst); 1247 *dst = NULL; 1248 1249 if (err == -ENETUNREACH) 1250 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES); 1251 return err; 1252 } 1253 1254 /** 1255 * ip6_dst_lookup - perform route lookup on flow 1256 * @net: Network namespace to perform lookup in 1257 * @sk: socket which provides route info 1258 * @dst: pointer to dst_entry * for result 1259 * @fl6: flow to lookup 1260 * 1261 * This function performs a route lookup on the given flow. 1262 * 1263 * It returns zero on success, or a standard errno code on error. 1264 */ 1265 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, 1266 struct flowi6 *fl6) 1267 { 1268 *dst = NULL; 1269 return ip6_dst_lookup_tail(net, sk, dst, fl6); 1270 } 1271 EXPORT_SYMBOL_GPL(ip6_dst_lookup); 1272 1273 /** 1274 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec 1275 * @net: Network namespace to perform lookup in 1276 * @sk: socket which provides route info 1277 * @fl6: flow to lookup 1278 * @final_dst: final destination address for ipsec lookup 1279 * 1280 * This function performs a route lookup on the given flow. 1281 * 1282 * It returns a valid dst pointer on success, or a pointer encoded 1283 * error code. 1284 */ 1285 struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, struct flowi6 *fl6, 1286 const struct in6_addr *final_dst) 1287 { 1288 struct dst_entry *dst = NULL; 1289 int err; 1290 1291 if (!ipv6_mod_enabled()) 1292 return ERR_PTR(-EAFNOSUPPORT); 1293 err = ip6_dst_lookup_tail(net, sk, &dst, fl6); 1294 if (err) 1295 return ERR_PTR(err); 1296 if (final_dst) 1297 fl6->daddr = *final_dst; 1298 1299 return xfrm_lookup_route(net, dst, flowi6_to_flowi(fl6), sk, 0); 1300 } 1301 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow); 1302 1303 /** 1304 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow 1305 * @sk: socket which provides the dst cache and route info 1306 * @fl6: flow to lookup 1307 * @final_dst: final destination address for ipsec lookup 1308 * @connected: whether @sk is connected or not 1309 * 1310 * This function performs a route lookup on the given flow with the 1311 * possibility of using the cached route in the socket if it is valid. 1312 * It will take the socket dst lock when operating on the dst cache. 1313 * As a result, this function can only be used in process context. 1314 * 1315 * In addition, for a connected socket, cache the dst in the socket 1316 * if the current cache is not valid. 1317 * 1318 * It returns a valid dst pointer on success, or a pointer encoded 1319 * error code. 1320 */ 1321 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6, 1322 const struct in6_addr *final_dst, 1323 bool connected) 1324 { 1325 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie); 1326 1327 dst = ip6_sk_dst_check(sk, dst, fl6); 1328 if (dst) 1329 return dst; 1330 1331 dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_dst); 1332 if (connected && !IS_ERR(dst)) 1333 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6); 1334 1335 return dst; 1336 } 1337 1338 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src, 1339 gfp_t gfp) 1340 { 1341 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1342 } 1343 1344 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src, 1345 gfp_t gfp) 1346 { 1347 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1348 } 1349 1350 static void ip6_append_data_mtu(unsigned int *mtu, 1351 int *maxfraglen, 1352 unsigned int fragheaderlen, 1353 struct sk_buff *skb, 1354 struct rt6_info *rt, 1355 unsigned int orig_mtu) 1356 { 1357 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) { 1358 if (!skb) { 1359 /* first fragment, reserve header_len */ 1360 *mtu = orig_mtu - rt->dst.header_len; 1361 1362 } else { 1363 /* 1364 * this fragment is not first, the headers 1365 * space is regarded as data space. 1366 */ 1367 *mtu = orig_mtu; 1368 } 1369 *maxfraglen = ((*mtu - fragheaderlen) & ~7) 1370 + fragheaderlen - sizeof(struct frag_hdr); 1371 } 1372 } 1373 1374 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, 1375 struct ipcm6_cookie *ipc6, 1376 struct rt6_info *rt) 1377 { 1378 struct ipv6_txoptions *nopt, *opt = ipc6->opt; 1379 struct inet6_cork *v6_cork = &cork->base6; 1380 struct ipv6_pinfo *np = inet6_sk(sk); 1381 unsigned int mtu, frag_size; 1382 1383 /* callers pass dst together with a reference, set it first so 1384 * ip6_cork_release() can put it down even in case of an error. 1385 */ 1386 cork->base.dst = &rt->dst; 1387 1388 /* 1389 * setup for corking 1390 */ 1391 if (unlikely(opt)) { 1392 if (WARN_ON(v6_cork->opt)) 1393 return -EINVAL; 1394 1395 nopt = v6_cork->opt = kzalloc_obj(*opt, sk->sk_allocation); 1396 if (unlikely(!nopt)) 1397 return -ENOBUFS; 1398 1399 nopt->tot_len = sizeof(*opt); 1400 nopt->opt_flen = opt->opt_flen; 1401 nopt->opt_nflen = opt->opt_nflen; 1402 1403 nopt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation); 1404 if (opt->dst0opt && !nopt->dst0opt) 1405 return -ENOBUFS; 1406 1407 nopt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation); 1408 if (opt->dst1opt && !nopt->dst1opt) 1409 return -ENOBUFS; 1410 1411 nopt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation); 1412 if (opt->hopopt && !nopt->hopopt) 1413 return -ENOBUFS; 1414 1415 nopt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation); 1416 if (opt->srcrt && !nopt->srcrt) 1417 return -ENOBUFS; 1418 1419 /* need source address above miyazawa*/ 1420 } 1421 v6_cork->hop_limit = ipc6->hlimit; 1422 v6_cork->tclass = ipc6->tclass; 1423 v6_cork->dontfrag = ipc6->dontfrag; 1424 if (rt->dst.flags & DST_XFRM_TUNNEL) 1425 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? 1426 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(&rt->dst); 1427 else 1428 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? 1429 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(xfrm_dst_path(&rt->dst)); 1430 1431 frag_size = READ_ONCE(np->frag_size); 1432 if (frag_size && frag_size < mtu) 1433 mtu = frag_size; 1434 1435 if (sk_is_udp(sk)) 1436 mtu = min(mtu, IP6_MAX_MTU); 1437 cork->base.fragsize = mtu; 1438 cork->base.gso_size = ipc6->gso_size; 1439 cork->base.tx_flags = 0; 1440 cork->base.mark = ipc6->sockc.mark; 1441 cork->base.priority = ipc6->sockc.priority; 1442 sock_tx_timestamp(sk, &ipc6->sockc, &cork->base.tx_flags); 1443 if (ipc6->sockc.tsflags & SOCKCM_FLAG_TS_OPT_ID) { 1444 cork->base.flags |= IPCORK_TS_OPT_ID; 1445 cork->base.ts_opt_id = ipc6->sockc.ts_opt_id; 1446 } 1447 cork->base.length = 0; 1448 cork->base.transmit_time = ipc6->sockc.transmit_time; 1449 1450 return 0; 1451 } 1452 1453 static int __ip6_append_data(struct sock *sk, 1454 struct sk_buff_head *queue, 1455 struct inet_cork_full *cork_full, 1456 struct page_frag *pfrag, 1457 int getfrag(void *from, char *to, int offset, 1458 int len, int odd, struct sk_buff *skb), 1459 void *from, size_t length, int transhdrlen, 1460 unsigned int flags) 1461 { 1462 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu; 1463 struct inet6_cork *v6_cork = &cork_full->base6; 1464 struct inet_cork *cork = &cork_full->base; 1465 struct flowi6 *fl6 = &cork_full->fl.u.ip6; 1466 struct sk_buff *skb, *skb_prev = NULL; 1467 struct ubuf_info *uarg = NULL; 1468 int exthdrlen = 0; 1469 int dst_exthdrlen = 0; 1470 int hh_len; 1471 int copy; 1472 int err; 1473 int offset = 0; 1474 bool zc = false; 1475 u32 tskey = 0; 1476 struct rt6_info *rt = dst_rt6_info(cork->dst); 1477 bool paged, hold_tskey = false, extra_uref = false; 1478 struct ipv6_txoptions *opt = v6_cork->opt; 1479 int csummode = CHECKSUM_NONE; 1480 unsigned int maxnonfragsize, headersize; 1481 unsigned int wmem_alloc_delta = 0; 1482 1483 skb = skb_peek_tail(queue); 1484 if (!skb) { 1485 exthdrlen = opt ? opt->opt_flen : 0; 1486 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; 1487 } 1488 1489 paged = !!cork->gso_size; 1490 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize; 1491 orig_mtu = mtu; 1492 1493 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1494 1495 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + 1496 (opt ? opt->opt_nflen : 0); 1497 1498 headersize = sizeof(struct ipv6hdr) + 1499 (opt ? opt->opt_flen + opt->opt_nflen : 0) + 1500 rt->rt6i_nfheader_len; 1501 1502 if (mtu <= fragheaderlen || 1503 ((mtu - fragheaderlen) & ~7) + fragheaderlen <= sizeof(struct frag_hdr)) 1504 goto emsgsize; 1505 1506 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - 1507 sizeof(struct frag_hdr); 1508 1509 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit 1510 * the first fragment 1511 */ 1512 if (headersize + transhdrlen > mtu) 1513 goto emsgsize; 1514 1515 if (cork->length + length > mtu - headersize && v6_cork->dontfrag && 1516 (sk->sk_protocol == IPPROTO_UDP || 1517 sk->sk_protocol == IPPROTO_ICMPV6 || 1518 sk->sk_protocol == IPPROTO_RAW)) { 1519 ipv6_local_rxpmtu(sk, fl6, mtu - headersize + 1520 sizeof(struct ipv6hdr)); 1521 goto emsgsize; 1522 } 1523 1524 if (ip6_sk_ignore_df(sk)) 1525 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN; 1526 else 1527 maxnonfragsize = mtu; 1528 1529 if (cork->length + length > maxnonfragsize - headersize) { 1530 emsgsize: 1531 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0); 1532 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu); 1533 return -EMSGSIZE; 1534 } 1535 1536 /* CHECKSUM_PARTIAL only with no extension headers and when 1537 * we are not going to fragment 1538 */ 1539 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP && 1540 headersize == sizeof(struct ipv6hdr) && 1541 length <= mtu - headersize && 1542 (!(flags & MSG_MORE) || cork->gso_size) && 1543 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 1544 csummode = CHECKSUM_PARTIAL; 1545 1546 if ((flags & MSG_ZEROCOPY) && length) { 1547 struct msghdr *msg = from; 1548 1549 if (getfrag == ip_generic_getfrag && msg->msg_ubuf) { 1550 if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb)) 1551 return -EINVAL; 1552 1553 /* Leave uarg NULL if can't zerocopy, callers should 1554 * be able to handle it. 1555 */ 1556 if ((rt->dst.dev->features & NETIF_F_SG) && 1557 csummode == CHECKSUM_PARTIAL) { 1558 paged = true; 1559 zc = true; 1560 uarg = msg->msg_ubuf; 1561 } 1562 } else if (sock_flag(sk, SOCK_ZEROCOPY)) { 1563 uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb), 1564 false); 1565 if (!uarg) 1566 return -ENOBUFS; 1567 extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ 1568 if (rt->dst.dev->features & NETIF_F_SG && 1569 csummode == CHECKSUM_PARTIAL) { 1570 paged = true; 1571 zc = true; 1572 } else { 1573 uarg_to_msgzc(uarg)->zerocopy = 0; 1574 skb_zcopy_set(skb, uarg, &extra_uref); 1575 } 1576 } 1577 } else if ((flags & MSG_SPLICE_PAGES) && length) { 1578 if (inet_test_bit(HDRINCL, sk)) 1579 return -EPERM; 1580 if (rt->dst.dev->features & NETIF_F_SG && 1581 getfrag == ip_generic_getfrag) 1582 /* We need an empty buffer to attach stuff to */ 1583 paged = true; 1584 else 1585 flags &= ~MSG_SPLICE_PAGES; 1586 } 1587 1588 if (cork->tx_flags & SKBTX_ANY_TSTAMP && 1589 READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) { 1590 if (cork->flags & IPCORK_TS_OPT_ID) { 1591 tskey = cork->ts_opt_id; 1592 } else { 1593 tskey = atomic_inc_return(&sk->sk_tskey) - 1; 1594 hold_tskey = true; 1595 } 1596 } 1597 1598 /* 1599 * Let's try using as much space as possible. 1600 * Use MTU if total length of the message fits into the MTU. 1601 * Otherwise, we need to reserve fragment header and 1602 * fragment alignment (= 8-15 octects, in total). 1603 * 1604 * Note that we may need to "move" the data from the tail 1605 * of the buffer to the new fragment when we split 1606 * the message. 1607 * 1608 * FIXME: It may be fragmented into multiple chunks 1609 * at once if non-fragmentable extension headers 1610 * are too large. 1611 * --yoshfuji 1612 */ 1613 1614 cork->length += length; 1615 if (!skb) 1616 goto alloc_new_skb; 1617 1618 while (length > 0) { 1619 /* Check if the remaining data fits into current packet. */ 1620 copy = (cork->length <= mtu ? mtu : maxfraglen) - skb->len; 1621 if (copy < length) 1622 copy = maxfraglen - skb->len; 1623 1624 if (copy <= 0) { 1625 char *data; 1626 unsigned int datalen; 1627 unsigned int fraglen; 1628 unsigned int fraggap; 1629 unsigned int alloclen, alloc_extra; 1630 unsigned int pagedlen; 1631 alloc_new_skb: 1632 /* There's no room in the current skb */ 1633 if (skb) 1634 fraggap = skb->len - maxfraglen; 1635 else 1636 fraggap = 0; 1637 /* update mtu and maxfraglen if necessary */ 1638 if (!skb || !skb_prev) 1639 ip6_append_data_mtu(&mtu, &maxfraglen, 1640 fragheaderlen, skb, rt, 1641 orig_mtu); 1642 1643 skb_prev = skb; 1644 1645 /* 1646 * If remaining data exceeds the mtu, 1647 * we know we need more fragment(s). 1648 */ 1649 datalen = length + fraggap; 1650 1651 if (datalen > (cork->length <= mtu ? mtu : maxfraglen) - fragheaderlen) 1652 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; 1653 fraglen = datalen + fragheaderlen; 1654 pagedlen = 0; 1655 1656 alloc_extra = hh_len; 1657 alloc_extra += dst_exthdrlen; 1658 alloc_extra += rt->dst.trailer_len; 1659 1660 /* We just reserve space for fragment header. 1661 * Note: this may be overallocation if the message 1662 * (without MSG_MORE) fits into the MTU. 1663 */ 1664 alloc_extra += sizeof(struct frag_hdr); 1665 1666 if ((flags & MSG_MORE) && 1667 !(rt->dst.dev->features&NETIF_F_SG)) 1668 alloclen = mtu; 1669 else if (!paged && 1670 (fraglen + alloc_extra < SKB_MAX_ALLOC || 1671 !(rt->dst.dev->features & NETIF_F_SG))) 1672 alloclen = fraglen; 1673 else { 1674 alloclen = fragheaderlen + transhdrlen + fraggap; 1675 pagedlen = datalen - transhdrlen - fraggap; 1676 } 1677 alloclen += alloc_extra; 1678 1679 if (datalen != length + fraggap) { 1680 /* 1681 * this is not the last fragment, the trailer 1682 * space is regarded as data space. 1683 */ 1684 datalen += rt->dst.trailer_len; 1685 } 1686 1687 fraglen = datalen + fragheaderlen; 1688 1689 copy = datalen - transhdrlen - fraggap - pagedlen; 1690 if (copy < 0) { 1691 err = -EINVAL; 1692 goto error; 1693 } 1694 if (transhdrlen) { 1695 skb = sock_alloc_send_skb(sk, alloclen, 1696 (flags & MSG_DONTWAIT), &err); 1697 } else { 1698 skb = NULL; 1699 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <= 1700 2 * sk->sk_sndbuf) 1701 skb = alloc_skb(alloclen, 1702 sk->sk_allocation); 1703 if (unlikely(!skb)) 1704 err = -ENOBUFS; 1705 } 1706 if (!skb) 1707 goto error; 1708 /* 1709 * Fill in the control structures 1710 */ 1711 skb->protocol = htons(ETH_P_IPV6); 1712 skb->ip_summed = csummode; 1713 skb->csum = 0; 1714 /* reserve for fragmentation and ipsec header */ 1715 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + 1716 dst_exthdrlen); 1717 1718 /* 1719 * Find where to start putting bytes 1720 */ 1721 data = skb_put(skb, fraglen - pagedlen); 1722 skb_set_network_header(skb, exthdrlen); 1723 data += fragheaderlen; 1724 skb->transport_header = (skb->network_header + 1725 fragheaderlen); 1726 if (fraggap) { 1727 skb->csum = skb_copy_and_csum_bits( 1728 skb_prev, maxfraglen, 1729 data + transhdrlen, fraggap); 1730 skb_prev->csum = csum_sub(skb_prev->csum, 1731 skb->csum); 1732 data += fraggap; 1733 pskb_trim_unique(skb_prev, maxfraglen); 1734 } 1735 if (copy > 0 && 1736 INDIRECT_CALL_1(getfrag, ip_generic_getfrag, 1737 from, data + transhdrlen, offset, 1738 copy, fraggap, skb) < 0) { 1739 err = -EFAULT; 1740 kfree_skb(skb); 1741 goto error; 1742 } else if (flags & MSG_SPLICE_PAGES) { 1743 copy = 0; 1744 } 1745 1746 offset += copy; 1747 length -= copy + transhdrlen; 1748 transhdrlen = 0; 1749 exthdrlen = 0; 1750 dst_exthdrlen = 0; 1751 1752 /* Only the initial fragment is time stamped */ 1753 skb_shinfo(skb)->tx_flags = cork->tx_flags; 1754 cork->tx_flags = 0; 1755 skb_shinfo(skb)->tskey = tskey; 1756 tskey = 0; 1757 skb_zcopy_set(skb, uarg, &extra_uref); 1758 1759 if ((flags & MSG_CONFIRM) && !skb_prev) 1760 skb_set_dst_pending_confirm(skb, 1); 1761 1762 /* 1763 * Put the packet on the pending queue 1764 */ 1765 if (!skb->destructor) { 1766 skb->destructor = sock_wfree; 1767 skb->sk = sk; 1768 wmem_alloc_delta += skb->truesize; 1769 } 1770 __skb_queue_tail(queue, skb); 1771 continue; 1772 } 1773 1774 if (copy > length) 1775 copy = length; 1776 1777 if (!(rt->dst.dev->features&NETIF_F_SG) && 1778 skb_tailroom(skb) >= copy) { 1779 unsigned int off; 1780 1781 off = skb->len; 1782 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag, 1783 from, skb_put(skb, copy), 1784 offset, copy, off, skb) < 0) { 1785 __skb_trim(skb, off); 1786 err = -EFAULT; 1787 goto error; 1788 } 1789 } else if (flags & MSG_SPLICE_PAGES) { 1790 struct msghdr *msg = from; 1791 1792 err = -EIO; 1793 if (WARN_ON_ONCE(copy > msg->msg_iter.count)) 1794 goto error; 1795 1796 err = skb_splice_from_iter(skb, &msg->msg_iter, copy); 1797 if (err < 0) 1798 goto error; 1799 copy = err; 1800 if (!(flags & MSG_NO_SHARED_FRAGS)) 1801 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; 1802 wmem_alloc_delta += copy; 1803 } else if (!zc) { 1804 int i = skb_shinfo(skb)->nr_frags; 1805 1806 err = -ENOMEM; 1807 if (!sk_page_frag_refill(sk, pfrag)) 1808 goto error; 1809 1810 skb_zcopy_downgrade_managed(skb); 1811 if (!skb_can_coalesce(skb, i, pfrag->page, 1812 pfrag->offset)) { 1813 err = -EMSGSIZE; 1814 if (i == MAX_SKB_FRAGS) 1815 goto error; 1816 1817 __skb_fill_page_desc(skb, i, pfrag->page, 1818 pfrag->offset, 0); 1819 skb_shinfo(skb)->nr_frags = ++i; 1820 get_page(pfrag->page); 1821 } 1822 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1823 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag, 1824 from, 1825 page_address(pfrag->page) + pfrag->offset, 1826 offset, copy, skb->len, skb) < 0) 1827 goto error_efault; 1828 1829 pfrag->offset += copy; 1830 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1831 skb->len += copy; 1832 skb->data_len += copy; 1833 skb->truesize += copy; 1834 wmem_alloc_delta += copy; 1835 } else { 1836 err = skb_zerocopy_iter_dgram(skb, from, copy); 1837 if (err < 0) 1838 goto error; 1839 } 1840 offset += copy; 1841 length -= copy; 1842 } 1843 1844 if (wmem_alloc_delta) 1845 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1846 return 0; 1847 1848 error_efault: 1849 err = -EFAULT; 1850 error: 1851 net_zcopy_put_abort(uarg, extra_uref); 1852 cork->length -= length; 1853 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); 1854 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1855 if (hold_tskey) 1856 atomic_dec(&sk->sk_tskey); 1857 return err; 1858 } 1859 1860 int ip6_append_data(struct sock *sk, 1861 int getfrag(void *from, char *to, int offset, int len, 1862 int odd, struct sk_buff *skb), 1863 void *from, size_t length, int transhdrlen, 1864 struct ipcm6_cookie *ipc6, struct flowi6 *fl6, 1865 struct rt6_info *rt, unsigned int flags) 1866 { 1867 struct inet_sock *inet = inet_sk(sk); 1868 int exthdrlen; 1869 int err; 1870 1871 if (flags&MSG_PROBE) 1872 return 0; 1873 if (skb_queue_empty(&sk->sk_write_queue)) { 1874 /* 1875 * setup for corking 1876 */ 1877 dst_hold(&rt->dst); 1878 err = ip6_setup_cork(sk, &inet->cork, 1879 ipc6, rt); 1880 if (err) 1881 return err; 1882 1883 inet->cork.fl.u.ip6 = *fl6; 1884 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 1885 length += exthdrlen; 1886 transhdrlen += exthdrlen; 1887 } else { 1888 transhdrlen = 0; 1889 } 1890 1891 return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork, 1892 sk_page_frag(sk), getfrag, 1893 from, length, transhdrlen, flags); 1894 } 1895 EXPORT_SYMBOL_GPL(ip6_append_data); 1896 1897 static void ip6_cork_steal_dst(struct sk_buff *skb, struct inet_cork_full *cork) 1898 { 1899 struct dst_entry *dst = cork->base.dst; 1900 1901 cork->base.dst = NULL; 1902 skb_dst_set(skb, dst); 1903 } 1904 1905 static void ip6_cork_release(struct inet_cork_full *cork) 1906 { 1907 struct inet6_cork *v6_cork = &cork->base6; 1908 1909 if (unlikely(v6_cork->opt)) { 1910 struct ipv6_txoptions *opt = v6_cork->opt; 1911 1912 kfree(opt->dst0opt); 1913 kfree(opt->dst1opt); 1914 kfree(opt->hopopt); 1915 kfree(opt->srcrt); 1916 kfree(opt); 1917 v6_cork->opt = NULL; 1918 } 1919 1920 if (cork->base.dst) { 1921 dst_release(cork->base.dst); 1922 cork->base.dst = NULL; 1923 } 1924 } 1925 1926 struct sk_buff *__ip6_make_skb(struct sock *sk, 1927 struct sk_buff_head *queue, 1928 struct inet_cork_full *cork) 1929 { 1930 struct sk_buff *skb, *tmp_skb; 1931 struct sk_buff **tail_skb; 1932 struct in6_addr *final_dst; 1933 struct net *net = sock_net(sk); 1934 struct ipv6hdr *hdr; 1935 struct ipv6_txoptions *opt; 1936 struct rt6_info *rt = dst_rt6_info(cork->base.dst); 1937 struct flowi6 *fl6 = &cork->fl.u.ip6; 1938 unsigned char proto = fl6->flowi6_proto; 1939 1940 skb = __skb_dequeue(queue); 1941 if (!skb) 1942 goto out; 1943 tail_skb = &(skb_shinfo(skb)->frag_list); 1944 1945 /* move skb->data to ip header from ext header */ 1946 if (skb->data < skb_network_header(skb)) 1947 __skb_pull(skb, skb_network_offset(skb)); 1948 while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1949 __skb_pull(tmp_skb, skb_network_header_len(skb)); 1950 *tail_skb = tmp_skb; 1951 tail_skb = &(tmp_skb->next); 1952 skb->len += tmp_skb->len; 1953 skb->data_len += tmp_skb->len; 1954 skb->truesize += tmp_skb->truesize; 1955 tmp_skb->destructor = NULL; 1956 tmp_skb->sk = NULL; 1957 } 1958 1959 /* Allow local fragmentation. */ 1960 skb->ignore_df = ip6_sk_ignore_df(sk); 1961 __skb_pull(skb, skb_network_header_len(skb)); 1962 1963 final_dst = &fl6->daddr; 1964 opt = cork->base6.opt; 1965 if (unlikely(opt)) { 1966 if (opt->opt_flen) 1967 proto = ipv6_push_frag_opts(skb, opt, proto); 1968 if (opt->opt_nflen) 1969 proto = ipv6_push_nfrag_opts(skb, opt, proto, 1970 &final_dst, &fl6->saddr); 1971 } 1972 skb_push(skb, sizeof(struct ipv6hdr)); 1973 skb_reset_network_header(skb); 1974 hdr = ipv6_hdr(skb); 1975 1976 ip6_flow_hdr(hdr, cork->base6.tclass, 1977 ip6_make_flowlabel(net, skb, fl6->flowlabel, 1978 ip6_autoflowlabel(net, sk), fl6)); 1979 hdr->hop_limit = cork->base6.hop_limit; 1980 hdr->nexthdr = proto; 1981 hdr->saddr = fl6->saddr; 1982 hdr->daddr = *final_dst; 1983 1984 skb->priority = cork->base.priority; 1985 skb->mark = cork->base.mark; 1986 if (sk_is_tcp(sk)) 1987 skb_set_delivery_time(skb, cork->base.transmit_time, SKB_CLOCK_MONOTONIC); 1988 else 1989 skb_set_delivery_type_by_clockid(skb, cork->base.transmit_time, sk->sk_clockid); 1990 1991 ip6_cork_steal_dst(skb, cork); 1992 IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS); 1993 if (unlikely(proto == IPPROTO_ICMPV6)) { 1994 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 1995 u8 icmp6_type; 1996 1997 if (sk->sk_socket->type == SOCK_RAW && 1998 !(fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH)) 1999 icmp6_type = fl6->fl6_icmp_type; 2000 else 2001 icmp6_type = icmp6_hdr(skb)->icmp6_type; 2002 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_type); 2003 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS); 2004 } 2005 2006 ip6_cork_release(cork); 2007 out: 2008 return skb; 2009 } 2010 2011 int ip6_send_skb(struct sk_buff *skb) 2012 { 2013 struct net *net = sock_net(skb->sk); 2014 struct rt6_info *rt = dst_rt6_info(skb_dst(skb)); 2015 int err; 2016 2017 rcu_read_lock(); 2018 err = ip6_local_out(net, skb->sk, skb); 2019 if (err) { 2020 if (err > 0) 2021 err = net_xmit_errno(err); 2022 if (err) 2023 IP6_INC_STATS(net, rt->rt6i_idev, 2024 IPSTATS_MIB_OUTDISCARDS); 2025 } 2026 2027 rcu_read_unlock(); 2028 return err; 2029 } 2030 2031 int ip6_push_pending_frames(struct sock *sk) 2032 { 2033 struct sk_buff *skb; 2034 2035 skb = ip6_finish_skb(sk); 2036 if (!skb) 2037 return 0; 2038 2039 return ip6_send_skb(skb); 2040 } 2041 EXPORT_SYMBOL_GPL(ip6_push_pending_frames); 2042 2043 static void __ip6_flush_pending_frames(struct sock *sk, 2044 struct sk_buff_head *queue, 2045 struct inet_cork_full *cork) 2046 { 2047 struct sk_buff *skb; 2048 2049 while ((skb = __skb_dequeue_tail(queue)) != NULL) { 2050 if (skb_dst(skb)) 2051 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)), 2052 IPSTATS_MIB_OUTDISCARDS); 2053 kfree_skb(skb); 2054 } 2055 2056 ip6_cork_release(cork); 2057 } 2058 2059 void ip6_flush_pending_frames(struct sock *sk) 2060 { 2061 __ip6_flush_pending_frames(sk, &sk->sk_write_queue, 2062 &inet_sk(sk)->cork); 2063 } 2064 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames); 2065 2066 struct sk_buff *ip6_make_skb(struct sock *sk, 2067 int getfrag(void *from, char *to, int offset, 2068 int len, int odd, struct sk_buff *skb), 2069 void *from, size_t length, int transhdrlen, 2070 struct ipcm6_cookie *ipc6, struct rt6_info *rt, 2071 unsigned int flags, struct inet_cork_full *cork) 2072 { 2073 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 2074 struct sk_buff_head queue; 2075 int err; 2076 2077 if (flags & MSG_PROBE) { 2078 dst_release(&rt->dst); 2079 return NULL; 2080 } 2081 2082 __skb_queue_head_init(&queue); 2083 2084 cork->base.flags = 0; 2085 cork->base.addr = 0; 2086 cork->base.opt = NULL; 2087 cork->base6.opt = NULL; 2088 err = ip6_setup_cork(sk, cork, ipc6, rt); 2089 if (err) { 2090 ip6_cork_release(cork); 2091 return ERR_PTR(err); 2092 } 2093 2094 err = __ip6_append_data(sk, &queue, cork, 2095 ¤t->task_frag, getfrag, from, 2096 length + exthdrlen, transhdrlen + exthdrlen, 2097 flags); 2098 if (err) { 2099 __ip6_flush_pending_frames(sk, &queue, cork); 2100 return ERR_PTR(err); 2101 } 2102 2103 return __ip6_make_skb(sk, &queue, cork); 2104 } 2105