1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * IPv6 output functions 4 * Linux INET6 implementation 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 * 9 * Based on linux/net/ipv4/ip_output.c 10 * 11 * Changes: 12 * A.N.Kuznetsov : airthmetics in fragmentation. 13 * extension headers are implemented. 14 * route changes now work. 15 * ip6_forward does not confuse sniffers. 16 * etc. 17 * 18 * H. von Brand : Added missing #include <linux/string.h> 19 * Imran Patel : frag id should be in NBO 20 * Kazunori MIYAZAWA @USAGI 21 * : add ip6_append_data and related functions 22 * for datagram xmit 23 */ 24 25 #include <linux/errno.h> 26 #include <linux/kernel.h> 27 #include <linux/string.h> 28 #include <linux/socket.h> 29 #include <linux/net.h> 30 #include <linux/netdevice.h> 31 #include <linux/if_arp.h> 32 #include <linux/in6.h> 33 #include <linux/tcp.h> 34 #include <linux/route.h> 35 #include <linux/module.h> 36 #include <linux/slab.h> 37 38 #include <linux/bpf-cgroup.h> 39 #include <linux/netfilter.h> 40 #include <linux/netfilter_ipv6.h> 41 42 #include <net/sock.h> 43 #include <net/snmp.h> 44 45 #include <net/gso.h> 46 #include <net/ipv6.h> 47 #include <net/ndisc.h> 48 #include <net/protocol.h> 49 #include <net/ip6_route.h> 50 #include <net/addrconf.h> 51 #include <net/rawv6.h> 52 #include <net/icmp.h> 53 #include <net/xfrm.h> 54 #include <net/checksum.h> 55 #include <linux/mroute6.h> 56 #include <net/l3mdev.h> 57 #include <net/lwtunnel.h> 58 #include <net/ip_tunnels.h> 59 60 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) 61 { 62 struct dst_entry *dst = skb_dst(skb); 63 struct net_device *dev = dst_dev_rcu(dst); 64 struct inet6_dev *idev = ip6_dst_idev(dst); 65 unsigned int hh_len = LL_RESERVED_SPACE(dev); 66 const struct in6_addr *daddr, *nexthop; 67 struct ipv6hdr *hdr; 68 struct neighbour *neigh; 69 int ret; 70 71 /* Be paranoid, rather than too clever. */ 72 if (unlikely(hh_len > skb_headroom(skb)) && dev->header_ops) { 73 /* idev stays alive because we hold rcu_read_lock(). */ 74 skb = skb_expand_head(skb, hh_len); 75 if (!skb) { 76 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 77 return -ENOMEM; 78 } 79 } 80 81 hdr = ipv6_hdr(skb); 82 daddr = &hdr->daddr; 83 if (unlikely(ipv6_addr_is_multicast(daddr))) { 84 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && 85 ((mroute6_is_socket(net, skb) && 86 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || 87 ipv6_chk_mcast_addr(dev, daddr, &hdr->saddr))) { 88 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 89 90 /* Do not check for IFF_ALLMULTI; multicast routing 91 is not supported in any case. 92 */ 93 if (newskb) 94 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING, 95 net, sk, newskb, NULL, newskb->dev, 96 dev_loopback_xmit); 97 98 if (hdr->hop_limit == 0) { 99 IP6_INC_STATS(net, idev, 100 IPSTATS_MIB_OUTDISCARDS); 101 kfree_skb(skb); 102 return 0; 103 } 104 } 105 106 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len); 107 if (IPV6_ADDR_MC_SCOPE(daddr) <= IPV6_ADDR_SCOPE_NODELOCAL && 108 !(dev->flags & IFF_LOOPBACK)) { 109 kfree_skb(skb); 110 return 0; 111 } 112 } 113 114 if (lwtunnel_xmit_redirect(dst->lwtstate)) { 115 int res = lwtunnel_xmit(skb); 116 117 if (res != LWTUNNEL_XMIT_CONTINUE) 118 return res; 119 hdr = ipv6_hdr(skb); 120 daddr = &hdr->daddr; 121 } 122 123 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len); 124 125 nexthop = rt6_nexthop(dst_rt6_info(dst), daddr); 126 neigh = __ipv6_neigh_lookup_noref(dev, nexthop); 127 128 if (IS_ERR_OR_NULL(neigh)) { 129 if (unlikely(!neigh)) 130 neigh = __neigh_create(&nd_tbl, nexthop, dev, false); 131 if (IS_ERR(neigh)) { 132 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTNOROUTES); 133 kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL); 134 return -EINVAL; 135 } 136 } 137 sock_confirm_neigh(skb, neigh); 138 ret = neigh_output(neigh, skb, false); 139 return ret; 140 } 141 142 static int 143 ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk, 144 struct sk_buff *skb, unsigned int mtu) 145 { 146 struct sk_buff *segs, *nskb; 147 netdev_features_t features; 148 int ret = 0; 149 150 /* Please see corresponding comment in ip_finish_output_gso 151 * describing the cases where GSO segment length exceeds the 152 * egress MTU. 153 */ 154 features = netif_skb_features(skb); 155 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 156 if (IS_ERR_OR_NULL(segs)) { 157 kfree_skb(skb); 158 return -ENOMEM; 159 } 160 161 consume_skb(skb); 162 163 skb_list_walk_safe(segs, segs, nskb) { 164 int err; 165 166 skb_mark_not_on_list(segs); 167 /* Last GSO segment can be smaller than gso_size (and MTU). 168 * Adding a fragment header would produce an "atomic fragment", 169 * which is considered harmful (RFC-8021). Avoid that. 170 */ 171 err = segs->len > mtu ? 172 ip6_fragment(net, sk, segs, ip6_finish_output2) : 173 ip6_finish_output2(net, sk, segs); 174 if (err && ret == 0) 175 ret = err; 176 } 177 178 return ret; 179 } 180 181 static int ip6_finish_output_gso(struct net *net, struct sock *sk, 182 struct sk_buff *skb, unsigned int mtu) 183 { 184 if (unlikely(!skb_gso_validate_network_len(skb, mtu))) 185 return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu); 186 187 return ip6_finish_output2(net, sk, skb); 188 } 189 190 static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 191 { 192 unsigned int mtu; 193 194 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM) 195 /* Policy lookup after SNAT yielded a new policy */ 196 if (skb_dst(skb)->xfrm) { 197 IP6CB(skb)->flags |= IP6SKB_REROUTED; 198 return dst_output(net, sk, skb); 199 } 200 #endif 201 202 mtu = ip6_skb_dst_mtu(skb); 203 if (skb_is_gso(skb)) 204 return ip6_finish_output_gso(net, sk, skb, mtu); 205 206 if (unlikely(skb->len > mtu || 207 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))) 208 return ip6_fragment(net, sk, skb, ip6_finish_output2); 209 210 return ip6_finish_output2(net, sk, skb); 211 } 212 213 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 214 { 215 int ret; 216 217 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); 218 switch (ret) { 219 case NET_XMIT_SUCCESS: 220 case NET_XMIT_CN: 221 return __ip6_finish_output(net, sk, skb) ? : ret; 222 default: 223 kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS); 224 return ret; 225 } 226 } 227 228 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb) 229 { 230 struct dst_entry *dst = skb_dst(skb); 231 struct net_device *dev, *indev = skb->dev; 232 struct inet6_dev *idev; 233 int ret; 234 235 skb->protocol = htons(ETH_P_IPV6); 236 rcu_read_lock(); 237 dev = dst_dev_rcu(dst); 238 idev = ip6_dst_idev(dst); 239 skb->dev = dev; 240 241 if (unlikely(!idev || READ_ONCE(idev->cnf.disable_ipv6))) { 242 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 243 rcu_read_unlock(); 244 kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED); 245 return 0; 246 } 247 248 ret = NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, 249 net, sk, skb, indev, dev, 250 ip6_finish_output, 251 !(IP6CB(skb)->flags & IP6SKB_REROUTED)); 252 rcu_read_unlock(); 253 return ret; 254 } 255 EXPORT_SYMBOL(ip6_output); 256 257 bool ip6_autoflowlabel(struct net *net, const struct sock *sk) 258 { 259 if (!inet6_test_bit(AUTOFLOWLABEL_SET, sk)) 260 return ip6_default_np_autolabel(net); 261 return inet6_test_bit(AUTOFLOWLABEL, sk); 262 } 263 264 int ip6_dst_hoplimit(struct dst_entry *dst) 265 { 266 int hoplimit = dst_metric_raw(dst, RTAX_HOPLIMIT); 267 268 rcu_read_lock(); 269 if (hoplimit == 0) { 270 struct net_device *dev = dst_dev_rcu(dst); 271 struct inet6_dev *idev; 272 273 idev = __in6_dev_get(dev); 274 if (idev) 275 hoplimit = READ_ONCE(idev->cnf.hop_limit); 276 else 277 hoplimit = READ_ONCE(dev_net(dev)->ipv6.devconf_all->hop_limit); 278 } 279 rcu_read_unlock(); 280 281 return hoplimit; 282 } 283 EXPORT_SYMBOL(ip6_dst_hoplimit); 284 285 /* 286 * xmit an sk_buff (used by TCP and SCTP) 287 * Note : socket lock is not held for SYNACK packets, but might be modified 288 * by calls to skb_set_owner_w() and ipv6_local_error(), 289 * which are using proper atomic operations or spinlocks. 290 */ 291 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, 292 __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority) 293 { 294 const struct ipv6_pinfo *np = inet6_sk(sk); 295 struct in6_addr *first_hop = &fl6->daddr; 296 struct dst_entry *dst = skb_dst(skb); 297 struct inet6_dev *idev = ip6_dst_idev(dst); 298 struct net *net = sock_net(sk); 299 unsigned int head_room; 300 struct net_device *dev; 301 struct ipv6hdr *hdr; 302 u8 proto = fl6->flowi6_proto; 303 int seg_len = skb->len; 304 int ret, hlimit = -1; 305 u32 mtu; 306 307 rcu_read_lock(); 308 309 dev = dst_dev_rcu(dst); 310 head_room = sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dev); 311 if (opt) 312 head_room += opt->opt_nflen + opt->opt_flen; 313 314 if (unlikely(head_room > skb_headroom(skb))) { 315 /* idev stays alive while we hold rcu_read_lock(). */ 316 skb = skb_expand_head(skb, head_room); 317 if (!skb) { 318 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 319 ret = -ENOBUFS; 320 goto unlock; 321 } 322 } 323 324 if (unlikely(opt)) { 325 seg_len += opt->opt_nflen + opt->opt_flen; 326 327 if (opt->opt_flen) 328 proto = ipv6_push_frag_opts(skb, opt, proto); 329 330 if (opt->opt_nflen) 331 proto = ipv6_push_nfrag_opts(skb, opt, proto, 332 &first_hop, 333 &fl6->saddr); 334 } 335 336 if (unlikely(seg_len > IPV6_MAXPLEN)) 337 seg_len = 0; 338 339 __skb_push(skb, sizeof(struct ipv6hdr)); 340 skb_reset_network_header(skb); 341 hdr = ipv6_hdr(skb); 342 343 /* 344 * Fill in the IPv6 header 345 */ 346 if (np) 347 hlimit = READ_ONCE(np->hop_limit); 348 if (hlimit < 0) 349 hlimit = ip6_dst_hoplimit(dst); 350 351 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel, 352 ip6_autoflowlabel(net, sk), fl6)); 353 354 hdr->payload_len = htons(seg_len); 355 hdr->nexthdr = proto; 356 hdr->hop_limit = hlimit; 357 358 hdr->saddr = fl6->saddr; 359 hdr->daddr = *first_hop; 360 361 skb->protocol = htons(ETH_P_IPV6); 362 skb->priority = priority; 363 skb->mark = mark; 364 365 mtu = dst6_mtu(dst); 366 if (likely((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb))) { 367 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS); 368 369 /* if egress device is enslaved to an L3 master device pass the 370 * skb to its handler for processing 371 */ 372 skb = l3mdev_ip6_out((struct sock *)sk, skb); 373 if (unlikely(!skb)) { 374 ret = 0; 375 goto unlock; 376 } 377 378 /* hooks should never assume socket lock is held. 379 * we promote our socket to non const 380 */ 381 ret = NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, 382 net, (struct sock *)sk, skb, NULL, dev, 383 dst_output); 384 goto unlock; 385 } 386 387 ret = -EMSGSIZE; 388 skb->dev = dev; 389 /* ipv6_local_error() does not require socket lock, 390 * we promote our socket to non const 391 */ 392 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu); 393 394 IP6_INC_STATS(net, idev, IPSTATS_MIB_FRAGFAILS); 395 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 396 unlock: 397 rcu_read_unlock(); 398 return ret; 399 } 400 EXPORT_SYMBOL(ip6_xmit); 401 402 static int ip6_call_ra_chain(struct sk_buff *skb, int sel) 403 { 404 struct ip6_ra_chain *ra; 405 struct sock *last = NULL; 406 407 read_lock(&ip6_ra_lock); 408 for (ra = ip6_ra_chain; ra; ra = ra->next) { 409 struct sock *sk = ra->sk; 410 if (sk && ra->sel == sel && 411 (!sk->sk_bound_dev_if || 412 sk->sk_bound_dev_if == skb->dev->ifindex)) { 413 414 if (inet6_test_bit(RTALERT_ISOLATE, sk) && 415 !net_eq(sock_net(sk), dev_net(skb->dev))) { 416 continue; 417 } 418 if (last) { 419 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 420 if (skb2) 421 rawv6_rcv(last, skb2); 422 } 423 last = sk; 424 } 425 } 426 427 if (last) { 428 rawv6_rcv(last, skb); 429 read_unlock(&ip6_ra_lock); 430 return 1; 431 } 432 read_unlock(&ip6_ra_lock); 433 return 0; 434 } 435 436 static int ip6_forward_proxy_check(struct sk_buff *skb) 437 { 438 struct ipv6hdr *hdr = ipv6_hdr(skb); 439 u8 nexthdr = hdr->nexthdr; 440 __be16 frag_off; 441 int offset; 442 443 if (ipv6_ext_hdr(nexthdr)) { 444 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off); 445 if (offset < 0) 446 return 0; 447 } else 448 offset = sizeof(struct ipv6hdr); 449 450 if (nexthdr == IPPROTO_ICMPV6) { 451 struct icmp6hdr *icmp6; 452 453 if (!pskb_may_pull(skb, (skb_network_header(skb) + 454 offset + 1 - skb->data))) 455 return 0; 456 457 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset); 458 459 switch (icmp6->icmp6_type) { 460 case NDISC_ROUTER_SOLICITATION: 461 case NDISC_ROUTER_ADVERTISEMENT: 462 case NDISC_NEIGHBOUR_SOLICITATION: 463 case NDISC_NEIGHBOUR_ADVERTISEMENT: 464 case NDISC_REDIRECT: 465 /* For reaction involving unicast neighbor discovery 466 * message destined to the proxied address, pass it to 467 * input function. 468 */ 469 return 1; 470 default: 471 break; 472 } 473 hdr = ipv6_hdr(skb); 474 } 475 476 /* 477 * The proxying router can't forward traffic sent to a link-local 478 * address, so signal the sender and discard the packet. This 479 * behavior is clarified by the MIPv6 specification. 480 */ 481 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) { 482 dst_link_failure(skb); 483 return -1; 484 } 485 486 return 0; 487 } 488 489 static inline int ip6_forward_finish(struct net *net, struct sock *sk, 490 struct sk_buff *skb) 491 { 492 #ifdef CONFIG_NET_SWITCHDEV 493 if (skb->offload_l3_fwd_mark) { 494 consume_skb(skb); 495 return 0; 496 } 497 #endif 498 499 skb_clear_tstamp(skb); 500 return dst_output(net, sk, skb); 501 } 502 503 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu) 504 { 505 if (skb->len <= mtu) 506 return false; 507 508 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */ 509 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu) 510 return true; 511 512 if (skb->ignore_df) 513 return false; 514 515 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 516 return false; 517 518 return true; 519 } 520 521 int ip6_forward(struct sk_buff *skb) 522 { 523 struct dst_entry *dst = skb_dst(skb); 524 struct ipv6hdr *hdr = ipv6_hdr(skb); 525 struct inet6_skb_parm *opt = IP6CB(skb); 526 struct net *net = dev_net(dst_dev(dst)); 527 struct net_device *dev; 528 struct inet6_dev *idev; 529 SKB_DR(reason); 530 u32 mtu; 531 532 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 533 if (!READ_ONCE(net->ipv6.devconf_all->forwarding) && 534 (!idev || !READ_ONCE(idev->cnf.force_forwarding))) 535 goto error; 536 537 if (skb->pkt_type != PACKET_HOST) 538 goto drop; 539 540 if (unlikely(skb->sk)) 541 goto drop; 542 543 if (skb_warn_if_lro(skb)) 544 goto drop; 545 546 if (!READ_ONCE(net->ipv6.devconf_all->disable_policy) && 547 (!idev || !READ_ONCE(idev->cnf.disable_policy)) && 548 !xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) { 549 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 550 goto drop; 551 } 552 553 skb_forward_csum(skb); 554 555 /* 556 * We DO NOT make any processing on 557 * RA packets, pushing them to user level AS IS 558 * without ane WARRANTY that application will be able 559 * to interpret them. The reason is that we 560 * cannot make anything clever here. 561 * 562 * We are not end-node, so that if packet contains 563 * AH/ESP, we cannot make anything. 564 * Defragmentation also would be mistake, RA packets 565 * cannot be fragmented, because there is no warranty 566 * that different fragments will go along one path. --ANK 567 */ 568 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) { 569 if (ip6_call_ra_chain(skb, ntohs(opt->ra))) 570 return 0; 571 } 572 573 /* 574 * check and decrement ttl 575 */ 576 if (hdr->hop_limit <= 1) { 577 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0); 578 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS); 579 580 kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR); 581 return -ETIMEDOUT; 582 } 583 584 /* XXX: idev->cnf.proxy_ndp? */ 585 if (READ_ONCE(net->ipv6.devconf_all->proxy_ndp) && 586 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev)) { 587 int proxied = ip6_forward_proxy_check(skb); 588 589 hdr = ipv6_hdr(skb); 590 if (proxied > 0) { 591 /* It's tempting to decrease the hop limit 592 * here by 1, as we do at the end of the 593 * function too. 594 * 595 * But that would be incorrect, as proxying is 596 * not forwarding. The ip6_input function 597 * will handle this packet locally, and it 598 * depends on the hop limit being unchanged. 599 * 600 * One example is the NDP hop limit, that 601 * always has to stay 255, but other would be 602 * similar checks around RA packets, where the 603 * user can even change the desired limit. 604 */ 605 return ip6_input(skb); 606 } else if (proxied < 0) { 607 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 608 goto drop; 609 } 610 } 611 612 if (!xfrm6_route_forward(skb)) { 613 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 614 SKB_DR_SET(reason, XFRM_POLICY); 615 goto drop; 616 } 617 dst = skb_dst(skb); 618 dev = dst_dev(dst); 619 /* IPv6 specs say nothing about it, but it is clear that we cannot 620 send redirects to source routed frames. 621 We don't send redirects to frames decapsulated from IPsec. 622 */ 623 if (IP6CB(skb)->iif == dev->ifindex && 624 opt->srcrt == 0 && !skb_sec_path(skb)) { 625 struct in6_addr *target = NULL; 626 struct inet_peer *peer; 627 struct rt6_info *rt; 628 629 /* 630 * incoming and outgoing devices are the same 631 * send a redirect. 632 */ 633 634 rt = dst_rt6_info(dst); 635 if (rt->rt6i_flags & RTF_GATEWAY) 636 target = &rt->rt6i_gateway; 637 else 638 target = &hdr->daddr; 639 640 rcu_read_lock(); 641 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr); 642 643 /* Limit redirects both by destination (here) 644 and by source (inside ndisc_send_redirect) 645 */ 646 if (peer && inet_peer_xrlim_allow(peer, 1*HZ)) 647 ndisc_send_redirect(skb, target); 648 rcu_read_unlock(); 649 } else { 650 int addrtype = ipv6_addr_type(&hdr->saddr); 651 652 /* This check is security critical. */ 653 if (addrtype == IPV6_ADDR_ANY || 654 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK)) 655 goto error; 656 if (addrtype & IPV6_ADDR_LINKLOCAL) { 657 icmpv6_send(skb, ICMPV6_DEST_UNREACH, 658 ICMPV6_NOT_NEIGHBOUR, 0); 659 goto error; 660 } 661 } 662 663 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS); 664 665 mtu = ip6_dst_mtu_maybe_forward(dst, true); 666 if (mtu < IPV6_MIN_MTU) 667 mtu = IPV6_MIN_MTU; 668 669 if (unlikely(ip6_pkt_too_big(skb, mtu))) { 670 /* Again, force OUTPUT device used as source address */ 671 skb->dev = dev; 672 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 673 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS); 674 __IP6_INC_STATS(net, ip6_dst_idev(dst), 675 IPSTATS_MIB_FRAGFAILS); 676 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 677 return -EMSGSIZE; 678 } 679 680 if (skb_cow(skb, dev->hard_header_len)) { 681 __IP6_INC_STATS(net, ip6_dst_idev(dst), 682 IPSTATS_MIB_OUTDISCARDS); 683 goto drop; 684 } 685 686 hdr = ipv6_hdr(skb); 687 688 /* Mangling hops number delayed to point after skb COW */ 689 690 hdr->hop_limit--; 691 692 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, 693 net, NULL, skb, skb->dev, dev, 694 ip6_forward_finish); 695 696 error: 697 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 698 SKB_DR_SET(reason, IP_INADDRERRORS); 699 drop: 700 kfree_skb_reason(skb, reason); 701 return -EINVAL; 702 } 703 704 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from) 705 { 706 to->pkt_type = from->pkt_type; 707 to->priority = from->priority; 708 to->protocol = from->protocol; 709 skb_dst_drop(to); 710 skb_dst_set(to, dst_clone(skb_dst(from))); 711 to->dev = from->dev; 712 to->mark = from->mark; 713 714 skb_copy_hash(to, from); 715 716 #ifdef CONFIG_NET_SCHED 717 to->tc_index = from->tc_index; 718 #endif 719 nf_copy(to, from); 720 skb_ext_copy(to, from); 721 skb_copy_secmark(to, from); 722 } 723 724 int ip6_fraglist_init(struct sk_buff *skb, unsigned int hlen, u8 *prevhdr, 725 u8 nexthdr, __be32 frag_id, 726 struct ip6_fraglist_iter *iter) 727 { 728 unsigned int first_len; 729 struct frag_hdr *fh; 730 731 /* BUILD HEADER */ 732 *prevhdr = NEXTHDR_FRAGMENT; 733 iter->tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC); 734 if (!iter->tmp_hdr) 735 return -ENOMEM; 736 737 iter->frag = skb_shinfo(skb)->frag_list; 738 skb_frag_list_init(skb); 739 740 iter->offset = 0; 741 iter->hlen = hlen; 742 iter->frag_id = frag_id; 743 iter->nexthdr = nexthdr; 744 745 __skb_pull(skb, hlen); 746 fh = __skb_push(skb, sizeof(struct frag_hdr)); 747 __skb_push(skb, hlen); 748 skb_reset_network_header(skb); 749 memcpy(skb_network_header(skb), iter->tmp_hdr, hlen); 750 751 fh->nexthdr = nexthdr; 752 fh->reserved = 0; 753 fh->frag_off = htons(IP6_MF); 754 fh->identification = frag_id; 755 756 first_len = skb_pagelen(skb); 757 skb->data_len = first_len - skb_headlen(skb); 758 skb->len = first_len; 759 ipv6_hdr(skb)->payload_len = htons(first_len - sizeof(struct ipv6hdr)); 760 761 return 0; 762 } 763 EXPORT_SYMBOL(ip6_fraglist_init); 764 765 void ip6_fraglist_prepare(struct sk_buff *skb, 766 struct ip6_fraglist_iter *iter) 767 { 768 struct sk_buff *frag = iter->frag; 769 unsigned int hlen = iter->hlen; 770 struct frag_hdr *fh; 771 772 frag->ip_summed = CHECKSUM_NONE; 773 skb_reset_transport_header(frag); 774 fh = __skb_push(frag, sizeof(struct frag_hdr)); 775 __skb_push(frag, hlen); 776 skb_reset_network_header(frag); 777 memcpy(skb_network_header(frag), iter->tmp_hdr, hlen); 778 iter->offset += skb->len - hlen - sizeof(struct frag_hdr); 779 fh->nexthdr = iter->nexthdr; 780 fh->reserved = 0; 781 fh->frag_off = htons(iter->offset); 782 if (frag->next) 783 fh->frag_off |= htons(IP6_MF); 784 fh->identification = iter->frag_id; 785 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); 786 ip6_copy_metadata(frag, skb); 787 } 788 EXPORT_SYMBOL(ip6_fraglist_prepare); 789 790 void ip6_frag_init(struct sk_buff *skb, unsigned int hlen, unsigned int mtu, 791 unsigned short needed_tailroom, int hdr_room, u8 *prevhdr, 792 u8 nexthdr, __be32 frag_id, struct ip6_frag_state *state) 793 { 794 state->prevhdr = prevhdr; 795 state->nexthdr = nexthdr; 796 state->frag_id = frag_id; 797 798 state->hlen = hlen; 799 state->mtu = mtu; 800 801 state->left = skb->len - hlen; /* Space per frame */ 802 state->ptr = hlen; /* Where to start from */ 803 804 state->hroom = hdr_room; 805 state->troom = needed_tailroom; 806 807 state->offset = 0; 808 } 809 EXPORT_SYMBOL(ip6_frag_init); 810 811 struct sk_buff *ip6_frag_next(struct sk_buff *skb, struct ip6_frag_state *state) 812 { 813 u8 *prevhdr = state->prevhdr, *fragnexthdr_offset; 814 struct sk_buff *frag; 815 struct frag_hdr *fh; 816 unsigned int len; 817 818 len = state->left; 819 /* IF: it doesn't fit, use 'mtu' - the data space left */ 820 if (len > state->mtu) 821 len = state->mtu; 822 /* IF: we are not sending up to and including the packet end 823 then align the next start on an eight byte boundary */ 824 if (len < state->left) 825 len &= ~7; 826 827 /* Allocate buffer */ 828 frag = alloc_skb(len + state->hlen + sizeof(struct frag_hdr) + 829 state->hroom + state->troom, GFP_ATOMIC); 830 if (!frag) 831 return ERR_PTR(-ENOMEM); 832 833 /* 834 * Set up data on packet 835 */ 836 837 ip6_copy_metadata(frag, skb); 838 skb_reserve(frag, state->hroom); 839 skb_put(frag, len + state->hlen + sizeof(struct frag_hdr)); 840 skb_reset_network_header(frag); 841 fh = (struct frag_hdr *)(skb_network_header(frag) + state->hlen); 842 frag->transport_header = (frag->network_header + state->hlen + 843 sizeof(struct frag_hdr)); 844 845 /* 846 * Charge the memory for the fragment to any owner 847 * it might possess 848 */ 849 if (skb->sk) 850 skb_set_owner_w(frag, skb->sk); 851 852 /* 853 * Copy the packet header into the new buffer. 854 */ 855 skb_copy_from_linear_data(skb, skb_network_header(frag), state->hlen); 856 857 fragnexthdr_offset = skb_network_header(frag); 858 fragnexthdr_offset += prevhdr - skb_network_header(skb); 859 *fragnexthdr_offset = NEXTHDR_FRAGMENT; 860 861 /* 862 * Build fragment header. 863 */ 864 fh->nexthdr = state->nexthdr; 865 fh->reserved = 0; 866 fh->identification = state->frag_id; 867 868 /* 869 * Copy a block of the IP datagram. 870 */ 871 BUG_ON(skb_copy_bits(skb, state->ptr, skb_transport_header(frag), 872 len)); 873 state->left -= len; 874 875 fh->frag_off = htons(state->offset); 876 if (state->left > 0) 877 fh->frag_off |= htons(IP6_MF); 878 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); 879 880 state->ptr += len; 881 state->offset += len; 882 883 return frag; 884 } 885 EXPORT_SYMBOL(ip6_frag_next); 886 887 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 888 int (*output)(struct net *, struct sock *, struct sk_buff *)) 889 { 890 struct sk_buff *frag; 891 struct rt6_info *rt = dst_rt6_info(skb_dst(skb)); 892 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? 893 inet6_sk(skb->sk) : NULL; 894 u8 tstamp_type = skb->tstamp_type; 895 struct ip6_frag_state state; 896 unsigned int mtu, hlen, nexthdr_offset; 897 ktime_t tstamp = skb->tstamp; 898 int hroom, err = 0; 899 __be32 frag_id; 900 u8 *prevhdr, nexthdr = 0; 901 902 if (!ipv6_mod_enabled()) { 903 kfree_skb(skb); 904 return -EAFNOSUPPORT; 905 } 906 907 err = ip6_find_1stfragopt(skb, &prevhdr); 908 if (err < 0) 909 goto fail; 910 hlen = err; 911 nexthdr = *prevhdr; 912 nexthdr_offset = prevhdr - skb_network_header(skb); 913 914 mtu = ip6_skb_dst_mtu(skb); 915 916 /* We must not fragment if the socket is set to force MTU discovery 917 * or if the skb it not generated by a local socket. 918 */ 919 if (unlikely(!skb->ignore_df && skb->len > mtu)) 920 goto fail_toobig; 921 922 if (IP6CB(skb)->frag_max_size) { 923 if (IP6CB(skb)->frag_max_size > mtu) 924 goto fail_toobig; 925 926 /* don't send fragments larger than what we received */ 927 mtu = IP6CB(skb)->frag_max_size; 928 if (mtu < IPV6_MIN_MTU) 929 mtu = IPV6_MIN_MTU; 930 } 931 932 if (np) { 933 u32 frag_size = READ_ONCE(np->frag_size); 934 935 if (frag_size && frag_size < mtu) 936 mtu = frag_size; 937 } 938 if (mtu < hlen + sizeof(struct frag_hdr) + 8) 939 goto fail_toobig; 940 mtu -= hlen + sizeof(struct frag_hdr); 941 942 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr, 943 &ipv6_hdr(skb)->saddr); 944 945 if (skb->ip_summed == CHECKSUM_PARTIAL && 946 (err = skb_checksum_help(skb))) 947 goto fail; 948 949 prevhdr = skb_network_header(skb) + nexthdr_offset; 950 hroom = LL_RESERVED_SPACE(rt->dst.dev); 951 if (skb_has_frag_list(skb)) { 952 unsigned int first_len = skb_pagelen(skb); 953 struct ip6_fraglist_iter iter; 954 struct sk_buff *frag2; 955 956 if (first_len - hlen > mtu || 957 ((first_len - hlen) & 7) || 958 skb_cloned(skb) || 959 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) 960 goto slow_path; 961 962 skb_walk_frags(skb, frag) { 963 /* Correct geometry. */ 964 if (frag->len > mtu || 965 ((frag->len & 7) && frag->next) || 966 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr))) 967 goto slow_path_clean; 968 969 /* Partially cloned skb? */ 970 if (skb_shared(frag)) 971 goto slow_path_clean; 972 973 BUG_ON(frag->sk); 974 if (skb->sk) { 975 frag->sk = skb->sk; 976 frag->destructor = sock_wfree; 977 } 978 skb->truesize -= frag->truesize; 979 } 980 981 err = ip6_fraglist_init(skb, hlen, prevhdr, nexthdr, frag_id, 982 &iter); 983 if (err < 0) 984 goto fail; 985 986 /* We prevent @rt from being freed. */ 987 rcu_read_lock(); 988 989 for (;;) { 990 /* Prepare header of the next frame, 991 * before previous one went down. */ 992 if (iter.frag) 993 ip6_fraglist_prepare(skb, &iter); 994 995 skb_set_delivery_time(skb, tstamp, tstamp_type); 996 err = output(net, sk, skb); 997 if (!err) 998 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 999 IPSTATS_MIB_FRAGCREATES); 1000 1001 if (err || !iter.frag) 1002 break; 1003 1004 skb = ip6_fraglist_next(&iter); 1005 } 1006 1007 kfree(iter.tmp_hdr); 1008 1009 if (err == 0) { 1010 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 1011 IPSTATS_MIB_FRAGOKS); 1012 rcu_read_unlock(); 1013 return 0; 1014 } 1015 1016 kfree_skb_list(iter.frag); 1017 1018 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 1019 IPSTATS_MIB_FRAGFAILS); 1020 rcu_read_unlock(); 1021 return err; 1022 1023 slow_path_clean: 1024 skb_walk_frags(skb, frag2) { 1025 if (frag2 == frag) 1026 break; 1027 frag2->sk = NULL; 1028 frag2->destructor = NULL; 1029 skb->truesize += frag2->truesize; 1030 } 1031 } 1032 1033 slow_path: 1034 /* 1035 * Fragment the datagram. 1036 */ 1037 1038 ip6_frag_init(skb, hlen, mtu, rt->dst.dev->needed_tailroom, 1039 LL_RESERVED_SPACE(rt->dst.dev), prevhdr, nexthdr, frag_id, 1040 &state); 1041 1042 /* 1043 * Keep copying data until we run out. 1044 */ 1045 1046 while (state.left > 0) { 1047 frag = ip6_frag_next(skb, &state); 1048 if (IS_ERR(frag)) { 1049 err = PTR_ERR(frag); 1050 goto fail; 1051 } 1052 1053 /* 1054 * Put this fragment into the sending queue. 1055 */ 1056 skb_set_delivery_time(frag, tstamp, tstamp_type); 1057 err = output(net, sk, frag); 1058 if (err) 1059 goto fail; 1060 1061 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1062 IPSTATS_MIB_FRAGCREATES); 1063 } 1064 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1065 IPSTATS_MIB_FRAGOKS); 1066 consume_skb(skb); 1067 return err; 1068 1069 fail_toobig: 1070 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 1071 err = -EMSGSIZE; 1072 1073 fail: 1074 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1075 IPSTATS_MIB_FRAGFAILS); 1076 kfree_skb(skb); 1077 return err; 1078 } 1079 EXPORT_SYMBOL_GPL(ip6_fragment); 1080 1081 static inline int ip6_rt_check(const struct rt6key *rt_key, 1082 const struct in6_addr *fl_addr, 1083 const struct in6_addr *addr_cache) 1084 { 1085 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) && 1086 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache)); 1087 } 1088 1089 static struct dst_entry *ip6_sk_dst_check(struct sock *sk, 1090 struct dst_entry *dst, 1091 const struct flowi6 *fl6) 1092 { 1093 struct ipv6_pinfo *np = inet6_sk(sk); 1094 struct rt6_info *rt; 1095 1096 if (!dst) 1097 goto out; 1098 1099 if (dst->ops->family != AF_INET6) { 1100 dst_release(dst); 1101 return NULL; 1102 } 1103 1104 rt = dst_rt6_info(dst); 1105 /* Yes, checking route validity in not connected 1106 * case is not very simple. Take into account, 1107 * that we do not support routing by source, TOS, 1108 * and MSG_DONTROUTE --ANK (980726) 1109 * 1110 * 1. ip6_rt_check(): If route was host route, 1111 * check that cached destination is current. 1112 * If it is network route, we still may 1113 * check its validity using saved pointer 1114 * to the last used address: daddr_cache. 1115 * We do not want to save whole address now, 1116 * (because main consumer of this service 1117 * is tcp, which has not this problem), 1118 * so that the last trick works only on connected 1119 * sockets. 1120 * 2. oif also should be the same. 1121 */ 1122 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, 1123 np->daddr_cache ? &sk->sk_v6_daddr : NULL) || 1124 #ifdef CONFIG_IPV6_SUBTREES 1125 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, 1126 np->saddr_cache ? &np->saddr : NULL) || 1127 #endif 1128 (fl6->flowi6_oif && fl6->flowi6_oif != dst_dev(dst)->ifindex)) { 1129 dst_release(dst); 1130 dst = NULL; 1131 } 1132 1133 out: 1134 return dst; 1135 } 1136 1137 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk, 1138 struct dst_entry **dst, struct flowi6 *fl6) 1139 { 1140 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 1141 struct neighbour *n; 1142 struct rt6_info *rt; 1143 #endif 1144 int err; 1145 int flags = 0; 1146 1147 /* The correct way to handle this would be to do 1148 * ip6_route_get_saddr, and then ip6_route_output; however, 1149 * the route-specific preferred source forces the 1150 * ip6_route_output call _before_ ip6_route_get_saddr. 1151 * 1152 * In source specific routing (no src=any default route), 1153 * ip6_route_output will fail given src=any saddr, though, so 1154 * that's why we try it again later. 1155 */ 1156 if (ipv6_addr_any(&fl6->saddr)) { 1157 struct fib6_info *from; 1158 struct rt6_info *rt; 1159 1160 *dst = ip6_route_output(net, sk, fl6); 1161 rt = (*dst)->error ? NULL : dst_rt6_info(*dst); 1162 1163 rcu_read_lock(); 1164 from = rt ? rcu_dereference(rt->from) : NULL; 1165 err = ip6_route_get_saddr(net, from, &fl6->daddr, 1166 sk ? READ_ONCE(inet6_sk(sk)->srcprefs) : 0, 1167 fl6->flowi6_l3mdev, 1168 &fl6->saddr); 1169 rcu_read_unlock(); 1170 1171 if (err) 1172 goto out_err_release; 1173 1174 /* If we had an erroneous initial result, pretend it 1175 * never existed and let the SA-enabled version take 1176 * over. 1177 */ 1178 if ((*dst)->error) { 1179 dst_release(*dst); 1180 *dst = NULL; 1181 } 1182 1183 if (fl6->flowi6_oif) 1184 flags |= RT6_LOOKUP_F_IFACE; 1185 } 1186 1187 if (!*dst) 1188 *dst = ip6_route_output_flags(net, sk, fl6, flags); 1189 1190 err = (*dst)->error; 1191 if (err) 1192 goto out_err_release; 1193 1194 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 1195 /* 1196 * Here if the dst entry we've looked up 1197 * has a neighbour entry that is in the INCOMPLETE 1198 * state and the src address from the flow is 1199 * marked as OPTIMISTIC, we release the found 1200 * dst entry and replace it instead with the 1201 * dst entry of the nexthop router 1202 */ 1203 rt = dst_rt6_info(*dst); 1204 rcu_read_lock(); 1205 n = __ipv6_neigh_lookup_noref(rt->dst.dev, 1206 rt6_nexthop(rt, &fl6->daddr)); 1207 err = n && !(READ_ONCE(n->nud_state) & NUD_VALID) ? -EINVAL : 0; 1208 rcu_read_unlock(); 1209 1210 if (err) { 1211 struct inet6_ifaddr *ifp; 1212 struct flowi6 fl_gw6; 1213 int redirect; 1214 1215 ifp = ipv6_get_ifaddr(net, &fl6->saddr, 1216 (*dst)->dev, 1); 1217 1218 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC); 1219 if (ifp) 1220 in6_ifa_put(ifp); 1221 1222 if (redirect) { 1223 /* 1224 * We need to get the dst entry for the 1225 * default router instead 1226 */ 1227 dst_release(*dst); 1228 memcpy(&fl_gw6, fl6, sizeof(struct flowi6)); 1229 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr)); 1230 *dst = ip6_route_output(net, sk, &fl_gw6); 1231 err = (*dst)->error; 1232 if (err) 1233 goto out_err_release; 1234 } 1235 } 1236 #endif 1237 if (ipv6_addr_v4mapped(&fl6->saddr) && 1238 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) { 1239 err = -EAFNOSUPPORT; 1240 goto out_err_release; 1241 } 1242 1243 return 0; 1244 1245 out_err_release: 1246 dst_release(*dst); 1247 *dst = NULL; 1248 1249 if (err == -ENETUNREACH) 1250 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES); 1251 return err; 1252 } 1253 1254 /** 1255 * ip6_dst_lookup - perform route lookup on flow 1256 * @net: Network namespace to perform lookup in 1257 * @sk: socket which provides route info 1258 * @dst: pointer to dst_entry * for result 1259 * @fl6: flow to lookup 1260 * 1261 * This function performs a route lookup on the given flow. 1262 * 1263 * It returns zero on success, or a standard errno code on error. 1264 */ 1265 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, 1266 struct flowi6 *fl6) 1267 { 1268 *dst = NULL; 1269 return ip6_dst_lookup_tail(net, sk, dst, fl6); 1270 } 1271 EXPORT_SYMBOL_GPL(ip6_dst_lookup); 1272 1273 /** 1274 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec 1275 * @net: Network namespace to perform lookup in 1276 * @sk: socket which provides route info 1277 * @fl6: flow to lookup 1278 * @final_dst: final destination address for ipsec lookup 1279 * 1280 * This function performs a route lookup on the given flow. 1281 * 1282 * It returns a valid dst pointer on success, or a pointer encoded 1283 * error code. 1284 */ 1285 struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, struct flowi6 *fl6, 1286 const struct in6_addr *final_dst) 1287 { 1288 struct dst_entry *dst = NULL; 1289 int err; 1290 1291 if (!ipv6_mod_enabled()) 1292 return ERR_PTR(-EAFNOSUPPORT); 1293 err = ip6_dst_lookup_tail(net, sk, &dst, fl6); 1294 if (err) 1295 return ERR_PTR(err); 1296 if (final_dst) 1297 fl6->daddr = *final_dst; 1298 1299 return xfrm_lookup_route(net, dst, flowi6_to_flowi(fl6), sk, 0); 1300 } 1301 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow); 1302 1303 /** 1304 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow 1305 * @sk: socket which provides the dst cache and route info 1306 * @fl6: flow to lookup 1307 * @final_dst: final destination address for ipsec lookup 1308 * @connected: whether @sk is connected or not 1309 * 1310 * This function performs a route lookup on the given flow with the 1311 * possibility of using the cached route in the socket if it is valid. 1312 * It will take the socket dst lock when operating on the dst cache. 1313 * As a result, this function can only be used in process context. 1314 * 1315 * In addition, for a connected socket, cache the dst in the socket 1316 * if the current cache is not valid. 1317 * 1318 * It returns a valid dst pointer on success, or a pointer encoded 1319 * error code. 1320 */ 1321 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6, 1322 const struct in6_addr *final_dst, 1323 bool connected) 1324 { 1325 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie); 1326 1327 dst = ip6_sk_dst_check(sk, dst, fl6); 1328 if (dst) 1329 return dst; 1330 1331 dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_dst); 1332 if (connected && !IS_ERR(dst)) 1333 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6); 1334 1335 return dst; 1336 } 1337 1338 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src, 1339 gfp_t gfp) 1340 { 1341 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1342 } 1343 1344 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src, 1345 gfp_t gfp) 1346 { 1347 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1348 } 1349 1350 static void ip6_append_data_mtu(unsigned int *mtu, 1351 int *maxfraglen, 1352 unsigned int fragheaderlen, 1353 struct sk_buff *skb, 1354 struct rt6_info *rt, 1355 unsigned int orig_mtu) 1356 { 1357 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) { 1358 if (!skb) { 1359 /* first fragment, reserve header_len */ 1360 *mtu = orig_mtu - rt->dst.header_len; 1361 1362 } else { 1363 /* 1364 * this fragment is not first, the headers 1365 * space is regarded as data space. 1366 */ 1367 *mtu = orig_mtu; 1368 } 1369 *maxfraglen = ((*mtu - fragheaderlen) & ~7) 1370 + fragheaderlen - sizeof(struct frag_hdr); 1371 } 1372 } 1373 1374 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, 1375 struct ipcm6_cookie *ipc6, 1376 struct rt6_info *rt) 1377 { 1378 struct ipv6_txoptions *nopt, *opt = ipc6->opt; 1379 struct inet6_cork *v6_cork = &cork->base6; 1380 struct ipv6_pinfo *np = inet6_sk(sk); 1381 unsigned int mtu, frag_size; 1382 1383 /* callers pass dst together with a reference, set it first so 1384 * ip6_cork_release() can put it down even in case of an error. 1385 */ 1386 cork->base.dst = &rt->dst; 1387 1388 /* 1389 * setup for corking 1390 */ 1391 if (unlikely(opt)) { 1392 if (WARN_ON(v6_cork->opt)) 1393 return -EINVAL; 1394 1395 nopt = v6_cork->opt = kzalloc_obj(*opt, sk->sk_allocation); 1396 if (unlikely(!nopt)) 1397 return -ENOBUFS; 1398 1399 nopt->tot_len = sizeof(*opt); 1400 nopt->opt_flen = opt->opt_flen; 1401 nopt->opt_nflen = opt->opt_nflen; 1402 1403 nopt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation); 1404 if (opt->dst0opt && !nopt->dst0opt) 1405 return -ENOBUFS; 1406 1407 nopt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation); 1408 if (opt->dst1opt && !nopt->dst1opt) 1409 return -ENOBUFS; 1410 1411 nopt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation); 1412 if (opt->hopopt && !nopt->hopopt) 1413 return -ENOBUFS; 1414 1415 nopt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation); 1416 if (opt->srcrt && !nopt->srcrt) 1417 return -ENOBUFS; 1418 1419 /* need source address above miyazawa*/ 1420 } 1421 v6_cork->hop_limit = ipc6->hlimit; 1422 v6_cork->tclass = ipc6->tclass; 1423 v6_cork->dontfrag = ipc6->dontfrag; 1424 if (rt->dst.flags & DST_XFRM_TUNNEL) 1425 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? 1426 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(&rt->dst); 1427 else 1428 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? 1429 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(xfrm_dst_path(&rt->dst)); 1430 1431 frag_size = READ_ONCE(np->frag_size); 1432 if (frag_size && frag_size < mtu) 1433 mtu = frag_size; 1434 1435 cork->base.fragsize = mtu; 1436 cork->base.gso_size = ipc6->gso_size; 1437 cork->base.tx_flags = 0; 1438 cork->base.mark = ipc6->sockc.mark; 1439 cork->base.priority = ipc6->sockc.priority; 1440 sock_tx_timestamp(sk, &ipc6->sockc, &cork->base.tx_flags); 1441 if (ipc6->sockc.tsflags & SOCKCM_FLAG_TS_OPT_ID) { 1442 cork->base.flags |= IPCORK_TS_OPT_ID; 1443 cork->base.ts_opt_id = ipc6->sockc.ts_opt_id; 1444 } 1445 cork->base.length = 0; 1446 cork->base.transmit_time = ipc6->sockc.transmit_time; 1447 1448 return 0; 1449 } 1450 1451 static int __ip6_append_data(struct sock *sk, 1452 struct sk_buff_head *queue, 1453 struct inet_cork_full *cork_full, 1454 struct page_frag *pfrag, 1455 int getfrag(void *from, char *to, int offset, 1456 int len, int odd, struct sk_buff *skb), 1457 void *from, size_t length, int transhdrlen, 1458 unsigned int flags) 1459 { 1460 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu; 1461 struct inet6_cork *v6_cork = &cork_full->base6; 1462 struct inet_cork *cork = &cork_full->base; 1463 struct flowi6 *fl6 = &cork_full->fl.u.ip6; 1464 struct sk_buff *skb, *skb_prev = NULL; 1465 struct ubuf_info *uarg = NULL; 1466 int exthdrlen = 0; 1467 int dst_exthdrlen = 0; 1468 int hh_len; 1469 int copy; 1470 int err; 1471 int offset = 0; 1472 bool zc = false; 1473 u32 tskey = 0; 1474 struct rt6_info *rt = dst_rt6_info(cork->dst); 1475 bool paged, hold_tskey = false, extra_uref = false; 1476 struct ipv6_txoptions *opt = v6_cork->opt; 1477 int csummode = CHECKSUM_NONE; 1478 unsigned int maxnonfragsize, headersize; 1479 unsigned int wmem_alloc_delta = 0; 1480 1481 skb = skb_peek_tail(queue); 1482 if (!skb) { 1483 exthdrlen = opt ? opt->opt_flen : 0; 1484 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; 1485 } 1486 1487 paged = !!cork->gso_size; 1488 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize; 1489 orig_mtu = mtu; 1490 1491 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1492 1493 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + 1494 (opt ? opt->opt_nflen : 0); 1495 1496 headersize = sizeof(struct ipv6hdr) + 1497 (opt ? opt->opt_flen + opt->opt_nflen : 0) + 1498 rt->rt6i_nfheader_len; 1499 1500 if (mtu <= fragheaderlen || 1501 ((mtu - fragheaderlen) & ~7) + fragheaderlen <= sizeof(struct frag_hdr)) 1502 goto emsgsize; 1503 1504 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - 1505 sizeof(struct frag_hdr); 1506 1507 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit 1508 * the first fragment 1509 */ 1510 if (headersize + transhdrlen > mtu) 1511 goto emsgsize; 1512 1513 if (cork->length + length > mtu - headersize && v6_cork->dontfrag && 1514 (sk->sk_protocol == IPPROTO_UDP || 1515 sk->sk_protocol == IPPROTO_ICMPV6 || 1516 sk->sk_protocol == IPPROTO_RAW)) { 1517 ipv6_local_rxpmtu(sk, fl6, mtu - headersize + 1518 sizeof(struct ipv6hdr)); 1519 goto emsgsize; 1520 } 1521 1522 if (ip6_sk_ignore_df(sk)) 1523 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN; 1524 else 1525 maxnonfragsize = mtu; 1526 1527 if (cork->length + length > maxnonfragsize - headersize) { 1528 emsgsize: 1529 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0); 1530 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu); 1531 return -EMSGSIZE; 1532 } 1533 1534 /* CHECKSUM_PARTIAL only with no extension headers and when 1535 * we are not going to fragment 1536 */ 1537 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP && 1538 headersize == sizeof(struct ipv6hdr) && 1539 length <= mtu - headersize && 1540 (!(flags & MSG_MORE) || cork->gso_size) && 1541 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 1542 csummode = CHECKSUM_PARTIAL; 1543 1544 if ((flags & MSG_ZEROCOPY) && length) { 1545 struct msghdr *msg = from; 1546 1547 if (getfrag == ip_generic_getfrag && msg->msg_ubuf) { 1548 if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb)) 1549 return -EINVAL; 1550 1551 /* Leave uarg NULL if can't zerocopy, callers should 1552 * be able to handle it. 1553 */ 1554 if ((rt->dst.dev->features & NETIF_F_SG) && 1555 csummode == CHECKSUM_PARTIAL) { 1556 paged = true; 1557 zc = true; 1558 uarg = msg->msg_ubuf; 1559 } 1560 } else if (sock_flag(sk, SOCK_ZEROCOPY)) { 1561 uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb), 1562 false); 1563 if (!uarg) 1564 return -ENOBUFS; 1565 extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ 1566 if (rt->dst.dev->features & NETIF_F_SG && 1567 csummode == CHECKSUM_PARTIAL) { 1568 paged = true; 1569 zc = true; 1570 } else { 1571 uarg_to_msgzc(uarg)->zerocopy = 0; 1572 skb_zcopy_set(skb, uarg, &extra_uref); 1573 } 1574 } 1575 } else if ((flags & MSG_SPLICE_PAGES) && length) { 1576 if (inet_test_bit(HDRINCL, sk)) 1577 return -EPERM; 1578 if (rt->dst.dev->features & NETIF_F_SG && 1579 getfrag == ip_generic_getfrag) 1580 /* We need an empty buffer to attach stuff to */ 1581 paged = true; 1582 else 1583 flags &= ~MSG_SPLICE_PAGES; 1584 } 1585 1586 if (cork->tx_flags & SKBTX_ANY_TSTAMP && 1587 READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) { 1588 if (cork->flags & IPCORK_TS_OPT_ID) { 1589 tskey = cork->ts_opt_id; 1590 } else { 1591 tskey = atomic_inc_return(&sk->sk_tskey) - 1; 1592 hold_tskey = true; 1593 } 1594 } 1595 1596 /* 1597 * Let's try using as much space as possible. 1598 * Use MTU if total length of the message fits into the MTU. 1599 * Otherwise, we need to reserve fragment header and 1600 * fragment alignment (= 8-15 octects, in total). 1601 * 1602 * Note that we may need to "move" the data from the tail 1603 * of the buffer to the new fragment when we split 1604 * the message. 1605 * 1606 * FIXME: It may be fragmented into multiple chunks 1607 * at once if non-fragmentable extension headers 1608 * are too large. 1609 * --yoshfuji 1610 */ 1611 1612 cork->length += length; 1613 if (!skb) 1614 goto alloc_new_skb; 1615 1616 while (length > 0) { 1617 /* Check if the remaining data fits into current packet. */ 1618 copy = (cork->length <= mtu ? mtu : maxfraglen) - skb->len; 1619 if (copy < length) 1620 copy = maxfraglen - skb->len; 1621 1622 if (copy <= 0) { 1623 char *data; 1624 unsigned int datalen; 1625 unsigned int fraglen; 1626 unsigned int fraggap; 1627 unsigned int alloclen, alloc_extra; 1628 unsigned int pagedlen; 1629 alloc_new_skb: 1630 /* There's no room in the current skb */ 1631 if (skb) 1632 fraggap = skb->len - maxfraglen; 1633 else 1634 fraggap = 0; 1635 /* update mtu and maxfraglen if necessary */ 1636 if (!skb || !skb_prev) 1637 ip6_append_data_mtu(&mtu, &maxfraglen, 1638 fragheaderlen, skb, rt, 1639 orig_mtu); 1640 1641 skb_prev = skb; 1642 1643 /* 1644 * If remaining data exceeds the mtu, 1645 * we know we need more fragment(s). 1646 */ 1647 datalen = length + fraggap; 1648 1649 if (datalen > (cork->length <= mtu ? mtu : maxfraglen) - fragheaderlen) 1650 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; 1651 fraglen = datalen + fragheaderlen; 1652 pagedlen = 0; 1653 1654 alloc_extra = hh_len; 1655 alloc_extra += dst_exthdrlen; 1656 alloc_extra += rt->dst.trailer_len; 1657 1658 /* We just reserve space for fragment header. 1659 * Note: this may be overallocation if the message 1660 * (without MSG_MORE) fits into the MTU. 1661 */ 1662 alloc_extra += sizeof(struct frag_hdr); 1663 1664 if ((flags & MSG_MORE) && 1665 !(rt->dst.dev->features&NETIF_F_SG)) 1666 alloclen = mtu; 1667 else if (!paged && 1668 (fraglen + alloc_extra < SKB_MAX_ALLOC || 1669 !(rt->dst.dev->features & NETIF_F_SG))) 1670 alloclen = fraglen; 1671 else { 1672 alloclen = fragheaderlen + transhdrlen + fraggap; 1673 pagedlen = datalen - transhdrlen - fraggap; 1674 } 1675 alloclen += alloc_extra; 1676 1677 if (datalen != length + fraggap) { 1678 /* 1679 * this is not the last fragment, the trailer 1680 * space is regarded as data space. 1681 */ 1682 datalen += rt->dst.trailer_len; 1683 } 1684 1685 fraglen = datalen + fragheaderlen; 1686 1687 copy = datalen - transhdrlen - fraggap - pagedlen; 1688 if (copy < 0) { 1689 err = -EINVAL; 1690 goto error; 1691 } 1692 if (transhdrlen) { 1693 skb = sock_alloc_send_skb(sk, alloclen, 1694 (flags & MSG_DONTWAIT), &err); 1695 } else { 1696 skb = NULL; 1697 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <= 1698 2 * sk->sk_sndbuf) 1699 skb = alloc_skb(alloclen, 1700 sk->sk_allocation); 1701 if (unlikely(!skb)) 1702 err = -ENOBUFS; 1703 } 1704 if (!skb) 1705 goto error; 1706 /* 1707 * Fill in the control structures 1708 */ 1709 skb->protocol = htons(ETH_P_IPV6); 1710 skb->ip_summed = csummode; 1711 skb->csum = 0; 1712 /* reserve for fragmentation and ipsec header */ 1713 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + 1714 dst_exthdrlen); 1715 1716 /* 1717 * Find where to start putting bytes 1718 */ 1719 data = skb_put(skb, fraglen - pagedlen); 1720 skb_set_network_header(skb, exthdrlen); 1721 data += fragheaderlen; 1722 skb->transport_header = (skb->network_header + 1723 fragheaderlen); 1724 if (fraggap) { 1725 skb->csum = skb_copy_and_csum_bits( 1726 skb_prev, maxfraglen, 1727 data + transhdrlen, fraggap); 1728 skb_prev->csum = csum_sub(skb_prev->csum, 1729 skb->csum); 1730 data += fraggap; 1731 pskb_trim_unique(skb_prev, maxfraglen); 1732 } 1733 if (copy > 0 && 1734 INDIRECT_CALL_1(getfrag, ip_generic_getfrag, 1735 from, data + transhdrlen, offset, 1736 copy, fraggap, skb) < 0) { 1737 err = -EFAULT; 1738 kfree_skb(skb); 1739 goto error; 1740 } else if (flags & MSG_SPLICE_PAGES) { 1741 copy = 0; 1742 } 1743 1744 offset += copy; 1745 length -= copy + transhdrlen; 1746 transhdrlen = 0; 1747 exthdrlen = 0; 1748 dst_exthdrlen = 0; 1749 1750 /* Only the initial fragment is time stamped */ 1751 skb_shinfo(skb)->tx_flags = cork->tx_flags; 1752 cork->tx_flags = 0; 1753 skb_shinfo(skb)->tskey = tskey; 1754 tskey = 0; 1755 skb_zcopy_set(skb, uarg, &extra_uref); 1756 1757 if ((flags & MSG_CONFIRM) && !skb_prev) 1758 skb_set_dst_pending_confirm(skb, 1); 1759 1760 /* 1761 * Put the packet on the pending queue 1762 */ 1763 if (!skb->destructor) { 1764 skb->destructor = sock_wfree; 1765 skb->sk = sk; 1766 wmem_alloc_delta += skb->truesize; 1767 } 1768 __skb_queue_tail(queue, skb); 1769 continue; 1770 } 1771 1772 if (copy > length) 1773 copy = length; 1774 1775 if (!(rt->dst.dev->features&NETIF_F_SG) && 1776 skb_tailroom(skb) >= copy) { 1777 unsigned int off; 1778 1779 off = skb->len; 1780 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag, 1781 from, skb_put(skb, copy), 1782 offset, copy, off, skb) < 0) { 1783 __skb_trim(skb, off); 1784 err = -EFAULT; 1785 goto error; 1786 } 1787 } else if (flags & MSG_SPLICE_PAGES) { 1788 struct msghdr *msg = from; 1789 1790 err = -EIO; 1791 if (WARN_ON_ONCE(copy > msg->msg_iter.count)) 1792 goto error; 1793 1794 err = skb_splice_from_iter(skb, &msg->msg_iter, copy); 1795 if (err < 0) 1796 goto error; 1797 copy = err; 1798 if (!(flags & MSG_NO_SHARED_FRAGS)) 1799 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; 1800 wmem_alloc_delta += copy; 1801 } else if (!zc) { 1802 int i = skb_shinfo(skb)->nr_frags; 1803 1804 err = -ENOMEM; 1805 if (!sk_page_frag_refill(sk, pfrag)) 1806 goto error; 1807 1808 skb_zcopy_downgrade_managed(skb); 1809 if (!skb_can_coalesce(skb, i, pfrag->page, 1810 pfrag->offset)) { 1811 err = -EMSGSIZE; 1812 if (i == MAX_SKB_FRAGS) 1813 goto error; 1814 1815 __skb_fill_page_desc(skb, i, pfrag->page, 1816 pfrag->offset, 0); 1817 skb_shinfo(skb)->nr_frags = ++i; 1818 get_page(pfrag->page); 1819 } 1820 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1821 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag, 1822 from, 1823 page_address(pfrag->page) + pfrag->offset, 1824 offset, copy, skb->len, skb) < 0) 1825 goto error_efault; 1826 1827 pfrag->offset += copy; 1828 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1829 skb->len += copy; 1830 skb->data_len += copy; 1831 skb->truesize += copy; 1832 wmem_alloc_delta += copy; 1833 } else { 1834 err = skb_zerocopy_iter_dgram(skb, from, copy); 1835 if (err < 0) 1836 goto error; 1837 } 1838 offset += copy; 1839 length -= copy; 1840 } 1841 1842 if (wmem_alloc_delta) 1843 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1844 return 0; 1845 1846 error_efault: 1847 err = -EFAULT; 1848 error: 1849 net_zcopy_put_abort(uarg, extra_uref); 1850 cork->length -= length; 1851 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); 1852 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1853 if (hold_tskey) 1854 atomic_dec(&sk->sk_tskey); 1855 return err; 1856 } 1857 1858 int ip6_append_data(struct sock *sk, 1859 int getfrag(void *from, char *to, int offset, int len, 1860 int odd, struct sk_buff *skb), 1861 void *from, size_t length, int transhdrlen, 1862 struct ipcm6_cookie *ipc6, struct flowi6 *fl6, 1863 struct rt6_info *rt, unsigned int flags) 1864 { 1865 struct inet_sock *inet = inet_sk(sk); 1866 int exthdrlen; 1867 int err; 1868 1869 if (flags&MSG_PROBE) 1870 return 0; 1871 if (skb_queue_empty(&sk->sk_write_queue)) { 1872 /* 1873 * setup for corking 1874 */ 1875 dst_hold(&rt->dst); 1876 err = ip6_setup_cork(sk, &inet->cork, 1877 ipc6, rt); 1878 if (err) 1879 return err; 1880 1881 inet->cork.fl.u.ip6 = *fl6; 1882 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 1883 length += exthdrlen; 1884 transhdrlen += exthdrlen; 1885 } else { 1886 transhdrlen = 0; 1887 } 1888 1889 return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork, 1890 sk_page_frag(sk), getfrag, 1891 from, length, transhdrlen, flags); 1892 } 1893 EXPORT_SYMBOL_GPL(ip6_append_data); 1894 1895 static void ip6_cork_steal_dst(struct sk_buff *skb, struct inet_cork_full *cork) 1896 { 1897 struct dst_entry *dst = cork->base.dst; 1898 1899 cork->base.dst = NULL; 1900 skb_dst_set(skb, dst); 1901 } 1902 1903 static void ip6_cork_release(struct inet_cork_full *cork) 1904 { 1905 struct inet6_cork *v6_cork = &cork->base6; 1906 1907 if (unlikely(v6_cork->opt)) { 1908 struct ipv6_txoptions *opt = v6_cork->opt; 1909 1910 kfree(opt->dst0opt); 1911 kfree(opt->dst1opt); 1912 kfree(opt->hopopt); 1913 kfree(opt->srcrt); 1914 kfree(opt); 1915 v6_cork->opt = NULL; 1916 } 1917 1918 if (cork->base.dst) { 1919 dst_release(cork->base.dst); 1920 cork->base.dst = NULL; 1921 } 1922 } 1923 1924 struct sk_buff *__ip6_make_skb(struct sock *sk, 1925 struct sk_buff_head *queue, 1926 struct inet_cork_full *cork) 1927 { 1928 struct sk_buff *skb, *tmp_skb; 1929 struct sk_buff **tail_skb; 1930 struct in6_addr *final_dst; 1931 struct net *net = sock_net(sk); 1932 struct ipv6hdr *hdr; 1933 struct ipv6_txoptions *opt; 1934 struct rt6_info *rt = dst_rt6_info(cork->base.dst); 1935 struct flowi6 *fl6 = &cork->fl.u.ip6; 1936 unsigned char proto = fl6->flowi6_proto; 1937 1938 skb = __skb_dequeue(queue); 1939 if (!skb) 1940 goto out; 1941 tail_skb = &(skb_shinfo(skb)->frag_list); 1942 1943 /* move skb->data to ip header from ext header */ 1944 if (skb->data < skb_network_header(skb)) 1945 __skb_pull(skb, skb_network_offset(skb)); 1946 while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1947 __skb_pull(tmp_skb, skb_network_header_len(skb)); 1948 *tail_skb = tmp_skb; 1949 tail_skb = &(tmp_skb->next); 1950 skb->len += tmp_skb->len; 1951 skb->data_len += tmp_skb->len; 1952 skb->truesize += tmp_skb->truesize; 1953 tmp_skb->destructor = NULL; 1954 tmp_skb->sk = NULL; 1955 } 1956 1957 /* Allow local fragmentation. */ 1958 skb->ignore_df = ip6_sk_ignore_df(sk); 1959 __skb_pull(skb, skb_network_header_len(skb)); 1960 1961 final_dst = &fl6->daddr; 1962 opt = cork->base6.opt; 1963 if (unlikely(opt)) { 1964 if (opt->opt_flen) 1965 proto = ipv6_push_frag_opts(skb, opt, proto); 1966 if (opt->opt_nflen) 1967 proto = ipv6_push_nfrag_opts(skb, opt, proto, 1968 &final_dst, &fl6->saddr); 1969 } 1970 skb_push(skb, sizeof(struct ipv6hdr)); 1971 skb_reset_network_header(skb); 1972 hdr = ipv6_hdr(skb); 1973 1974 ip6_flow_hdr(hdr, cork->base6.tclass, 1975 ip6_make_flowlabel(net, skb, fl6->flowlabel, 1976 ip6_autoflowlabel(net, sk), fl6)); 1977 hdr->hop_limit = cork->base6.hop_limit; 1978 hdr->nexthdr = proto; 1979 hdr->saddr = fl6->saddr; 1980 hdr->daddr = *final_dst; 1981 1982 skb->priority = cork->base.priority; 1983 skb->mark = cork->base.mark; 1984 if (sk_is_tcp(sk)) 1985 skb_set_delivery_time(skb, cork->base.transmit_time, SKB_CLOCK_MONOTONIC); 1986 else 1987 skb_set_delivery_type_by_clockid(skb, cork->base.transmit_time, sk->sk_clockid); 1988 1989 ip6_cork_steal_dst(skb, cork); 1990 IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS); 1991 if (unlikely(proto == IPPROTO_ICMPV6)) { 1992 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 1993 u8 icmp6_type; 1994 1995 if (sk->sk_socket->type == SOCK_RAW && 1996 !(fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH)) 1997 icmp6_type = fl6->fl6_icmp_type; 1998 else 1999 icmp6_type = icmp6_hdr(skb)->icmp6_type; 2000 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_type); 2001 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS); 2002 } 2003 2004 ip6_cork_release(cork); 2005 out: 2006 return skb; 2007 } 2008 2009 int ip6_send_skb(struct sk_buff *skb) 2010 { 2011 struct net *net = sock_net(skb->sk); 2012 struct rt6_info *rt = dst_rt6_info(skb_dst(skb)); 2013 int err; 2014 2015 rcu_read_lock(); 2016 err = ip6_local_out(net, skb->sk, skb); 2017 if (err) { 2018 if (err > 0) 2019 err = net_xmit_errno(err); 2020 if (err) 2021 IP6_INC_STATS(net, rt->rt6i_idev, 2022 IPSTATS_MIB_OUTDISCARDS); 2023 } 2024 2025 rcu_read_unlock(); 2026 return err; 2027 } 2028 2029 int ip6_push_pending_frames(struct sock *sk) 2030 { 2031 struct sk_buff *skb; 2032 2033 skb = ip6_finish_skb(sk); 2034 if (!skb) 2035 return 0; 2036 2037 return ip6_send_skb(skb); 2038 } 2039 EXPORT_SYMBOL_GPL(ip6_push_pending_frames); 2040 2041 static void __ip6_flush_pending_frames(struct sock *sk, 2042 struct sk_buff_head *queue, 2043 struct inet_cork_full *cork) 2044 { 2045 struct sk_buff *skb; 2046 2047 while ((skb = __skb_dequeue_tail(queue)) != NULL) { 2048 if (skb_dst(skb)) 2049 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)), 2050 IPSTATS_MIB_OUTDISCARDS); 2051 kfree_skb(skb); 2052 } 2053 2054 ip6_cork_release(cork); 2055 } 2056 2057 void ip6_flush_pending_frames(struct sock *sk) 2058 { 2059 __ip6_flush_pending_frames(sk, &sk->sk_write_queue, 2060 &inet_sk(sk)->cork); 2061 } 2062 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames); 2063 2064 struct sk_buff *ip6_make_skb(struct sock *sk, 2065 int getfrag(void *from, char *to, int offset, 2066 int len, int odd, struct sk_buff *skb), 2067 void *from, size_t length, int transhdrlen, 2068 struct ipcm6_cookie *ipc6, struct rt6_info *rt, 2069 unsigned int flags, struct inet_cork_full *cork) 2070 { 2071 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 2072 struct sk_buff_head queue; 2073 int err; 2074 2075 if (flags & MSG_PROBE) { 2076 dst_release(&rt->dst); 2077 return NULL; 2078 } 2079 2080 __skb_queue_head_init(&queue); 2081 2082 cork->base.flags = 0; 2083 cork->base.addr = 0; 2084 cork->base.opt = NULL; 2085 cork->base6.opt = NULL; 2086 err = ip6_setup_cork(sk, cork, ipc6, rt); 2087 if (err) { 2088 ip6_cork_release(cork); 2089 return ERR_PTR(err); 2090 } 2091 2092 err = __ip6_append_data(sk, &queue, cork, 2093 ¤t->task_frag, getfrag, from, 2094 length + exthdrlen, transhdrlen + exthdrlen, 2095 flags); 2096 if (err) { 2097 __ip6_flush_pending_frames(sk, &queue, cork); 2098 return ERR_PTR(err); 2099 } 2100 2101 return __ip6_make_skb(sk, &queue, cork); 2102 } 2103