1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * IPv6 output functions
4 * Linux INET6 implementation
5 *
6 * Authors:
7 * Pedro Roque <roque@di.fc.ul.pt>
8 *
9 * Based on linux/net/ipv4/ip_output.c
10 *
11 * Changes:
12 * A.N.Kuznetsov : airthmetics in fragmentation.
13 * extension headers are implemented.
14 * route changes now work.
15 * ip6_forward does not confuse sniffers.
16 * etc.
17 *
18 * H. von Brand : Added missing #include <linux/string.h>
19 * Imran Patel : frag id should be in NBO
20 * Kazunori MIYAZAWA @USAGI
21 * : add ip6_append_data and related functions
22 * for datagram xmit
23 */
24
25 #include <linux/errno.h>
26 #include <linux/kernel.h>
27 #include <linux/string.h>
28 #include <linux/socket.h>
29 #include <linux/net.h>
30 #include <linux/netdevice.h>
31 #include <linux/if_arp.h>
32 #include <linux/in6.h>
33 #include <linux/tcp.h>
34 #include <linux/route.h>
35 #include <linux/module.h>
36 #include <linux/slab.h>
37
38 #include <linux/bpf-cgroup.h>
39 #include <linux/netfilter.h>
40 #include <linux/netfilter_ipv6.h>
41
42 #include <net/sock.h>
43 #include <net/snmp.h>
44
45 #include <net/gso.h>
46 #include <net/ipv6.h>
47 #include <net/ndisc.h>
48 #include <net/protocol.h>
49 #include <net/ip6_route.h>
50 #include <net/addrconf.h>
51 #include <net/rawv6.h>
52 #include <net/icmp.h>
53 #include <net/xfrm.h>
54 #include <net/checksum.h>
55 #include <linux/mroute6.h>
56 #include <net/l3mdev.h>
57 #include <net/lwtunnel.h>
58 #include <net/ip_tunnels.h>
59
ip6_finish_output2(struct net * net,struct sock * sk,struct sk_buff * skb)60 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
61 {
62 struct dst_entry *dst = skb_dst(skb);
63 struct net_device *dev = dst_dev_rcu(dst);
64 struct inet6_dev *idev = ip6_dst_idev(dst);
65 unsigned int hh_len = LL_RESERVED_SPACE(dev);
66 const struct in6_addr *daddr, *nexthop;
67 struct ipv6hdr *hdr;
68 struct neighbour *neigh;
69 int ret;
70
71 /* Be paranoid, rather than too clever. */
72 if (unlikely(hh_len > skb_headroom(skb)) && dev->header_ops) {
73 /* idev stays alive because we hold rcu_read_lock(). */
74 skb = skb_expand_head(skb, hh_len);
75 if (!skb) {
76 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
77 return -ENOMEM;
78 }
79 }
80
81 hdr = ipv6_hdr(skb);
82 daddr = &hdr->daddr;
83 if (unlikely(ipv6_addr_is_multicast(daddr))) {
84 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) &&
85 ((mroute6_is_socket(net, skb) &&
86 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
87 ipv6_chk_mcast_addr(dev, daddr, &hdr->saddr))) {
88 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
89
90 /* Do not check for IFF_ALLMULTI; multicast routing
91 is not supported in any case.
92 */
93 if (newskb)
94 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
95 net, sk, newskb, NULL, newskb->dev,
96 dev_loopback_xmit);
97
98 if (hdr->hop_limit == 0) {
99 IP6_INC_STATS(net, idev,
100 IPSTATS_MIB_OUTDISCARDS);
101 kfree_skb(skb);
102 return 0;
103 }
104 }
105
106 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len);
107 if (IPV6_ADDR_MC_SCOPE(daddr) <= IPV6_ADDR_SCOPE_NODELOCAL &&
108 !(dev->flags & IFF_LOOPBACK)) {
109 kfree_skb(skb);
110 return 0;
111 }
112 }
113
114 if (lwtunnel_xmit_redirect(dst->lwtstate)) {
115 int res = lwtunnel_xmit(skb);
116
117 if (res != LWTUNNEL_XMIT_CONTINUE)
118 return res;
119 hdr = ipv6_hdr(skb);
120 daddr = &hdr->daddr;
121 }
122
123 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len);
124
125 nexthop = rt6_nexthop(dst_rt6_info(dst), daddr);
126 neigh = __ipv6_neigh_lookup_noref(dev, nexthop);
127
128 if (IS_ERR_OR_NULL(neigh)) {
129 if (unlikely(!neigh))
130 neigh = __neigh_create(&nd_tbl, nexthop, dev, false);
131 if (IS_ERR(neigh)) {
132 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTNOROUTES);
133 kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL);
134 return -EINVAL;
135 }
136 }
137 sock_confirm_neigh(skb, neigh);
138 ret = neigh_output(neigh, skb, false);
139 return ret;
140 }
141
142 static int
ip6_finish_output_gso_slowpath_drop(struct net * net,struct sock * sk,struct sk_buff * skb,unsigned int mtu)143 ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk,
144 struct sk_buff *skb, unsigned int mtu)
145 {
146 struct sk_buff *segs, *nskb;
147 netdev_features_t features;
148 int ret = 0;
149
150 /* Please see corresponding comment in ip_finish_output_gso
151 * describing the cases where GSO segment length exceeds the
152 * egress MTU.
153 */
154 features = netif_skb_features(skb);
155 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
156 if (IS_ERR_OR_NULL(segs)) {
157 kfree_skb(skb);
158 return -ENOMEM;
159 }
160
161 consume_skb(skb);
162
163 skb_list_walk_safe(segs, segs, nskb) {
164 int err;
165
166 skb_mark_not_on_list(segs);
167 /* Last GSO segment can be smaller than gso_size (and MTU).
168 * Adding a fragment header would produce an "atomic fragment",
169 * which is considered harmful (RFC-8021). Avoid that.
170 */
171 err = segs->len > mtu ?
172 ip6_fragment(net, sk, segs, ip6_finish_output2) :
173 ip6_finish_output2(net, sk, segs);
174 if (err && ret == 0)
175 ret = err;
176 }
177
178 return ret;
179 }
180
ip6_finish_output_gso(struct net * net,struct sock * sk,struct sk_buff * skb,unsigned int mtu)181 static int ip6_finish_output_gso(struct net *net, struct sock *sk,
182 struct sk_buff *skb, unsigned int mtu)
183 {
184 if (unlikely(!skb_gso_validate_network_len(skb, mtu)))
185 return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu);
186
187 return ip6_finish_output2(net, sk, skb);
188 }
189
__ip6_finish_output(struct net * net,struct sock * sk,struct sk_buff * skb)190 static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
191 {
192 unsigned int mtu;
193
194 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
195 /* Policy lookup after SNAT yielded a new policy */
196 if (skb_dst(skb)->xfrm) {
197 IP6CB(skb)->flags |= IP6SKB_REROUTED;
198 return dst_output(net, sk, skb);
199 }
200 #endif
201
202 mtu = ip6_skb_dst_mtu(skb);
203 if (skb_is_gso(skb))
204 return ip6_finish_output_gso(net, sk, skb, mtu);
205
206 if (unlikely(skb->len > mtu ||
207 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)))
208 return ip6_fragment(net, sk, skb, ip6_finish_output2);
209
210 return ip6_finish_output2(net, sk, skb);
211 }
212
ip6_finish_output(struct net * net,struct sock * sk,struct sk_buff * skb)213 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
214 {
215 int ret;
216
217 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
218 switch (ret) {
219 case NET_XMIT_SUCCESS:
220 case NET_XMIT_CN:
221 return __ip6_finish_output(net, sk, skb) ? : ret;
222 default:
223 kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS);
224 return ret;
225 }
226 }
227
ip6_output(struct net * net,struct sock * sk,struct sk_buff * skb)228 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
229 {
230 struct dst_entry *dst = skb_dst(skb);
231 struct net_device *dev, *indev = skb->dev;
232 struct inet6_dev *idev;
233 int ret;
234
235 skb->protocol = htons(ETH_P_IPV6);
236 rcu_read_lock();
237 dev = dst_dev_rcu(dst);
238 idev = ip6_dst_idev(dst);
239 skb->dev = dev;
240
241 if (unlikely(!idev || READ_ONCE(idev->cnf.disable_ipv6))) {
242 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
243 rcu_read_unlock();
244 kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED);
245 return 0;
246 }
247
248 ret = NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING,
249 net, sk, skb, indev, dev,
250 ip6_finish_output,
251 !(IP6CB(skb)->flags & IP6SKB_REROUTED));
252 rcu_read_unlock();
253 return ret;
254 }
255 EXPORT_SYMBOL(ip6_output);
256
ip6_autoflowlabel(struct net * net,const struct sock * sk)257 bool ip6_autoflowlabel(struct net *net, const struct sock *sk)
258 {
259 if (!inet6_test_bit(AUTOFLOWLABEL_SET, sk))
260 return ip6_default_np_autolabel(net);
261 return inet6_test_bit(AUTOFLOWLABEL, sk);
262 }
263
ip6_dst_hoplimit(struct dst_entry * dst)264 int ip6_dst_hoplimit(struct dst_entry *dst)
265 {
266 int hoplimit = dst_metric_raw(dst, RTAX_HOPLIMIT);
267
268 rcu_read_lock();
269 if (hoplimit == 0) {
270 struct net_device *dev = dst_dev_rcu(dst);
271 struct inet6_dev *idev;
272
273 idev = __in6_dev_get(dev);
274 if (idev)
275 hoplimit = READ_ONCE(idev->cnf.hop_limit);
276 else
277 hoplimit = READ_ONCE(dev_net(dev)->ipv6.devconf_all->hop_limit);
278 }
279 rcu_read_unlock();
280
281 return hoplimit;
282 }
283 EXPORT_SYMBOL(ip6_dst_hoplimit);
284
285 /*
286 * xmit an sk_buff (used by TCP and SCTP)
287 * Note : socket lock is not held for SYNACK packets, but might be modified
288 * by calls to skb_set_owner_w() and ipv6_local_error(),
289 * which are using proper atomic operations or spinlocks.
290 */
ip6_xmit(const struct sock * sk,struct sk_buff * skb,struct flowi6 * fl6,__u32 mark,struct ipv6_txoptions * opt,int tclass,u32 priority)291 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
292 __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority)
293 {
294 const struct ipv6_pinfo *np = inet6_sk(sk);
295 struct in6_addr *first_hop = &fl6->daddr;
296 struct dst_entry *dst = skb_dst(skb);
297 struct inet6_dev *idev = ip6_dst_idev(dst);
298 struct net *net = sock_net(sk);
299 unsigned int head_room;
300 struct net_device *dev;
301 struct ipv6hdr *hdr;
302 u8 proto = fl6->flowi6_proto;
303 int seg_len = skb->len;
304 int ret, hlimit = -1;
305 u32 mtu;
306
307 rcu_read_lock();
308
309 dev = dst_dev_rcu(dst);
310 head_room = sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dev);
311 if (opt)
312 head_room += opt->opt_nflen + opt->opt_flen;
313
314 if (unlikely(head_room > skb_headroom(skb))) {
315 /* idev stays alive while we hold rcu_read_lock(). */
316 skb = skb_expand_head(skb, head_room);
317 if (!skb) {
318 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
319 ret = -ENOBUFS;
320 goto unlock;
321 }
322 }
323
324 if (unlikely(opt)) {
325 seg_len += opt->opt_nflen + opt->opt_flen;
326
327 if (opt->opt_flen)
328 proto = ipv6_push_frag_opts(skb, opt, proto);
329
330 if (opt->opt_nflen)
331 proto = ipv6_push_nfrag_opts(skb, opt, proto,
332 &first_hop,
333 &fl6->saddr);
334 }
335
336 if (unlikely(seg_len > IPV6_MAXPLEN))
337 seg_len = 0;
338
339 __skb_push(skb, sizeof(struct ipv6hdr));
340 skb_reset_network_header(skb);
341 hdr = ipv6_hdr(skb);
342
343 /*
344 * Fill in the IPv6 header
345 */
346 if (np)
347 hlimit = READ_ONCE(np->hop_limit);
348 if (hlimit < 0)
349 hlimit = ip6_dst_hoplimit(dst);
350
351 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel,
352 ip6_autoflowlabel(net, sk), fl6));
353
354 hdr->payload_len = htons(seg_len);
355 hdr->nexthdr = proto;
356 hdr->hop_limit = hlimit;
357
358 hdr->saddr = fl6->saddr;
359 hdr->daddr = *first_hop;
360
361 skb->protocol = htons(ETH_P_IPV6);
362 skb->priority = priority;
363 skb->mark = mark;
364
365 mtu = dst6_mtu(dst);
366 if (likely((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb))) {
367 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS);
368
369 /* if egress device is enslaved to an L3 master device pass the
370 * skb to its handler for processing
371 */
372 skb = l3mdev_ip6_out((struct sock *)sk, skb);
373 if (unlikely(!skb)) {
374 ret = 0;
375 goto unlock;
376 }
377
378 /* hooks should never assume socket lock is held.
379 * we promote our socket to non const
380 */
381 ret = NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
382 net, (struct sock *)sk, skb, NULL, dev,
383 dst_output);
384 goto unlock;
385 }
386
387 ret = -EMSGSIZE;
388 skb->dev = dev;
389 /* ipv6_local_error() does not require socket lock,
390 * we promote our socket to non const
391 */
392 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu);
393
394 IP6_INC_STATS(net, idev, IPSTATS_MIB_FRAGFAILS);
395 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG);
396 unlock:
397 rcu_read_unlock();
398 return ret;
399 }
400 EXPORT_SYMBOL(ip6_xmit);
401
ip6_call_ra_chain(struct sk_buff * skb,int sel)402 static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
403 {
404 struct ip6_ra_chain *ra;
405 struct sock *last = NULL;
406
407 read_lock(&ip6_ra_lock);
408 for (ra = ip6_ra_chain; ra; ra = ra->next) {
409 struct sock *sk = ra->sk;
410 if (sk && ra->sel == sel &&
411 (!sk->sk_bound_dev_if ||
412 sk->sk_bound_dev_if == skb->dev->ifindex)) {
413
414 if (inet6_test_bit(RTALERT_ISOLATE, sk) &&
415 !net_eq(sock_net(sk), dev_net(skb->dev))) {
416 continue;
417 }
418 if (last) {
419 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
420 if (skb2)
421 rawv6_rcv(last, skb2);
422 }
423 last = sk;
424 }
425 }
426
427 if (last) {
428 rawv6_rcv(last, skb);
429 read_unlock(&ip6_ra_lock);
430 return 1;
431 }
432 read_unlock(&ip6_ra_lock);
433 return 0;
434 }
435
ip6_forward_proxy_check(struct sk_buff * skb)436 static int ip6_forward_proxy_check(struct sk_buff *skb)
437 {
438 struct ipv6hdr *hdr = ipv6_hdr(skb);
439 u8 nexthdr = hdr->nexthdr;
440 __be16 frag_off;
441 int offset;
442
443 if (ipv6_ext_hdr(nexthdr)) {
444 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
445 if (offset < 0)
446 return 0;
447 } else
448 offset = sizeof(struct ipv6hdr);
449
450 if (nexthdr == IPPROTO_ICMPV6) {
451 struct icmp6hdr *icmp6;
452
453 if (!pskb_may_pull(skb, (skb_network_header(skb) +
454 offset + 1 - skb->data)))
455 return 0;
456
457 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
458
459 switch (icmp6->icmp6_type) {
460 case NDISC_ROUTER_SOLICITATION:
461 case NDISC_ROUTER_ADVERTISEMENT:
462 case NDISC_NEIGHBOUR_SOLICITATION:
463 case NDISC_NEIGHBOUR_ADVERTISEMENT:
464 case NDISC_REDIRECT:
465 /* For reaction involving unicast neighbor discovery
466 * message destined to the proxied address, pass it to
467 * input function.
468 */
469 return 1;
470 default:
471 break;
472 }
473 hdr = ipv6_hdr(skb);
474 }
475
476 /*
477 * The proxying router can't forward traffic sent to a link-local
478 * address, so signal the sender and discard the packet. This
479 * behavior is clarified by the MIPv6 specification.
480 */
481 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
482 dst_link_failure(skb);
483 return -1;
484 }
485
486 return 0;
487 }
488
ip6_forward_finish(struct net * net,struct sock * sk,struct sk_buff * skb)489 static inline int ip6_forward_finish(struct net *net, struct sock *sk,
490 struct sk_buff *skb)
491 {
492 #ifdef CONFIG_NET_SWITCHDEV
493 if (skb->offload_l3_fwd_mark) {
494 consume_skb(skb);
495 return 0;
496 }
497 #endif
498
499 skb_clear_tstamp(skb);
500 return dst_output(net, sk, skb);
501 }
502
ip6_pkt_too_big(const struct sk_buff * skb,unsigned int mtu)503 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
504 {
505 if (skb->len <= mtu)
506 return false;
507
508 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */
509 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)
510 return true;
511
512 if (skb->ignore_df)
513 return false;
514
515 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
516 return false;
517
518 return true;
519 }
520
ip6_forward(struct sk_buff * skb)521 int ip6_forward(struct sk_buff *skb)
522 {
523 struct dst_entry *dst = skb_dst(skb);
524 struct ipv6hdr *hdr = ipv6_hdr(skb);
525 struct inet6_skb_parm *opt = IP6CB(skb);
526 struct net *net = dev_net(dst_dev(dst));
527 struct net_device *dev;
528 struct inet6_dev *idev;
529 SKB_DR(reason);
530 u32 mtu;
531
532 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif));
533 if (!READ_ONCE(net->ipv6.devconf_all->forwarding) &&
534 (!idev || !READ_ONCE(idev->cnf.force_forwarding)))
535 goto error;
536
537 if (skb->pkt_type != PACKET_HOST)
538 goto drop;
539
540 if (unlikely(skb->sk))
541 goto drop;
542
543 if (skb_warn_if_lro(skb))
544 goto drop;
545
546 if (!READ_ONCE(net->ipv6.devconf_all->disable_policy) &&
547 (!idev || !READ_ONCE(idev->cnf.disable_policy)) &&
548 !xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
549 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
550 goto drop;
551 }
552
553 skb_forward_csum(skb);
554
555 /*
556 * We DO NOT make any processing on
557 * RA packets, pushing them to user level AS IS
558 * without ane WARRANTY that application will be able
559 * to interpret them. The reason is that we
560 * cannot make anything clever here.
561 *
562 * We are not end-node, so that if packet contains
563 * AH/ESP, we cannot make anything.
564 * Defragmentation also would be mistake, RA packets
565 * cannot be fragmented, because there is no warranty
566 * that different fragments will go along one path. --ANK
567 */
568 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
569 if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
570 return 0;
571 }
572
573 /*
574 * check and decrement ttl
575 */
576 if (hdr->hop_limit <= 1) {
577 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
578 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
579
580 kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR);
581 return -ETIMEDOUT;
582 }
583
584 /* XXX: idev->cnf.proxy_ndp? */
585 if (READ_ONCE(net->ipv6.devconf_all->proxy_ndp) &&
586 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev)) {
587 int proxied = ip6_forward_proxy_check(skb);
588
589 hdr = ipv6_hdr(skb);
590 if (proxied > 0) {
591 /* It's tempting to decrease the hop limit
592 * here by 1, as we do at the end of the
593 * function too.
594 *
595 * But that would be incorrect, as proxying is
596 * not forwarding. The ip6_input function
597 * will handle this packet locally, and it
598 * depends on the hop limit being unchanged.
599 *
600 * One example is the NDP hop limit, that
601 * always has to stay 255, but other would be
602 * similar checks around RA packets, where the
603 * user can even change the desired limit.
604 */
605 return ip6_input(skb);
606 } else if (proxied < 0) {
607 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
608 goto drop;
609 }
610 }
611
612 if (!xfrm6_route_forward(skb)) {
613 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
614 SKB_DR_SET(reason, XFRM_POLICY);
615 goto drop;
616 }
617 dst = skb_dst(skb);
618 dev = dst_dev(dst);
619 /* IPv6 specs say nothing about it, but it is clear that we cannot
620 send redirects to source routed frames.
621 We don't send redirects to frames decapsulated from IPsec.
622 */
623 if (IP6CB(skb)->iif == dev->ifindex &&
624 opt->srcrt == 0 && !skb_sec_path(skb)) {
625 struct in6_addr *target = NULL;
626 struct inet_peer *peer;
627 struct rt6_info *rt;
628
629 /*
630 * incoming and outgoing devices are the same
631 * send a redirect.
632 */
633
634 rt = dst_rt6_info(dst);
635 if (rt->rt6i_flags & RTF_GATEWAY)
636 target = &rt->rt6i_gateway;
637 else
638 target = &hdr->daddr;
639
640 rcu_read_lock();
641 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr);
642
643 /* Limit redirects both by destination (here)
644 and by source (inside ndisc_send_redirect)
645 */
646 if (peer && inet_peer_xrlim_allow(peer, 1*HZ))
647 ndisc_send_redirect(skb, target);
648 rcu_read_unlock();
649 } else {
650 int addrtype = ipv6_addr_type(&hdr->saddr);
651
652 /* This check is security critical. */
653 if (addrtype == IPV6_ADDR_ANY ||
654 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
655 goto error;
656 if (addrtype & IPV6_ADDR_LINKLOCAL) {
657 icmpv6_send(skb, ICMPV6_DEST_UNREACH,
658 ICMPV6_NOT_NEIGHBOUR, 0);
659 goto error;
660 }
661 }
662
663 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
664
665 mtu = ip6_dst_mtu_maybe_forward(dst, true);
666 if (mtu < IPV6_MIN_MTU)
667 mtu = IPV6_MIN_MTU;
668
669 if (unlikely(ip6_pkt_too_big(skb, mtu))) {
670 /* Again, force OUTPUT device used as source address */
671 skb->dev = dev;
672 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
673 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS);
674 __IP6_INC_STATS(net, ip6_dst_idev(dst),
675 IPSTATS_MIB_FRAGFAILS);
676 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG);
677 return -EMSGSIZE;
678 }
679
680 if (skb_cow(skb, dev->hard_header_len)) {
681 __IP6_INC_STATS(net, ip6_dst_idev(dst),
682 IPSTATS_MIB_OUTDISCARDS);
683 goto drop;
684 }
685
686 hdr = ipv6_hdr(skb);
687
688 /* Mangling hops number delayed to point after skb COW */
689
690 hdr->hop_limit--;
691
692 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD,
693 net, NULL, skb, skb->dev, dev,
694 ip6_forward_finish);
695
696 error:
697 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
698 SKB_DR_SET(reason, IP_INADDRERRORS);
699 drop:
700 kfree_skb_reason(skb, reason);
701 return -EINVAL;
702 }
703
ip6_copy_metadata(struct sk_buff * to,struct sk_buff * from)704 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
705 {
706 to->pkt_type = from->pkt_type;
707 to->priority = from->priority;
708 to->protocol = from->protocol;
709 skb_dst_drop(to);
710 skb_dst_set(to, dst_clone(skb_dst(from)));
711 to->dev = from->dev;
712 to->mark = from->mark;
713
714 skb_copy_hash(to, from);
715
716 #ifdef CONFIG_NET_SCHED
717 to->tc_index = from->tc_index;
718 #endif
719 nf_copy(to, from);
720 skb_ext_copy(to, from);
721 skb_copy_secmark(to, from);
722 }
723
ip6_fraglist_init(struct sk_buff * skb,unsigned int hlen,u8 * prevhdr,u8 nexthdr,__be32 frag_id,struct ip6_fraglist_iter * iter)724 int ip6_fraglist_init(struct sk_buff *skb, unsigned int hlen, u8 *prevhdr,
725 u8 nexthdr, __be32 frag_id,
726 struct ip6_fraglist_iter *iter)
727 {
728 unsigned int first_len;
729 struct frag_hdr *fh;
730
731 /* BUILD HEADER */
732 *prevhdr = NEXTHDR_FRAGMENT;
733 iter->tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
734 if (!iter->tmp_hdr)
735 return -ENOMEM;
736
737 iter->frag = skb_shinfo(skb)->frag_list;
738 skb_frag_list_init(skb);
739
740 iter->offset = 0;
741 iter->hlen = hlen;
742 iter->frag_id = frag_id;
743 iter->nexthdr = nexthdr;
744
745 __skb_pull(skb, hlen);
746 fh = __skb_push(skb, sizeof(struct frag_hdr));
747 __skb_push(skb, hlen);
748 skb_reset_network_header(skb);
749 memcpy(skb_network_header(skb), iter->tmp_hdr, hlen);
750
751 fh->nexthdr = nexthdr;
752 fh->reserved = 0;
753 fh->frag_off = htons(IP6_MF);
754 fh->identification = frag_id;
755
756 first_len = skb_pagelen(skb);
757 skb->data_len = first_len - skb_headlen(skb);
758 skb->len = first_len;
759 ipv6_hdr(skb)->payload_len = htons(first_len - sizeof(struct ipv6hdr));
760
761 return 0;
762 }
763 EXPORT_SYMBOL(ip6_fraglist_init);
764
ip6_fraglist_prepare(struct sk_buff * skb,struct ip6_fraglist_iter * iter)765 void ip6_fraglist_prepare(struct sk_buff *skb,
766 struct ip6_fraglist_iter *iter)
767 {
768 struct sk_buff *frag = iter->frag;
769 unsigned int hlen = iter->hlen;
770 struct frag_hdr *fh;
771
772 frag->ip_summed = CHECKSUM_NONE;
773 skb_reset_transport_header(frag);
774 fh = __skb_push(frag, sizeof(struct frag_hdr));
775 __skb_push(frag, hlen);
776 skb_reset_network_header(frag);
777 memcpy(skb_network_header(frag), iter->tmp_hdr, hlen);
778 iter->offset += skb->len - hlen - sizeof(struct frag_hdr);
779 fh->nexthdr = iter->nexthdr;
780 fh->reserved = 0;
781 fh->frag_off = htons(iter->offset);
782 if (frag->next)
783 fh->frag_off |= htons(IP6_MF);
784 fh->identification = iter->frag_id;
785 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr));
786 ip6_copy_metadata(frag, skb);
787 }
788 EXPORT_SYMBOL(ip6_fraglist_prepare);
789
ip6_frag_init(struct sk_buff * skb,unsigned int hlen,unsigned int mtu,unsigned short needed_tailroom,int hdr_room,u8 * prevhdr,u8 nexthdr,__be32 frag_id,struct ip6_frag_state * state)790 void ip6_frag_init(struct sk_buff *skb, unsigned int hlen, unsigned int mtu,
791 unsigned short needed_tailroom, int hdr_room, u8 *prevhdr,
792 u8 nexthdr, __be32 frag_id, struct ip6_frag_state *state)
793 {
794 state->prevhdr = prevhdr;
795 state->nexthdr = nexthdr;
796 state->frag_id = frag_id;
797
798 state->hlen = hlen;
799 state->mtu = mtu;
800
801 state->left = skb->len - hlen; /* Space per frame */
802 state->ptr = hlen; /* Where to start from */
803
804 state->hroom = hdr_room;
805 state->troom = needed_tailroom;
806
807 state->offset = 0;
808 }
809 EXPORT_SYMBOL(ip6_frag_init);
810
ip6_frag_next(struct sk_buff * skb,struct ip6_frag_state * state)811 struct sk_buff *ip6_frag_next(struct sk_buff *skb, struct ip6_frag_state *state)
812 {
813 u8 *prevhdr = state->prevhdr, *fragnexthdr_offset;
814 struct sk_buff *frag;
815 struct frag_hdr *fh;
816 unsigned int len;
817
818 len = state->left;
819 /* IF: it doesn't fit, use 'mtu' - the data space left */
820 if (len > state->mtu)
821 len = state->mtu;
822 /* IF: we are not sending up to and including the packet end
823 then align the next start on an eight byte boundary */
824 if (len < state->left)
825 len &= ~7;
826
827 /* Allocate buffer */
828 frag = alloc_skb(len + state->hlen + sizeof(struct frag_hdr) +
829 state->hroom + state->troom, GFP_ATOMIC);
830 if (!frag)
831 return ERR_PTR(-ENOMEM);
832
833 /*
834 * Set up data on packet
835 */
836
837 ip6_copy_metadata(frag, skb);
838 skb_reserve(frag, state->hroom);
839 skb_put(frag, len + state->hlen + sizeof(struct frag_hdr));
840 skb_reset_network_header(frag);
841 fh = (struct frag_hdr *)(skb_network_header(frag) + state->hlen);
842 frag->transport_header = (frag->network_header + state->hlen +
843 sizeof(struct frag_hdr));
844
845 /*
846 * Charge the memory for the fragment to any owner
847 * it might possess
848 */
849 if (skb->sk)
850 skb_set_owner_w(frag, skb->sk);
851
852 /*
853 * Copy the packet header into the new buffer.
854 */
855 skb_copy_from_linear_data(skb, skb_network_header(frag), state->hlen);
856
857 fragnexthdr_offset = skb_network_header(frag);
858 fragnexthdr_offset += prevhdr - skb_network_header(skb);
859 *fragnexthdr_offset = NEXTHDR_FRAGMENT;
860
861 /*
862 * Build fragment header.
863 */
864 fh->nexthdr = state->nexthdr;
865 fh->reserved = 0;
866 fh->identification = state->frag_id;
867
868 /*
869 * Copy a block of the IP datagram.
870 */
871 BUG_ON(skb_copy_bits(skb, state->ptr, skb_transport_header(frag),
872 len));
873 state->left -= len;
874
875 fh->frag_off = htons(state->offset);
876 if (state->left > 0)
877 fh->frag_off |= htons(IP6_MF);
878 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr));
879
880 state->ptr += len;
881 state->offset += len;
882
883 return frag;
884 }
885 EXPORT_SYMBOL(ip6_frag_next);
886
ip6_fragment(struct net * net,struct sock * sk,struct sk_buff * skb,int (* output)(struct net *,struct sock *,struct sk_buff *))887 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
888 int (*output)(struct net *, struct sock *, struct sk_buff *))
889 {
890 struct sk_buff *frag;
891 struct rt6_info *rt = dst_rt6_info(skb_dst(skb));
892 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
893 inet6_sk(skb->sk) : NULL;
894 u8 tstamp_type = skb->tstamp_type;
895 struct ip6_frag_state state;
896 unsigned int mtu, hlen, nexthdr_offset;
897 ktime_t tstamp = skb->tstamp;
898 int hroom, err = 0;
899 __be32 frag_id;
900 u8 *prevhdr, nexthdr = 0;
901
902 if (!ipv6_mod_enabled()) {
903 kfree_skb(skb);
904 return -EAFNOSUPPORT;
905 }
906
907 err = ip6_find_1stfragopt(skb, &prevhdr);
908 if (err < 0)
909 goto fail;
910 hlen = err;
911 nexthdr = *prevhdr;
912 nexthdr_offset = prevhdr - skb_network_header(skb);
913
914 mtu = ip6_skb_dst_mtu(skb);
915
916 /* We must not fragment if the socket is set to force MTU discovery
917 * or if the skb it not generated by a local socket.
918 */
919 if (unlikely(!skb->ignore_df && skb->len > mtu))
920 goto fail_toobig;
921
922 if (IP6CB(skb)->frag_max_size) {
923 if (IP6CB(skb)->frag_max_size > mtu)
924 goto fail_toobig;
925
926 /* don't send fragments larger than what we received */
927 mtu = IP6CB(skb)->frag_max_size;
928 if (mtu < IPV6_MIN_MTU)
929 mtu = IPV6_MIN_MTU;
930 }
931
932 if (np) {
933 u32 frag_size = READ_ONCE(np->frag_size);
934
935 if (frag_size && frag_size < mtu)
936 mtu = frag_size;
937 }
938 if (mtu < hlen + sizeof(struct frag_hdr) + 8)
939 goto fail_toobig;
940 mtu -= hlen + sizeof(struct frag_hdr);
941
942 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
943 &ipv6_hdr(skb)->saddr);
944
945 if (skb->ip_summed == CHECKSUM_PARTIAL &&
946 (err = skb_checksum_help(skb)))
947 goto fail;
948
949 prevhdr = skb_network_header(skb) + nexthdr_offset;
950 hroom = LL_RESERVED_SPACE(rt->dst.dev);
951 if (skb_has_frag_list(skb)) {
952 unsigned int first_len = skb_pagelen(skb);
953 struct ip6_fraglist_iter iter;
954 struct sk_buff *frag2;
955
956 if (first_len - hlen > mtu ||
957 ((first_len - hlen) & 7) ||
958 skb_cloned(skb) ||
959 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
960 goto slow_path;
961
962 skb_walk_frags(skb, frag) {
963 /* Correct geometry. */
964 if (frag->len > mtu ||
965 ((frag->len & 7) && frag->next) ||
966 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr)))
967 goto slow_path_clean;
968
969 /* Partially cloned skb? */
970 if (skb_shared(frag))
971 goto slow_path_clean;
972
973 BUG_ON(frag->sk);
974 if (skb->sk) {
975 frag->sk = skb->sk;
976 frag->destructor = sock_wfree;
977 }
978 skb->truesize -= frag->truesize;
979 }
980
981 err = ip6_fraglist_init(skb, hlen, prevhdr, nexthdr, frag_id,
982 &iter);
983 if (err < 0)
984 goto fail;
985
986 /* We prevent @rt from being freed. */
987 rcu_read_lock();
988
989 for (;;) {
990 /* Prepare header of the next frame,
991 * before previous one went down. */
992 if (iter.frag)
993 ip6_fraglist_prepare(skb, &iter);
994
995 skb_set_delivery_time(skb, tstamp, tstamp_type);
996 err = output(net, sk, skb);
997 if (!err)
998 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
999 IPSTATS_MIB_FRAGCREATES);
1000
1001 if (err || !iter.frag)
1002 break;
1003
1004 skb = ip6_fraglist_next(&iter);
1005 }
1006
1007 kfree(iter.tmp_hdr);
1008
1009 if (err == 0) {
1010 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
1011 IPSTATS_MIB_FRAGOKS);
1012 rcu_read_unlock();
1013 return 0;
1014 }
1015
1016 kfree_skb_list(iter.frag);
1017
1018 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
1019 IPSTATS_MIB_FRAGFAILS);
1020 rcu_read_unlock();
1021 return err;
1022
1023 slow_path_clean:
1024 skb_walk_frags(skb, frag2) {
1025 if (frag2 == frag)
1026 break;
1027 frag2->sk = NULL;
1028 frag2->destructor = NULL;
1029 skb->truesize += frag2->truesize;
1030 }
1031 }
1032
1033 slow_path:
1034 /*
1035 * Fragment the datagram.
1036 */
1037
1038 ip6_frag_init(skb, hlen, mtu, rt->dst.dev->needed_tailroom,
1039 LL_RESERVED_SPACE(rt->dst.dev), prevhdr, nexthdr, frag_id,
1040 &state);
1041
1042 /*
1043 * Keep copying data until we run out.
1044 */
1045
1046 while (state.left > 0) {
1047 frag = ip6_frag_next(skb, &state);
1048 if (IS_ERR(frag)) {
1049 err = PTR_ERR(frag);
1050 goto fail;
1051 }
1052
1053 /*
1054 * Put this fragment into the sending queue.
1055 */
1056 skb_set_delivery_time(frag, tstamp, tstamp_type);
1057 err = output(net, sk, frag);
1058 if (err)
1059 goto fail;
1060
1061 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1062 IPSTATS_MIB_FRAGCREATES);
1063 }
1064 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1065 IPSTATS_MIB_FRAGOKS);
1066 consume_skb(skb);
1067 return err;
1068
1069 fail_toobig:
1070 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
1071 err = -EMSGSIZE;
1072
1073 fail:
1074 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1075 IPSTATS_MIB_FRAGFAILS);
1076 kfree_skb(skb);
1077 return err;
1078 }
1079 EXPORT_SYMBOL_GPL(ip6_fragment);
1080
ip6_rt_check(const struct rt6key * rt_key,const struct in6_addr * fl_addr,const struct in6_addr * addr_cache)1081 static inline int ip6_rt_check(const struct rt6key *rt_key,
1082 const struct in6_addr *fl_addr,
1083 const struct in6_addr *addr_cache)
1084 {
1085 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
1086 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache));
1087 }
1088
ip6_sk_dst_check(struct sock * sk,struct dst_entry * dst,const struct flowi6 * fl6)1089 static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
1090 struct dst_entry *dst,
1091 const struct flowi6 *fl6)
1092 {
1093 struct ipv6_pinfo *np = inet6_sk(sk);
1094 struct rt6_info *rt;
1095
1096 if (!dst)
1097 goto out;
1098
1099 if (dst->ops->family != AF_INET6) {
1100 dst_release(dst);
1101 return NULL;
1102 }
1103
1104 rt = dst_rt6_info(dst);
1105 /* Yes, checking route validity in not connected
1106 * case is not very simple. Take into account,
1107 * that we do not support routing by source, TOS,
1108 * and MSG_DONTROUTE --ANK (980726)
1109 *
1110 * 1. ip6_rt_check(): If route was host route,
1111 * check that cached destination is current.
1112 * If it is network route, we still may
1113 * check its validity using saved pointer
1114 * to the last used address: daddr_cache.
1115 * We do not want to save whole address now,
1116 * (because main consumer of this service
1117 * is tcp, which has not this problem),
1118 * so that the last trick works only on connected
1119 * sockets.
1120 * 2. oif also should be the same.
1121 */
1122 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr,
1123 np->daddr_cache ? &sk->sk_v6_daddr : NULL) ||
1124 #ifdef CONFIG_IPV6_SUBTREES
1125 ip6_rt_check(&rt->rt6i_src, &fl6->saddr,
1126 np->saddr_cache ? &np->saddr : NULL) ||
1127 #endif
1128 (fl6->flowi6_oif && fl6->flowi6_oif != dst_dev(dst)->ifindex)) {
1129 dst_release(dst);
1130 dst = NULL;
1131 }
1132
1133 out:
1134 return dst;
1135 }
1136
ip6_dst_lookup_tail(struct net * net,const struct sock * sk,struct dst_entry ** dst,struct flowi6 * fl6)1137 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
1138 struct dst_entry **dst, struct flowi6 *fl6)
1139 {
1140 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
1141 struct neighbour *n;
1142 struct rt6_info *rt;
1143 #endif
1144 int err;
1145 int flags = 0;
1146
1147 /* The correct way to handle this would be to do
1148 * ip6_route_get_saddr, and then ip6_route_output; however,
1149 * the route-specific preferred source forces the
1150 * ip6_route_output call _before_ ip6_route_get_saddr.
1151 *
1152 * In source specific routing (no src=any default route),
1153 * ip6_route_output will fail given src=any saddr, though, so
1154 * that's why we try it again later.
1155 */
1156 if (ipv6_addr_any(&fl6->saddr)) {
1157 struct fib6_info *from;
1158 struct rt6_info *rt;
1159
1160 *dst = ip6_route_output(net, sk, fl6);
1161 rt = (*dst)->error ? NULL : dst_rt6_info(*dst);
1162
1163 rcu_read_lock();
1164 from = rt ? rcu_dereference(rt->from) : NULL;
1165 err = ip6_route_get_saddr(net, from, &fl6->daddr,
1166 sk ? READ_ONCE(inet6_sk(sk)->srcprefs) : 0,
1167 fl6->flowi6_l3mdev,
1168 &fl6->saddr);
1169 rcu_read_unlock();
1170
1171 if (err)
1172 goto out_err_release;
1173
1174 /* If we had an erroneous initial result, pretend it
1175 * never existed and let the SA-enabled version take
1176 * over.
1177 */
1178 if ((*dst)->error) {
1179 dst_release(*dst);
1180 *dst = NULL;
1181 }
1182
1183 if (fl6->flowi6_oif)
1184 flags |= RT6_LOOKUP_F_IFACE;
1185 }
1186
1187 if (!*dst)
1188 *dst = ip6_route_output_flags(net, sk, fl6, flags);
1189
1190 err = (*dst)->error;
1191 if (err)
1192 goto out_err_release;
1193
1194 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
1195 /*
1196 * Here if the dst entry we've looked up
1197 * has a neighbour entry that is in the INCOMPLETE
1198 * state and the src address from the flow is
1199 * marked as OPTIMISTIC, we release the found
1200 * dst entry and replace it instead with the
1201 * dst entry of the nexthop router
1202 */
1203 rt = dst_rt6_info(*dst);
1204 rcu_read_lock();
1205 n = __ipv6_neigh_lookup_noref(rt->dst.dev,
1206 rt6_nexthop(rt, &fl6->daddr));
1207 err = n && !(READ_ONCE(n->nud_state) & NUD_VALID) ? -EINVAL : 0;
1208 rcu_read_unlock();
1209
1210 if (err) {
1211 struct inet6_ifaddr *ifp;
1212 struct flowi6 fl_gw6;
1213 int redirect;
1214
1215 ifp = ipv6_get_ifaddr(net, &fl6->saddr,
1216 (*dst)->dev, 1);
1217
1218 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
1219 if (ifp)
1220 in6_ifa_put(ifp);
1221
1222 if (redirect) {
1223 /*
1224 * We need to get the dst entry for the
1225 * default router instead
1226 */
1227 dst_release(*dst);
1228 memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
1229 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
1230 *dst = ip6_route_output(net, sk, &fl_gw6);
1231 err = (*dst)->error;
1232 if (err)
1233 goto out_err_release;
1234 }
1235 }
1236 #endif
1237 if (ipv6_addr_v4mapped(&fl6->saddr) &&
1238 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) {
1239 err = -EAFNOSUPPORT;
1240 goto out_err_release;
1241 }
1242
1243 return 0;
1244
1245 out_err_release:
1246 dst_release(*dst);
1247 *dst = NULL;
1248
1249 if (err == -ENETUNREACH)
1250 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
1251 return err;
1252 }
1253
1254 /**
1255 * ip6_dst_lookup - perform route lookup on flow
1256 * @net: Network namespace to perform lookup in
1257 * @sk: socket which provides route info
1258 * @dst: pointer to dst_entry * for result
1259 * @fl6: flow to lookup
1260 *
1261 * This function performs a route lookup on the given flow.
1262 *
1263 * It returns zero on success, or a standard errno code on error.
1264 */
ip6_dst_lookup(struct net * net,struct sock * sk,struct dst_entry ** dst,struct flowi6 * fl6)1265 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst,
1266 struct flowi6 *fl6)
1267 {
1268 *dst = NULL;
1269 return ip6_dst_lookup_tail(net, sk, dst, fl6);
1270 }
1271 EXPORT_SYMBOL_GPL(ip6_dst_lookup);
1272
1273 /**
1274 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec
1275 * @net: Network namespace to perform lookup in
1276 * @sk: socket which provides route info
1277 * @fl6: flow to lookup
1278 * @final_dst: final destination address for ipsec lookup
1279 *
1280 * This function performs a route lookup on the given flow.
1281 *
1282 * It returns a valid dst pointer on success, or a pointer encoded
1283 * error code.
1284 */
ip6_dst_lookup_flow(struct net * net,const struct sock * sk,struct flowi6 * fl6,const struct in6_addr * final_dst)1285 struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, struct flowi6 *fl6,
1286 const struct in6_addr *final_dst)
1287 {
1288 struct dst_entry *dst = NULL;
1289 int err;
1290
1291 if (!ipv6_mod_enabled())
1292 return ERR_PTR(-EAFNOSUPPORT);
1293 err = ip6_dst_lookup_tail(net, sk, &dst, fl6);
1294 if (err)
1295 return ERR_PTR(err);
1296 if (final_dst)
1297 fl6->daddr = *final_dst;
1298
1299 return xfrm_lookup_route(net, dst, flowi6_to_flowi(fl6), sk, 0);
1300 }
1301 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);
1302
1303 /**
1304 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1305 * @sk: socket which provides the dst cache and route info
1306 * @fl6: flow to lookup
1307 * @final_dst: final destination address for ipsec lookup
1308 * @connected: whether @sk is connected or not
1309 *
1310 * This function performs a route lookup on the given flow with the
1311 * possibility of using the cached route in the socket if it is valid.
1312 * It will take the socket dst lock when operating on the dst cache.
1313 * As a result, this function can only be used in process context.
1314 *
1315 * In addition, for a connected socket, cache the dst in the socket
1316 * if the current cache is not valid.
1317 *
1318 * It returns a valid dst pointer on success, or a pointer encoded
1319 * error code.
1320 */
ip6_sk_dst_lookup_flow(struct sock * sk,struct flowi6 * fl6,const struct in6_addr * final_dst,bool connected)1321 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1322 const struct in6_addr *final_dst,
1323 bool connected)
1324 {
1325 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
1326
1327 dst = ip6_sk_dst_check(sk, dst, fl6);
1328 if (dst)
1329 return dst;
1330
1331 dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_dst);
1332 if (connected && !IS_ERR(dst))
1333 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6);
1334
1335 return dst;
1336 }
1337
ip6_opt_dup(struct ipv6_opt_hdr * src,gfp_t gfp)1338 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
1339 gfp_t gfp)
1340 {
1341 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1342 }
1343
ip6_rthdr_dup(struct ipv6_rt_hdr * src,gfp_t gfp)1344 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
1345 gfp_t gfp)
1346 {
1347 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1348 }
1349
ip6_append_data_mtu(unsigned int * mtu,int * maxfraglen,unsigned int fragheaderlen,struct sk_buff * skb,struct rt6_info * rt,unsigned int orig_mtu)1350 static void ip6_append_data_mtu(unsigned int *mtu,
1351 int *maxfraglen,
1352 unsigned int fragheaderlen,
1353 struct sk_buff *skb,
1354 struct rt6_info *rt,
1355 unsigned int orig_mtu)
1356 {
1357 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
1358 if (!skb) {
1359 /* first fragment, reserve header_len */
1360 *mtu = orig_mtu - rt->dst.header_len;
1361
1362 } else {
1363 /*
1364 * this fragment is not first, the headers
1365 * space is regarded as data space.
1366 */
1367 *mtu = orig_mtu;
1368 }
1369 *maxfraglen = ((*mtu - fragheaderlen) & ~7)
1370 + fragheaderlen - sizeof(struct frag_hdr);
1371 }
1372 }
1373
ip6_setup_cork(struct sock * sk,struct inet_cork_full * cork,struct ipcm6_cookie * ipc6,struct rt6_info * rt)1374 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
1375 struct ipcm6_cookie *ipc6,
1376 struct rt6_info *rt)
1377 {
1378 struct ipv6_txoptions *nopt, *opt = ipc6->opt;
1379 struct inet6_cork *v6_cork = &cork->base6;
1380 struct ipv6_pinfo *np = inet6_sk(sk);
1381 unsigned int mtu, frag_size;
1382
1383 /* callers pass dst together with a reference, set it first so
1384 * ip6_cork_release() can put it down even in case of an error.
1385 */
1386 cork->base.dst = &rt->dst;
1387
1388 /*
1389 * setup for corking
1390 */
1391 if (unlikely(opt)) {
1392 if (WARN_ON(v6_cork->opt))
1393 return -EINVAL;
1394
1395 nopt = v6_cork->opt = kzalloc_obj(*opt, sk->sk_allocation);
1396 if (unlikely(!nopt))
1397 return -ENOBUFS;
1398
1399 nopt->tot_len = sizeof(*opt);
1400 nopt->opt_flen = opt->opt_flen;
1401 nopt->opt_nflen = opt->opt_nflen;
1402
1403 nopt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation);
1404 if (opt->dst0opt && !nopt->dst0opt)
1405 return -ENOBUFS;
1406
1407 nopt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation);
1408 if (opt->dst1opt && !nopt->dst1opt)
1409 return -ENOBUFS;
1410
1411 nopt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation);
1412 if (opt->hopopt && !nopt->hopopt)
1413 return -ENOBUFS;
1414
1415 nopt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation);
1416 if (opt->srcrt && !nopt->srcrt)
1417 return -ENOBUFS;
1418
1419 /* need source address above miyazawa*/
1420 }
1421 v6_cork->hop_limit = ipc6->hlimit;
1422 v6_cork->tclass = ipc6->tclass;
1423 v6_cork->dontfrag = ipc6->dontfrag;
1424 if (rt->dst.flags & DST_XFRM_TUNNEL)
1425 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ?
1426 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(&rt->dst);
1427 else
1428 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ?
1429 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(xfrm_dst_path(&rt->dst));
1430
1431 frag_size = READ_ONCE(np->frag_size);
1432 if (frag_size && frag_size < mtu)
1433 mtu = frag_size;
1434
1435 cork->base.fragsize = mtu;
1436 cork->base.gso_size = ipc6->gso_size;
1437 cork->base.tx_flags = 0;
1438 cork->base.mark = ipc6->sockc.mark;
1439 cork->base.priority = ipc6->sockc.priority;
1440 sock_tx_timestamp(sk, &ipc6->sockc, &cork->base.tx_flags);
1441 if (ipc6->sockc.tsflags & SOCKCM_FLAG_TS_OPT_ID) {
1442 cork->base.flags |= IPCORK_TS_OPT_ID;
1443 cork->base.ts_opt_id = ipc6->sockc.ts_opt_id;
1444 }
1445 cork->base.length = 0;
1446 cork->base.transmit_time = ipc6->sockc.transmit_time;
1447
1448 return 0;
1449 }
1450
__ip6_append_data(struct sock * sk,struct sk_buff_head * queue,struct inet_cork_full * cork_full,struct page_frag * pfrag,int getfrag (void * from,char * to,int offset,int len,int odd,struct sk_buff * skb),void * from,size_t length,int transhdrlen,unsigned int flags)1451 static int __ip6_append_data(struct sock *sk,
1452 struct sk_buff_head *queue,
1453 struct inet_cork_full *cork_full,
1454 struct page_frag *pfrag,
1455 int getfrag(void *from, char *to, int offset,
1456 int len, int odd, struct sk_buff *skb),
1457 void *from, size_t length, int transhdrlen,
1458 unsigned int flags)
1459 {
1460 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu;
1461 struct inet6_cork *v6_cork = &cork_full->base6;
1462 struct inet_cork *cork = &cork_full->base;
1463 struct flowi6 *fl6 = &cork_full->fl.u.ip6;
1464 struct sk_buff *skb, *skb_prev = NULL;
1465 struct ubuf_info *uarg = NULL;
1466 int exthdrlen = 0;
1467 int dst_exthdrlen = 0;
1468 int hh_len;
1469 int copy;
1470 int err;
1471 int offset = 0;
1472 bool zc = false;
1473 u32 tskey = 0;
1474 struct rt6_info *rt = dst_rt6_info(cork->dst);
1475 bool paged, hold_tskey = false, extra_uref = false;
1476 struct ipv6_txoptions *opt = v6_cork->opt;
1477 int csummode = CHECKSUM_NONE;
1478 unsigned int maxnonfragsize, headersize;
1479 unsigned int wmem_alloc_delta = 0;
1480
1481 skb = skb_peek_tail(queue);
1482 if (!skb) {
1483 exthdrlen = opt ? opt->opt_flen : 0;
1484 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
1485 }
1486
1487 paged = !!cork->gso_size;
1488 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;
1489 orig_mtu = mtu;
1490
1491 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1492
1493 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
1494 (opt ? opt->opt_nflen : 0);
1495
1496 headersize = sizeof(struct ipv6hdr) +
1497 (opt ? opt->opt_flen + opt->opt_nflen : 0) +
1498 rt->rt6i_nfheader_len;
1499
1500 if (mtu <= fragheaderlen ||
1501 ((mtu - fragheaderlen) & ~7) + fragheaderlen <= sizeof(struct frag_hdr))
1502 goto emsgsize;
1503
1504 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
1505 sizeof(struct frag_hdr);
1506
1507 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit
1508 * the first fragment
1509 */
1510 if (headersize + transhdrlen > mtu)
1511 goto emsgsize;
1512
1513 if (cork->length + length > mtu - headersize && v6_cork->dontfrag &&
1514 (sk->sk_protocol == IPPROTO_UDP ||
1515 sk->sk_protocol == IPPROTO_ICMPV6 ||
1516 sk->sk_protocol == IPPROTO_RAW)) {
1517 ipv6_local_rxpmtu(sk, fl6, mtu - headersize +
1518 sizeof(struct ipv6hdr));
1519 goto emsgsize;
1520 }
1521
1522 if (ip6_sk_ignore_df(sk))
1523 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN;
1524 else
1525 maxnonfragsize = mtu;
1526
1527 if (cork->length + length > maxnonfragsize - headersize) {
1528 emsgsize:
1529 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0);
1530 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu);
1531 return -EMSGSIZE;
1532 }
1533
1534 /* CHECKSUM_PARTIAL only with no extension headers and when
1535 * we are not going to fragment
1536 */
1537 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP &&
1538 headersize == sizeof(struct ipv6hdr) &&
1539 length <= mtu - headersize &&
1540 (!(flags & MSG_MORE) || cork->gso_size) &&
1541 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
1542 csummode = CHECKSUM_PARTIAL;
1543
1544 if ((flags & MSG_ZEROCOPY) && length) {
1545 struct msghdr *msg = from;
1546
1547 if (getfrag == ip_generic_getfrag && msg->msg_ubuf) {
1548 if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb))
1549 return -EINVAL;
1550
1551 /* Leave uarg NULL if can't zerocopy, callers should
1552 * be able to handle it.
1553 */
1554 if ((rt->dst.dev->features & NETIF_F_SG) &&
1555 csummode == CHECKSUM_PARTIAL) {
1556 paged = true;
1557 zc = true;
1558 uarg = msg->msg_ubuf;
1559 }
1560 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1561 uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb),
1562 false);
1563 if (!uarg)
1564 return -ENOBUFS;
1565 extra_uref = !skb_zcopy(skb); /* only ref on new uarg */
1566 if (rt->dst.dev->features & NETIF_F_SG &&
1567 csummode == CHECKSUM_PARTIAL) {
1568 paged = true;
1569 zc = true;
1570 } else {
1571 uarg_to_msgzc(uarg)->zerocopy = 0;
1572 skb_zcopy_set(skb, uarg, &extra_uref);
1573 }
1574 }
1575 } else if ((flags & MSG_SPLICE_PAGES) && length) {
1576 if (inet_test_bit(HDRINCL, sk))
1577 return -EPERM;
1578 if (rt->dst.dev->features & NETIF_F_SG &&
1579 getfrag == ip_generic_getfrag)
1580 /* We need an empty buffer to attach stuff to */
1581 paged = true;
1582 else
1583 flags &= ~MSG_SPLICE_PAGES;
1584 }
1585
1586 if (cork->tx_flags & SKBTX_ANY_TSTAMP &&
1587 READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
1588 if (cork->flags & IPCORK_TS_OPT_ID) {
1589 tskey = cork->ts_opt_id;
1590 } else {
1591 tskey = atomic_inc_return(&sk->sk_tskey) - 1;
1592 hold_tskey = true;
1593 }
1594 }
1595
1596 /*
1597 * Let's try using as much space as possible.
1598 * Use MTU if total length of the message fits into the MTU.
1599 * Otherwise, we need to reserve fragment header and
1600 * fragment alignment (= 8-15 octects, in total).
1601 *
1602 * Note that we may need to "move" the data from the tail
1603 * of the buffer to the new fragment when we split
1604 * the message.
1605 *
1606 * FIXME: It may be fragmented into multiple chunks
1607 * at once if non-fragmentable extension headers
1608 * are too large.
1609 * --yoshfuji
1610 */
1611
1612 cork->length += length;
1613 if (!skb)
1614 goto alloc_new_skb;
1615
1616 while (length > 0) {
1617 /* Check if the remaining data fits into current packet. */
1618 copy = (cork->length <= mtu ? mtu : maxfraglen) - skb->len;
1619 if (copy < length)
1620 copy = maxfraglen - skb->len;
1621
1622 if (copy <= 0) {
1623 char *data;
1624 unsigned int datalen;
1625 unsigned int fraglen;
1626 unsigned int fraggap;
1627 unsigned int alloclen, alloc_extra;
1628 unsigned int pagedlen;
1629 alloc_new_skb:
1630 /* There's no room in the current skb */
1631 if (skb)
1632 fraggap = skb->len - maxfraglen;
1633 else
1634 fraggap = 0;
1635 /* update mtu and maxfraglen if necessary */
1636 if (!skb || !skb_prev)
1637 ip6_append_data_mtu(&mtu, &maxfraglen,
1638 fragheaderlen, skb, rt,
1639 orig_mtu);
1640
1641 skb_prev = skb;
1642
1643 /*
1644 * If remaining data exceeds the mtu,
1645 * we know we need more fragment(s).
1646 */
1647 datalen = length + fraggap;
1648
1649 if (datalen > (cork->length <= mtu ? mtu : maxfraglen) - fragheaderlen)
1650 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
1651 fraglen = datalen + fragheaderlen;
1652 pagedlen = 0;
1653
1654 alloc_extra = hh_len;
1655 alloc_extra += dst_exthdrlen;
1656 alloc_extra += rt->dst.trailer_len;
1657
1658 /* We just reserve space for fragment header.
1659 * Note: this may be overallocation if the message
1660 * (without MSG_MORE) fits into the MTU.
1661 */
1662 alloc_extra += sizeof(struct frag_hdr);
1663
1664 if ((flags & MSG_MORE) &&
1665 !(rt->dst.dev->features&NETIF_F_SG))
1666 alloclen = mtu;
1667 else if (!paged &&
1668 (fraglen + alloc_extra < SKB_MAX_ALLOC ||
1669 !(rt->dst.dev->features & NETIF_F_SG)))
1670 alloclen = fraglen;
1671 else {
1672 alloclen = fragheaderlen + transhdrlen + fraggap;
1673 pagedlen = datalen - transhdrlen - fraggap;
1674 }
1675 alloclen += alloc_extra;
1676
1677 if (datalen != length + fraggap) {
1678 /*
1679 * this is not the last fragment, the trailer
1680 * space is regarded as data space.
1681 */
1682 datalen += rt->dst.trailer_len;
1683 }
1684
1685 fraglen = datalen + fragheaderlen;
1686
1687 copy = datalen - transhdrlen - fraggap - pagedlen;
1688 if (copy < 0) {
1689 err = -EINVAL;
1690 goto error;
1691 }
1692 if (transhdrlen) {
1693 skb = sock_alloc_send_skb(sk, alloclen,
1694 (flags & MSG_DONTWAIT), &err);
1695 } else {
1696 skb = NULL;
1697 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
1698 2 * sk->sk_sndbuf)
1699 skb = alloc_skb(alloclen,
1700 sk->sk_allocation);
1701 if (unlikely(!skb))
1702 err = -ENOBUFS;
1703 }
1704 if (!skb)
1705 goto error;
1706 /*
1707 * Fill in the control structures
1708 */
1709 skb->protocol = htons(ETH_P_IPV6);
1710 skb->ip_summed = csummode;
1711 skb->csum = 0;
1712 /* reserve for fragmentation and ipsec header */
1713 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
1714 dst_exthdrlen);
1715
1716 /*
1717 * Find where to start putting bytes
1718 */
1719 data = skb_put(skb, fraglen - pagedlen);
1720 skb_set_network_header(skb, exthdrlen);
1721 data += fragheaderlen;
1722 skb->transport_header = (skb->network_header +
1723 fragheaderlen);
1724 if (fraggap) {
1725 skb->csum = skb_copy_and_csum_bits(
1726 skb_prev, maxfraglen,
1727 data + transhdrlen, fraggap);
1728 skb_prev->csum = csum_sub(skb_prev->csum,
1729 skb->csum);
1730 data += fraggap;
1731 pskb_trim_unique(skb_prev, maxfraglen);
1732 }
1733 if (copy > 0 &&
1734 INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
1735 from, data + transhdrlen, offset,
1736 copy, fraggap, skb) < 0) {
1737 err = -EFAULT;
1738 kfree_skb(skb);
1739 goto error;
1740 } else if (flags & MSG_SPLICE_PAGES) {
1741 copy = 0;
1742 }
1743
1744 offset += copy;
1745 length -= copy + transhdrlen;
1746 transhdrlen = 0;
1747 exthdrlen = 0;
1748 dst_exthdrlen = 0;
1749
1750 /* Only the initial fragment is time stamped */
1751 skb_shinfo(skb)->tx_flags = cork->tx_flags;
1752 cork->tx_flags = 0;
1753 skb_shinfo(skb)->tskey = tskey;
1754 tskey = 0;
1755 skb_zcopy_set(skb, uarg, &extra_uref);
1756
1757 if ((flags & MSG_CONFIRM) && !skb_prev)
1758 skb_set_dst_pending_confirm(skb, 1);
1759
1760 /*
1761 * Put the packet on the pending queue
1762 */
1763 if (!skb->destructor) {
1764 skb->destructor = sock_wfree;
1765 skb->sk = sk;
1766 wmem_alloc_delta += skb->truesize;
1767 }
1768 __skb_queue_tail(queue, skb);
1769 continue;
1770 }
1771
1772 if (copy > length)
1773 copy = length;
1774
1775 if (!(rt->dst.dev->features&NETIF_F_SG) &&
1776 skb_tailroom(skb) >= copy) {
1777 unsigned int off;
1778
1779 off = skb->len;
1780 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
1781 from, skb_put(skb, copy),
1782 offset, copy, off, skb) < 0) {
1783 __skb_trim(skb, off);
1784 err = -EFAULT;
1785 goto error;
1786 }
1787 } else if (flags & MSG_SPLICE_PAGES) {
1788 struct msghdr *msg = from;
1789
1790 err = -EIO;
1791 if (WARN_ON_ONCE(copy > msg->msg_iter.count))
1792 goto error;
1793
1794 err = skb_splice_from_iter(skb, &msg->msg_iter, copy);
1795 if (err < 0)
1796 goto error;
1797 copy = err;
1798 if (!(flags & MSG_NO_SHARED_FRAGS))
1799 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1800 wmem_alloc_delta += copy;
1801 } else if (!zc) {
1802 int i = skb_shinfo(skb)->nr_frags;
1803
1804 err = -ENOMEM;
1805 if (!sk_page_frag_refill(sk, pfrag))
1806 goto error;
1807
1808 skb_zcopy_downgrade_managed(skb);
1809 if (!skb_can_coalesce(skb, i, pfrag->page,
1810 pfrag->offset)) {
1811 err = -EMSGSIZE;
1812 if (i == MAX_SKB_FRAGS)
1813 goto error;
1814
1815 __skb_fill_page_desc(skb, i, pfrag->page,
1816 pfrag->offset, 0);
1817 skb_shinfo(skb)->nr_frags = ++i;
1818 get_page(pfrag->page);
1819 }
1820 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1821 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
1822 from,
1823 page_address(pfrag->page) + pfrag->offset,
1824 offset, copy, skb->len, skb) < 0)
1825 goto error_efault;
1826
1827 pfrag->offset += copy;
1828 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1829 skb->len += copy;
1830 skb->data_len += copy;
1831 skb->truesize += copy;
1832 wmem_alloc_delta += copy;
1833 } else {
1834 err = skb_zerocopy_iter_dgram(skb, from, copy);
1835 if (err < 0)
1836 goto error;
1837 }
1838 offset += copy;
1839 length -= copy;
1840 }
1841
1842 if (wmem_alloc_delta)
1843 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1844 return 0;
1845
1846 error_efault:
1847 err = -EFAULT;
1848 error:
1849 net_zcopy_put_abort(uarg, extra_uref);
1850 cork->length -= length;
1851 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
1852 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1853 if (hold_tskey)
1854 atomic_dec(&sk->sk_tskey);
1855 return err;
1856 }
1857
ip6_append_data(struct sock * sk,int getfrag (void * from,char * to,int offset,int len,int odd,struct sk_buff * skb),void * from,size_t length,int transhdrlen,struct ipcm6_cookie * ipc6,struct flowi6 * fl6,struct rt6_info * rt,unsigned int flags)1858 int ip6_append_data(struct sock *sk,
1859 int getfrag(void *from, char *to, int offset, int len,
1860 int odd, struct sk_buff *skb),
1861 void *from, size_t length, int transhdrlen,
1862 struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
1863 struct rt6_info *rt, unsigned int flags)
1864 {
1865 struct inet_sock *inet = inet_sk(sk);
1866 int exthdrlen;
1867 int err;
1868
1869 if (flags&MSG_PROBE)
1870 return 0;
1871 if (skb_queue_empty(&sk->sk_write_queue)) {
1872 /*
1873 * setup for corking
1874 */
1875 dst_hold(&rt->dst);
1876 err = ip6_setup_cork(sk, &inet->cork,
1877 ipc6, rt);
1878 if (err)
1879 return err;
1880
1881 inet->cork.fl.u.ip6 = *fl6;
1882 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1883 length += exthdrlen;
1884 transhdrlen += exthdrlen;
1885 } else {
1886 transhdrlen = 0;
1887 }
1888
1889 return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork,
1890 sk_page_frag(sk), getfrag,
1891 from, length, transhdrlen, flags);
1892 }
1893 EXPORT_SYMBOL_GPL(ip6_append_data);
1894
ip6_cork_steal_dst(struct sk_buff * skb,struct inet_cork_full * cork)1895 static void ip6_cork_steal_dst(struct sk_buff *skb, struct inet_cork_full *cork)
1896 {
1897 struct dst_entry *dst = cork->base.dst;
1898
1899 cork->base.dst = NULL;
1900 skb_dst_set(skb, dst);
1901 }
1902
ip6_cork_release(struct inet_cork_full * cork)1903 static void ip6_cork_release(struct inet_cork_full *cork)
1904 {
1905 struct inet6_cork *v6_cork = &cork->base6;
1906
1907 if (unlikely(v6_cork->opt)) {
1908 struct ipv6_txoptions *opt = v6_cork->opt;
1909
1910 kfree(opt->dst0opt);
1911 kfree(opt->dst1opt);
1912 kfree(opt->hopopt);
1913 kfree(opt->srcrt);
1914 kfree(opt);
1915 v6_cork->opt = NULL;
1916 }
1917
1918 if (cork->base.dst) {
1919 dst_release(cork->base.dst);
1920 cork->base.dst = NULL;
1921 }
1922 }
1923
__ip6_make_skb(struct sock * sk,struct sk_buff_head * queue,struct inet_cork_full * cork)1924 struct sk_buff *__ip6_make_skb(struct sock *sk,
1925 struct sk_buff_head *queue,
1926 struct inet_cork_full *cork)
1927 {
1928 struct sk_buff *skb, *tmp_skb;
1929 struct sk_buff **tail_skb;
1930 struct in6_addr *final_dst;
1931 struct net *net = sock_net(sk);
1932 struct ipv6hdr *hdr;
1933 struct ipv6_txoptions *opt;
1934 struct rt6_info *rt = dst_rt6_info(cork->base.dst);
1935 struct flowi6 *fl6 = &cork->fl.u.ip6;
1936 unsigned char proto = fl6->flowi6_proto;
1937
1938 skb = __skb_dequeue(queue);
1939 if (!skb)
1940 goto out;
1941 tail_skb = &(skb_shinfo(skb)->frag_list);
1942
1943 /* move skb->data to ip header from ext header */
1944 if (skb->data < skb_network_header(skb))
1945 __skb_pull(skb, skb_network_offset(skb));
1946 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1947 __skb_pull(tmp_skb, skb_network_header_len(skb));
1948 *tail_skb = tmp_skb;
1949 tail_skb = &(tmp_skb->next);
1950 skb->len += tmp_skb->len;
1951 skb->data_len += tmp_skb->len;
1952 skb->truesize += tmp_skb->truesize;
1953 tmp_skb->destructor = NULL;
1954 tmp_skb->sk = NULL;
1955 }
1956
1957 /* Allow local fragmentation. */
1958 skb->ignore_df = ip6_sk_ignore_df(sk);
1959 __skb_pull(skb, skb_network_header_len(skb));
1960
1961 final_dst = &fl6->daddr;
1962 opt = cork->base6.opt;
1963 if (unlikely(opt)) {
1964 if (opt->opt_flen)
1965 proto = ipv6_push_frag_opts(skb, opt, proto);
1966 if (opt->opt_nflen)
1967 proto = ipv6_push_nfrag_opts(skb, opt, proto,
1968 &final_dst, &fl6->saddr);
1969 }
1970 skb_push(skb, sizeof(struct ipv6hdr));
1971 skb_reset_network_header(skb);
1972 hdr = ipv6_hdr(skb);
1973
1974 ip6_flow_hdr(hdr, cork->base6.tclass,
1975 ip6_make_flowlabel(net, skb, fl6->flowlabel,
1976 ip6_autoflowlabel(net, sk), fl6));
1977 hdr->hop_limit = cork->base6.hop_limit;
1978 hdr->nexthdr = proto;
1979 hdr->saddr = fl6->saddr;
1980 hdr->daddr = *final_dst;
1981
1982 skb->priority = cork->base.priority;
1983 skb->mark = cork->base.mark;
1984 if (sk_is_tcp(sk))
1985 skb_set_delivery_time(skb, cork->base.transmit_time, SKB_CLOCK_MONOTONIC);
1986 else
1987 skb_set_delivery_type_by_clockid(skb, cork->base.transmit_time, sk->sk_clockid);
1988
1989 ip6_cork_steal_dst(skb, cork);
1990 IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS);
1991 if (unlikely(proto == IPPROTO_ICMPV6)) {
1992 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
1993 u8 icmp6_type;
1994
1995 if (sk->sk_socket->type == SOCK_RAW &&
1996 !(fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH))
1997 icmp6_type = fl6->fl6_icmp_type;
1998 else
1999 icmp6_type = icmp6_hdr(skb)->icmp6_type;
2000 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_type);
2001 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
2002 }
2003
2004 ip6_cork_release(cork);
2005 out:
2006 return skb;
2007 }
2008
ip6_send_skb(struct sk_buff * skb)2009 int ip6_send_skb(struct sk_buff *skb)
2010 {
2011 struct net *net = sock_net(skb->sk);
2012 struct rt6_info *rt = dst_rt6_info(skb_dst(skb));
2013 int err;
2014
2015 rcu_read_lock();
2016 err = ip6_local_out(net, skb->sk, skb);
2017 if (err) {
2018 if (err > 0)
2019 err = net_xmit_errno(err);
2020 if (err)
2021 IP6_INC_STATS(net, rt->rt6i_idev,
2022 IPSTATS_MIB_OUTDISCARDS);
2023 }
2024
2025 rcu_read_unlock();
2026 return err;
2027 }
2028
ip6_push_pending_frames(struct sock * sk)2029 int ip6_push_pending_frames(struct sock *sk)
2030 {
2031 struct sk_buff *skb;
2032
2033 skb = ip6_finish_skb(sk);
2034 if (!skb)
2035 return 0;
2036
2037 return ip6_send_skb(skb);
2038 }
2039 EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
2040
__ip6_flush_pending_frames(struct sock * sk,struct sk_buff_head * queue,struct inet_cork_full * cork)2041 static void __ip6_flush_pending_frames(struct sock *sk,
2042 struct sk_buff_head *queue,
2043 struct inet_cork_full *cork)
2044 {
2045 struct sk_buff *skb;
2046
2047 while ((skb = __skb_dequeue_tail(queue)) != NULL) {
2048 if (skb_dst(skb))
2049 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
2050 IPSTATS_MIB_OUTDISCARDS);
2051 kfree_skb(skb);
2052 }
2053
2054 ip6_cork_release(cork);
2055 }
2056
ip6_flush_pending_frames(struct sock * sk)2057 void ip6_flush_pending_frames(struct sock *sk)
2058 {
2059 __ip6_flush_pending_frames(sk, &sk->sk_write_queue,
2060 &inet_sk(sk)->cork);
2061 }
2062 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
2063
ip6_make_skb(struct sock * sk,int getfrag (void * from,char * to,int offset,int len,int odd,struct sk_buff * skb),void * from,size_t length,int transhdrlen,struct ipcm6_cookie * ipc6,struct rt6_info * rt,unsigned int flags,struct inet_cork_full * cork)2064 struct sk_buff *ip6_make_skb(struct sock *sk,
2065 int getfrag(void *from, char *to, int offset,
2066 int len, int odd, struct sk_buff *skb),
2067 void *from, size_t length, int transhdrlen,
2068 struct ipcm6_cookie *ipc6, struct rt6_info *rt,
2069 unsigned int flags, struct inet_cork_full *cork)
2070 {
2071 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
2072 struct sk_buff_head queue;
2073 int err;
2074
2075 if (flags & MSG_PROBE) {
2076 dst_release(&rt->dst);
2077 return NULL;
2078 }
2079
2080 __skb_queue_head_init(&queue);
2081
2082 cork->base.flags = 0;
2083 cork->base.addr = 0;
2084 cork->base.opt = NULL;
2085 cork->base6.opt = NULL;
2086 err = ip6_setup_cork(sk, cork, ipc6, rt);
2087 if (err) {
2088 ip6_cork_release(cork);
2089 return ERR_PTR(err);
2090 }
2091
2092 err = __ip6_append_data(sk, &queue, cork,
2093 ¤t->task_frag, getfrag, from,
2094 length + exthdrlen, transhdrlen + exthdrlen,
2095 flags);
2096 if (err) {
2097 __ip6_flush_pending_frames(sk, &queue, cork);
2098 return ERR_PTR(err);
2099 }
2100
2101 return __ip6_make_skb(sk, &queue, cork);
2102 }
2103