1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * TCP over IPv6
4 * Linux INET6 implementation
5 *
6 * Authors:
7 * Pedro Roque <roque@di.fc.ul.pt>
8 *
9 * Based on:
10 * linux/net/ipv4/tcp.c
11 * linux/net/ipv4/tcp_input.c
12 * linux/net/ipv4/tcp_output.c
13 *
14 * Fixes:
15 * Hideaki YOSHIFUJI : sin6_scope_id support
16 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
17 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
18 * a single port at the same time.
19 * YOSHIFUJI Hideaki @USAGI: convert /proc/net/tcp6 to seq_file.
20 */
21
22 #include <linux/bottom_half.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/types.h>
26 #include <linux/socket.h>
27 #include <linux/sockios.h>
28 #include <linux/net.h>
29 #include <linux/jiffies.h>
30 #include <linux/in.h>
31 #include <linux/in6.h>
32 #include <linux/netdevice.h>
33 #include <linux/init.h>
34 #include <linux/jhash.h>
35 #include <linux/ipsec.h>
36 #include <linux/times.h>
37 #include <linux/slab.h>
38 #include <linux/uaccess.h>
39 #include <linux/ipv6.h>
40 #include <linux/icmpv6.h>
41 #include <linux/random.h>
42 #include <linux/indirect_call_wrapper.h>
43
44 #include <net/aligned_data.h>
45 #include <net/tcp.h>
46 #include <net/ndisc.h>
47 #include <net/inet6_hashtables.h>
48 #include <net/inet6_connection_sock.h>
49 #include <net/ipv6.h>
50 #include <net/transp_v6.h>
51 #include <net/addrconf.h>
52 #include <net/ip6_route.h>
53 #include <net/ip6_checksum.h>
54 #include <net/inet_ecn.h>
55 #include <net/protocol.h>
56 #include <net/xfrm.h>
57 #include <net/snmp.h>
58 #include <net/dsfield.h>
59 #include <net/timewait_sock.h>
60 #include <net/inet_common.h>
61 #include <net/secure_seq.h>
62 #include <net/hotdata.h>
63 #include <net/busy_poll.h>
64 #include <net/rstreason.h>
65 #include <net/psp.h>
66
67 #include <linux/proc_fs.h>
68 #include <linux/seq_file.h>
69
70 #include <crypto/md5.h>
71 #include <crypto/utils.h>
72
73 #include <trace/events/tcp.h>
74
75 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
76 enum sk_rst_reason reason);
77 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
78 struct request_sock *req);
79
80 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
81
82 static const struct inet_connection_sock_af_ops ipv6_mapped;
83 const struct inet_connection_sock_af_ops ipv6_specific;
84 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
85 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
86 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
87 #endif
88
89 /* Helper returning the inet6 address from a given tcp socket.
90 * It can be used in TCP stack instead of inet6_sk(sk).
91 * This avoids a dereference and allow compiler optimizations.
92 * It is a specialized version of inet6_sk_generic().
93 */
94 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
95 struct tcp6_sock, tcp)->inet6)
96
inet6_sk_rx_dst_set(struct sock * sk,const struct sk_buff * skb)97 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
98 {
99 struct dst_entry *dst = skb_dst(skb);
100
101 if (dst && dst_hold_safe(dst)) {
102 rcu_assign_pointer(sk->sk_rx_dst, dst);
103 sk->sk_rx_dst_ifindex = skb->skb_iif;
104 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
105 }
106 }
107
108 INDIRECT_CALLABLE_SCOPE union tcp_seq_and_ts_off
tcp_v6_init_seq_and_ts_off(const struct net * net,const struct sk_buff * skb)109 tcp_v6_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
110 {
111 return secure_tcpv6_seq_and_ts_off(net,
112 ipv6_hdr(skb)->daddr.s6_addr32,
113 ipv6_hdr(skb)->saddr.s6_addr32,
114 tcp_hdr(skb)->dest,
115 tcp_hdr(skb)->source);
116 }
117
tcp_v6_pre_connect(struct sock * sk,struct sockaddr_unsized * uaddr,int addr_len)118 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
119 int addr_len)
120 {
121 /* This check is replicated from tcp_v6_connect() and intended to
122 * prevent BPF program called below from accessing bytes that are out
123 * of the bound specified by user in addr_len.
124 */
125 if (addr_len < SIN6_LEN_RFC2133)
126 return -EINVAL;
127
128 sock_owned_by_me(sk);
129
130 return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
131 }
132
tcp_v6_connect(struct sock * sk,struct sockaddr_unsized * uaddr,int addr_len)133 static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
134 int addr_len)
135 {
136 struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
137 struct inet_connection_sock *icsk = inet_csk(sk);
138 struct inet_timewait_death_row *tcp_death_row;
139 struct ipv6_pinfo *np = tcp_inet6_sk(sk);
140 struct in6_addr *saddr = NULL, *final_p;
141 struct inet_sock *inet = inet_sk(sk);
142 struct tcp_sock *tp = tcp_sk(sk);
143 struct net *net = sock_net(sk);
144 struct ipv6_txoptions *opt;
145 struct dst_entry *dst;
146 struct flowi6 *fl6;
147 int addr_type;
148 int err;
149
150 if (addr_len < SIN6_LEN_RFC2133)
151 return -EINVAL;
152
153 if (usin->sin6_family != AF_INET6)
154 return -EAFNOSUPPORT;
155
156 fl6 = &inet_sk(sk)->cork.fl.u.ip6;
157 memset(fl6, 0, sizeof(*fl6));
158
159 if (inet6_test_bit(SNDFLOW, sk)) {
160 fl6->flowlabel = usin->sin6_flowinfo & IPV6_FLOWINFO_MASK;
161 IP6_ECN_flow_init(fl6->flowlabel);
162 if (fl6->flowlabel & IPV6_FLOWLABEL_MASK) {
163 struct ip6_flowlabel *flowlabel;
164 flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
165 if (IS_ERR(flowlabel))
166 return -EINVAL;
167 fl6_sock_release(flowlabel);
168 }
169 }
170
171 /*
172 * connect() to INADDR_ANY means loopback (BSD'ism).
173 */
174
175 if (ipv6_addr_any(&usin->sin6_addr)) {
176 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
177 ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
178 &usin->sin6_addr);
179 else
180 usin->sin6_addr = in6addr_loopback;
181 }
182
183 addr_type = ipv6_addr_type(&usin->sin6_addr);
184
185 if (addr_type & IPV6_ADDR_MULTICAST)
186 return -ENETUNREACH;
187
188 if (addr_type&IPV6_ADDR_LINKLOCAL) {
189 if (addr_len >= sizeof(struct sockaddr_in6) &&
190 usin->sin6_scope_id) {
191 /* If interface is set while binding, indices
192 * must coincide.
193 */
194 if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
195 return -EINVAL;
196
197 sk->sk_bound_dev_if = usin->sin6_scope_id;
198 }
199
200 /* Connect to link-local address requires an interface */
201 if (!sk->sk_bound_dev_if)
202 return -EINVAL;
203 }
204
205 if (tp->rx_opt.ts_recent_stamp &&
206 !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
207 tp->rx_opt.ts_recent = 0;
208 tp->rx_opt.ts_recent_stamp = 0;
209 WRITE_ONCE(tp->write_seq, 0);
210 }
211
212 sk->sk_v6_daddr = usin->sin6_addr;
213 np->flow_label = fl6->flowlabel;
214
215 /*
216 * TCP over IPv4
217 */
218
219 if (addr_type & IPV6_ADDR_MAPPED) {
220 u32 exthdrlen = icsk->icsk_ext_hdr_len;
221 struct sockaddr_in sin;
222
223 if (ipv6_only_sock(sk))
224 return -ENETUNREACH;
225
226 sin.sin_family = AF_INET;
227 sin.sin_port = usin->sin6_port;
228 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
229
230 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
231 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
232 if (sk_is_mptcp(sk))
233 mptcpv6_handle_mapped(sk, true);
234 sk->sk_backlog_rcv = tcp_v4_do_rcv;
235 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
236 tp->af_specific = &tcp_sock_ipv6_mapped_specific;
237 #endif
238
239 err = tcp_v4_connect(sk, (struct sockaddr_unsized *)&sin, sizeof(sin));
240
241 if (err) {
242 icsk->icsk_ext_hdr_len = exthdrlen;
243 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
244 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
245 if (sk_is_mptcp(sk))
246 mptcpv6_handle_mapped(sk, false);
247 sk->sk_backlog_rcv = tcp_v6_do_rcv;
248 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
249 tp->af_specific = &tcp_sock_ipv6_specific;
250 #endif
251 goto failure;
252 }
253 np->saddr = sk->sk_v6_rcv_saddr;
254
255 return err;
256 }
257
258 if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
259 saddr = &sk->sk_v6_rcv_saddr;
260
261 sk_set_txhash(sk);
262
263 fl6->flowi6_proto = IPPROTO_TCP;
264 fl6->daddr = sk->sk_v6_daddr;
265 fl6->saddr = saddr ? *saddr : np->saddr;
266 fl6->flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
267 fl6->flowi6_oif = sk->sk_bound_dev_if;
268 fl6->flowi6_mark = sk->sk_mark;
269 fl6->fl6_dport = usin->sin6_port;
270 fl6->fl6_sport = inet->inet_sport;
271 if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6->fl6_sport)
272 fl6->flowi6_flags = FLOWI_FLAG_ANY_SPORT;
273 fl6->flowi6_uid = sk_uid(sk);
274
275 opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
276 final_p = fl6_update_dst(fl6, opt, &np->final);
277
278 security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));
279
280 /* Non-zero mp_hash bypasses rt6_multipath_hash() in
281 * fib6_select_path(), letting txhash control ECMP path
282 * selection so that sk_rethink_txhash() rehashes onto a
283 * different path. Policies 1-3 derive a deterministic
284 * hash from the flow keys and must not be overridden.
285 */
286 ip6_ecmp_set_mp_hash(net, fl6, sk->sk_txhash);
287
288 dst = ip6_dst_lookup_flow(net, sk, fl6, final_p);
289 if (IS_ERR(dst)) {
290 err = PTR_ERR(dst);
291 goto failure;
292 }
293
294 tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
295 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
296
297 if (!saddr) {
298 saddr = &fl6->saddr;
299
300 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
301 if (err) {
302 dst_release(dst);
303 goto failure;
304 }
305 }
306
307 /* set the source address */
308 np->saddr = *saddr;
309 inet->inet_rcv_saddr = LOOPBACK4_IPV6;
310
311 sk->sk_gso_type = SKB_GSO_TCPV6;
312 ip6_dst_store(sk, dst, false, false);
313
314 icsk->icsk_ext_hdr_len = psp_sk_overhead(sk);
315 if (opt)
316 icsk->icsk_ext_hdr_len += opt->opt_flen +
317 opt->opt_nflen;
318
319 tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
320
321 inet->inet_dport = usin->sin6_port;
322
323 tcp_set_state(sk, TCP_SYN_SENT);
324 err = inet6_hash_connect(tcp_death_row, sk);
325 if (err)
326 goto late_failure;
327
328 if (likely(!tp->repair)) {
329 union tcp_seq_and_ts_off st;
330
331 st = secure_tcpv6_seq_and_ts_off(net,
332 np->saddr.s6_addr32,
333 sk->sk_v6_daddr.s6_addr32,
334 inet->inet_sport,
335 inet->inet_dport);
336 if (!tp->write_seq)
337 WRITE_ONCE(tp->write_seq, st.seq);
338 WRITE_ONCE(tp->tsoffset, st.ts_off);
339 }
340
341 if (tcp_fastopen_defer_connect(sk, &err))
342 return err;
343 if (err)
344 goto late_failure;
345
346 err = tcp_connect(sk);
347 if (err)
348 goto late_failure;
349
350 return 0;
351
352 late_failure:
353 tcp_set_state(sk, TCP_CLOSE);
354 inet_bhash2_reset_saddr(sk);
355 failure:
356 inet->inet_dport = 0;
357 sk->sk_route_caps = 0;
358 return err;
359 }
360
inet6_csk_update_pmtu(struct sock * sk,u32 mtu)361 static struct dst_entry *inet6_csk_update_pmtu(struct sock *sk, u32 mtu)
362 {
363 struct flowi6 *fl6 = &inet_sk(sk)->cork.fl.u.ip6;
364 struct dst_entry *dst;
365
366 dst = inet6_csk_route_socket(sk, fl6);
367
368 if (IS_ERR(dst))
369 return NULL;
370 dst->ops->update_pmtu(dst, sk, NULL, mtu, true);
371
372 dst = inet6_csk_route_socket(sk, fl6);
373 return IS_ERR(dst) ? NULL : dst;
374 }
375
tcp_v6_mtu_reduced(struct sock * sk)376 static void tcp_v6_mtu_reduced(struct sock *sk)
377 {
378 struct dst_entry *dst;
379 u32 mtu, dmtu;
380
381 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
382 return;
383
384 mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
385
386 /* Drop requests trying to increase our current mss.
387 * Check done in __ip6_rt_update_pmtu() is too late.
388 */
389 if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
390 return;
391
392 dst = inet6_csk_update_pmtu(sk, mtu);
393 if (!dst)
394 return;
395
396 dmtu = dst6_mtu(dst);
397 if (inet_csk(sk)->icsk_pmtu_cookie > dmtu) {
398 tcp_sync_mss(sk, dmtu);
399 tcp_simple_retransmit(sk);
400 }
401 }
402
tcp_v6_err(struct sk_buff * skb,struct inet6_skb_parm * opt,u8 type,u8 code,int offset,__be32 info)403 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
404 u8 type, u8 code, int offset, __be32 info)
405 {
406 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
407 const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
408 struct net *net = dev_net_rcu(skb->dev);
409 struct request_sock *fastopen;
410 struct ipv6_pinfo *np;
411 struct tcp_sock *tp;
412 __u32 seq, snd_una;
413 struct sock *sk;
414 bool fatal;
415 int err;
416
417 sk = __inet6_lookup_established(net, &hdr->daddr, th->dest,
418 &hdr->saddr, ntohs(th->source),
419 skb->dev->ifindex, inet6_sdif(skb));
420
421 if (!sk) {
422 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
423 ICMP6_MIB_INERRORS);
424 return -ENOENT;
425 }
426
427 if (sk->sk_state == TCP_TIME_WAIT) {
428 /* To increase the counter of ignored icmps for TCP-AO */
429 tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
430 inet_twsk_put(inet_twsk(sk));
431 return 0;
432 }
433 seq = ntohl(th->seq);
434 fatal = icmpv6_err_convert(type, code, &err);
435 if (sk->sk_state == TCP_NEW_SYN_RECV) {
436 tcp_req_err(sk, seq, fatal);
437 return 0;
438 }
439
440 if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
441 sock_put(sk);
442 return 0;
443 }
444
445 bh_lock_sock(sk);
446 if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
447 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
448
449 if (sk->sk_state == TCP_CLOSE)
450 goto out;
451
452 if (static_branch_unlikely(&ip6_min_hopcount)) {
453 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
454 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
455 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
456 goto out;
457 }
458 }
459
460 tp = tcp_sk(sk);
461 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
462 fastopen = rcu_dereference(tp->fastopen_rsk);
463 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
464 if (sk->sk_state != TCP_LISTEN &&
465 !between(seq, snd_una, tp->snd_nxt)) {
466 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
467 goto out;
468 }
469
470 np = tcp_inet6_sk(sk);
471
472 if (type == NDISC_REDIRECT) {
473 if (!sock_owned_by_user(sk)) {
474 struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
475
476 if (dst)
477 dst->ops->redirect(dst, sk, skb);
478 }
479 goto out;
480 }
481
482 if (type == ICMPV6_PKT_TOOBIG) {
483 u32 mtu = ntohl(info);
484
485 /* We are not interested in TCP_LISTEN and open_requests
486 * (SYN-ACKs send out by Linux are always <576bytes so
487 * they should go through unfragmented).
488 */
489 if (sk->sk_state == TCP_LISTEN)
490 goto out;
491
492 if (!ip6_sk_accept_pmtu(sk))
493 goto out;
494
495 if (mtu < IPV6_MIN_MTU)
496 goto out;
497
498 WRITE_ONCE(tp->mtu_info, mtu);
499
500 if (!sock_owned_by_user(sk))
501 tcp_v6_mtu_reduced(sk);
502 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
503 &sk->sk_tsq_flags))
504 sock_hold(sk);
505 goto out;
506 }
507
508
509 /* Might be for an request_sock */
510 switch (sk->sk_state) {
511 case TCP_SYN_SENT:
512 case TCP_SYN_RECV:
513 /* Only in fast or simultaneous open. If a fast open socket is
514 * already accepted it is treated as a connected one below.
515 */
516 if (fastopen && !fastopen->sk)
517 break;
518
519 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
520
521 if (!sock_owned_by_user(sk))
522 tcp_done_with_error(sk, err);
523 else
524 WRITE_ONCE(sk->sk_err_soft, err);
525 goto out;
526 case TCP_LISTEN:
527 break;
528 default:
529 /* check if this ICMP message allows revert of backoff.
530 * (see RFC 6069)
531 */
532 if (!fastopen && type == ICMPV6_DEST_UNREACH &&
533 code == ICMPV6_NOROUTE)
534 tcp_ld_RTO_revert(sk, seq);
535 }
536
537 if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
538 WRITE_ONCE(sk->sk_err, err);
539 sk_error_report(sk);
540 } else {
541 WRITE_ONCE(sk->sk_err_soft, err);
542 }
543 out:
544 bh_unlock_sock(sk);
545 sock_put(sk);
546 return 0;
547 }
548
549
tcp_v6_send_synack(const struct sock * sk,struct dst_entry * dst,struct flowi * fl,struct request_sock * req,struct tcp_fastopen_cookie * foc,enum tcp_synack_type synack_type,struct sk_buff * syn_skb)550 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
551 struct flowi *fl,
552 struct request_sock *req,
553 struct tcp_fastopen_cookie *foc,
554 enum tcp_synack_type synack_type,
555 struct sk_buff *syn_skb)
556 {
557 struct inet_request_sock *ireq = inet_rsk(req);
558 const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
559 struct ipv6_txoptions *opt;
560 struct flowi6 *fl6 = &fl->u.ip6;
561 struct sk_buff *skb;
562 int err = -ENOMEM;
563 u8 tclass;
564
565 /* First, grab a route. */
566 if (!dst && (dst = inet6_csk_route_req(sk, NULL, fl6, req,
567 IPPROTO_TCP)) == NULL)
568 goto done;
569
570 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
571
572 if (skb) {
573 tcp_rsk(req)->syn_ect_snt = np->tclass & INET_ECN_MASK;
574 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
575 &ireq->ir_v6_rmt_addr);
576
577 fl6->daddr = ireq->ir_v6_rmt_addr;
578 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
579 fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
580
581 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
582 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
583 (np->tclass & INET_ECN_MASK) :
584 np->tclass;
585
586 if (!INET_ECN_is_capable(tclass) &&
587 tcp_bpf_ca_needs_ecn((struct sock *)req))
588 tclass |= INET_ECN_ECT_0;
589
590 rcu_read_lock();
591 opt = ireq->ipv6_opt;
592 if (!opt)
593 opt = rcu_dereference(np->opt);
594 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
595 opt, tclass, READ_ONCE(sk->sk_priority));
596 rcu_read_unlock();
597 err = net_xmit_eval(err);
598 }
599
600 done:
601 return err;
602 }
603
604
tcp_v6_reqsk_destructor(struct request_sock * req)605 static void tcp_v6_reqsk_destructor(struct request_sock *req)
606 {
607 kfree(inet_rsk(req)->ipv6_opt);
608 consume_skb(inet_rsk(req)->pktopts);
609 }
610
611 #ifdef CONFIG_TCP_MD5SIG
tcp_v6_md5_do_lookup(const struct sock * sk,const struct in6_addr * addr,int l3index)612 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
613 const struct in6_addr *addr,
614 int l3index)
615 {
616 return tcp_md5_do_lookup(sk, l3index,
617 (union tcp_md5_addr *)addr, AF_INET6);
618 }
619
tcp_v6_md5_lookup(const struct sock * sk,const struct sock * addr_sk)620 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
621 const struct sock *addr_sk)
622 {
623 int l3index;
624
625 l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
626 addr_sk->sk_bound_dev_if);
627 return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
628 l3index);
629 }
630
tcp_v6_parse_md5_keys(struct sock * sk,int optname,sockptr_t optval,int optlen)631 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
632 sockptr_t optval, int optlen)
633 {
634 struct tcp_md5sig cmd;
635 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
636 union tcp_ao_addr *addr;
637 int l3index = 0;
638 u8 prefixlen;
639 bool l3flag;
640 u8 flags;
641
642 if (optlen < sizeof(cmd))
643 return -EINVAL;
644
645 if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
646 return -EFAULT;
647
648 if (sin6->sin6_family != AF_INET6)
649 return -EINVAL;
650
651 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
652 l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
653
654 if (optname == TCP_MD5SIG_EXT &&
655 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
656 prefixlen = cmd.tcpm_prefixlen;
657 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
658 prefixlen > 32))
659 return -EINVAL;
660 } else {
661 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
662 }
663
664 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
665 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
666 struct net_device *dev;
667
668 rcu_read_lock();
669 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
670 if (dev && netif_is_l3_master(dev))
671 l3index = dev->ifindex;
672 rcu_read_unlock();
673
674 /* ok to reference set/not set outside of rcu;
675 * right now device MUST be an L3 master
676 */
677 if (!dev || !l3index)
678 return -EINVAL;
679 }
680
681 if (!cmd.tcpm_keylen) {
682 if (ipv6_addr_v4mapped(&sin6->sin6_addr))
683 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
684 AF_INET, prefixlen,
685 l3index, flags);
686 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
687 AF_INET6, prefixlen, l3index, flags);
688 }
689
690 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
691 return -EINVAL;
692
693 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
694 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
695
696 /* Don't allow keys for peers that have a matching TCP-AO key.
697 * See the comment in tcp_ao_add_cmd()
698 */
699 if (tcp_ao_required(sk, addr, AF_INET,
700 l3flag ? l3index : -1, false))
701 return -EKEYREJECTED;
702 return tcp_md5_do_add(sk, addr,
703 AF_INET, prefixlen, l3index, flags,
704 cmd.tcpm_key, cmd.tcpm_keylen);
705 }
706
707 addr = (union tcp_md5_addr *)&sin6->sin6_addr;
708
709 /* Don't allow keys for peers that have a matching TCP-AO key.
710 * See the comment in tcp_ao_add_cmd()
711 */
712 if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
713 return -EKEYREJECTED;
714
715 return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
716 cmd.tcpm_key, cmd.tcpm_keylen);
717 }
718
tcp_v6_md5_hash_headers(struct md5_ctx * ctx,const struct in6_addr * daddr,const struct in6_addr * saddr,const struct tcphdr * th,int nbytes)719 static void tcp_v6_md5_hash_headers(struct md5_ctx *ctx,
720 const struct in6_addr *daddr,
721 const struct in6_addr *saddr,
722 const struct tcphdr *th, int nbytes)
723 {
724 struct {
725 struct tcp6_pseudohdr ip; /* TCP pseudo-header (RFC2460) */
726 struct tcphdr tcp;
727 } h;
728
729 h.ip.saddr = *saddr;
730 h.ip.daddr = *daddr;
731 h.ip.protocol = cpu_to_be32(IPPROTO_TCP);
732 h.ip.len = cpu_to_be32(nbytes);
733 h.tcp = *th;
734 h.tcp.check = 0;
735 md5_update(ctx, (const u8 *)&h, sizeof(h.ip) + sizeof(h.tcp));
736 }
737
738 static noinline_for_stack void
tcp_v6_md5_hash_hdr(char * md5_hash,const struct tcp_md5sig_key * key,const struct in6_addr * daddr,struct in6_addr * saddr,const struct tcphdr * th)739 tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
740 const struct in6_addr *daddr, struct in6_addr *saddr,
741 const struct tcphdr *th)
742 {
743 struct md5_ctx ctx;
744
745 md5_init(&ctx);
746 tcp_v6_md5_hash_headers(&ctx, daddr, saddr, th, th->doff << 2);
747 tcp_md5_hash_key(&ctx, key);
748 md5_final(&ctx, md5_hash);
749 }
750
751 static noinline_for_stack void
tcp_v6_md5_hash_skb(char * md5_hash,const struct tcp_md5sig_key * key,const struct sock * sk,const struct sk_buff * skb)752 tcp_v6_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
753 const struct sock *sk, const struct sk_buff *skb)
754 {
755 const struct tcphdr *th = tcp_hdr(skb);
756 const struct in6_addr *saddr, *daddr;
757 struct md5_ctx ctx;
758
759 if (sk) { /* valid for establish/request sockets */
760 saddr = &sk->sk_v6_rcv_saddr;
761 daddr = &sk->sk_v6_daddr;
762 } else {
763 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
764 saddr = &ip6h->saddr;
765 daddr = &ip6h->daddr;
766 }
767
768 md5_init(&ctx);
769 tcp_v6_md5_hash_headers(&ctx, daddr, saddr, th, skb->len);
770 tcp_md5_hash_skb_data(&ctx, skb, th->doff << 2);
771 tcp_md5_hash_key(&ctx, key);
772 md5_final(&ctx, md5_hash);
773 }
774 #endif
775
tcp_v6_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb,u32 tw_isn)776 static void tcp_v6_init_req(struct request_sock *req,
777 const struct sock *sk_listener,
778 struct sk_buff *skb,
779 u32 tw_isn)
780 {
781 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
782 struct inet_request_sock *ireq = inet_rsk(req);
783 const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
784
785 ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
786 ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
787 ireq->ir_rmt_addr = LOOPBACK4_IPV6;
788 ireq->ir_loc_addr = LOOPBACK4_IPV6;
789
790 /* So that link locals have meaning */
791 if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
792 ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
793 ireq->ir_iif = tcp_v6_iif(skb);
794
795 if (!tw_isn &&
796 (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
797 np->rxopt.bits.rxinfo ||
798 np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
799 np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
800 refcount_inc(&skb->users);
801 ireq->pktopts = skb;
802 }
803 }
804
tcp_v6_route_req(const struct sock * sk,struct sk_buff * skb,struct flowi * fl,struct request_sock * req,u32 tw_isn)805 static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
806 struct sk_buff *skb,
807 struct flowi *fl,
808 struct request_sock *req,
809 u32 tw_isn)
810 {
811 tcp_v6_init_req(req, sk, skb, tw_isn);
812
813 if (security_inet_conn_request(sk, skb, req))
814 return NULL;
815
816 return inet6_csk_route_req(sk, NULL, &fl->u.ip6, req, IPPROTO_TCP);
817 }
818
819 struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
820 .family = AF_INET6,
821 .obj_size = sizeof(struct tcp6_request_sock),
822 .send_ack = tcp_v6_reqsk_send_ack,
823 .destructor = tcp_v6_reqsk_destructor,
824 .send_reset = tcp_v6_send_reset,
825 };
826
827 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
828 .mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) -
829 sizeof(struct ipv6hdr),
830 #ifdef CONFIG_TCP_MD5SIG
831 .req_md5_lookup = tcp_v6_md5_lookup,
832 .calc_md5_hash = tcp_v6_md5_hash_skb,
833 #endif
834 #ifdef CONFIG_TCP_AO
835 .ao_lookup = tcp_v6_ao_lookup_rsk,
836 .ao_calc_key = tcp_v6_ao_calc_key_rsk,
837 .ao_synack_hash = tcp_v6_ao_synack_hash,
838 #endif
839 #ifdef CONFIG_SYN_COOKIES
840 .cookie_init_seq = cookie_v6_init_sequence,
841 #endif
842 .route_req = tcp_v6_route_req,
843 .init_seq_and_ts_off = tcp_v6_init_seq_and_ts_off,
844 .send_synack = tcp_v6_send_synack,
845 };
846
tcp_v6_send_response(const struct sock * sk,struct sk_buff * skb,u32 seq,u32 ack,u32 win,u32 tsval,u32 tsecr,int oif,int rst,u8 tclass,__be32 label,u32 priority,u32 txhash,struct tcp_key * key)847 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
848 u32 ack, u32 win, u32 tsval, u32 tsecr,
849 int oif, int rst, u8 tclass, __be32 label,
850 u32 priority, u32 txhash, struct tcp_key *key)
851 {
852 struct net *net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
853 unsigned int tot_len = sizeof(struct tcphdr);
854 struct sock *ctl_sk = net->ipv6.tcp_sk;
855 const struct tcphdr *th = tcp_hdr(skb);
856 __be32 mrst = 0, *topt;
857 struct dst_entry *dst;
858 struct sk_buff *buff;
859 struct tcphdr *t1;
860 struct flowi6 fl6;
861 u32 mark = 0;
862
863 if (tsecr)
864 tot_len += TCPOLEN_TSTAMP_ALIGNED;
865 if (tcp_key_is_md5(key))
866 tot_len += TCPOLEN_MD5SIG_ALIGNED;
867 if (tcp_key_is_ao(key))
868 tot_len += tcp_ao_len_aligned(key->ao_key);
869
870 #ifdef CONFIG_MPTCP
871 if (rst && !tcp_key_is_md5(key)) {
872 mrst = mptcp_reset_option(skb);
873
874 if (mrst)
875 tot_len += sizeof(__be32);
876 }
877 #endif
878
879 buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
880 if (!buff)
881 return;
882
883 skb_reserve(buff, MAX_TCP_HEADER);
884
885 t1 = skb_push(buff, tot_len);
886 skb_reset_transport_header(buff);
887
888 /* Swap the send and the receive. */
889 memset(t1, 0, sizeof(*t1));
890 t1->dest = th->source;
891 t1->source = th->dest;
892 t1->doff = tot_len / 4;
893 t1->seq = htonl(seq);
894 t1->ack_seq = htonl(ack);
895 t1->ack = !rst || !th->ack;
896 t1->rst = rst;
897 t1->window = htons(win);
898
899 topt = (__be32 *)(t1 + 1);
900
901 if (tsecr) {
902 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
903 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
904 *topt++ = htonl(tsval);
905 *topt++ = htonl(tsecr);
906 }
907
908 if (mrst)
909 *topt++ = mrst;
910
911 #ifdef CONFIG_TCP_MD5SIG
912 if (tcp_key_is_md5(key)) {
913 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
914 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
915 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
916 &ipv6_hdr(skb)->saddr,
917 &ipv6_hdr(skb)->daddr, t1);
918 }
919 #endif
920 #ifdef CONFIG_TCP_AO
921 if (tcp_key_is_ao(key)) {
922 *topt++ = htonl((TCPOPT_AO << 24) |
923 (tcp_ao_len(key->ao_key) << 16) |
924 (key->ao_key->sndid << 8) |
925 (key->rcv_next));
926 memset((u8 *)topt + tcp_ao_maclen(key->ao_key), TCPOPT_NOP,
927 tcp_ao_len_aligned(key->ao_key) - tcp_ao_len(key->ao_key));
928
929 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
930 key->traffic_key,
931 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
932 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
933 t1, key->sne);
934 }
935 #endif
936
937 memset(&fl6, 0, sizeof(fl6));
938 fl6.daddr = ipv6_hdr(skb)->saddr;
939 fl6.saddr = ipv6_hdr(skb)->daddr;
940 fl6.flowlabel = label;
941
942 buff->ip_summed = CHECKSUM_PARTIAL;
943
944 __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
945
946 fl6.flowi6_proto = IPPROTO_TCP;
947 if (rt6_need_strict(&fl6.daddr) && !oif)
948 fl6.flowi6_oif = tcp_v6_iif(skb);
949 else {
950 if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
951 oif = skb->skb_iif;
952
953 fl6.flowi6_oif = oif;
954 }
955
956 if (sk) {
957 /* unconstify the socket only to attach it to buff with care. */
958 skb_set_owner_edemux(buff, (struct sock *)sk);
959 psp_reply_set_decrypted(sk, buff);
960
961 if (sk->sk_state == TCP_TIME_WAIT)
962 mark = inet_twsk(sk)->tw_mark;
963 else
964 mark = READ_ONCE(sk->sk_mark);
965 skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC);
966 }
967 if (txhash) {
968 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
969 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
970
971 /* Select the local ECMP path from the connection's txhash,
972 * so a control packet (RST, or ACK from a time-wait socket)
973 * uses the same nexthop as the data. Only policy 0 uses
974 * mp_hash; policies 1-3 derive a deterministic hash.
975 */
976 ip6_ecmp_set_mp_hash(net, &fl6, txhash);
977 }
978 fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
979 fl6.fl6_dport = t1->dest;
980 fl6.fl6_sport = t1->source;
981 fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
982 security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
983
984 /* Pass a socket to ip6_dst_lookup either it is for RST
985 * Underlying function will use this to retrieve the network
986 * namespace
987 */
988 if (sk && sk->sk_state != TCP_TIME_WAIT)
989 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
990 else
991 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
992 if (!IS_ERR(dst)) {
993 skb_dst_set(buff, dst);
994 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
995 tclass, priority);
996 TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
997 if (rst)
998 TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
999 return;
1000 }
1001
1002 sk_skb_reason_drop(sk, buff, SKB_DROP_REASON_IP_OUTNOROUTES);
1003 }
1004
tcp_v6_send_reset(const struct sock * sk,struct sk_buff * skb,enum sk_rst_reason reason)1005 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
1006 enum sk_rst_reason reason)
1007 {
1008 const struct tcphdr *th = tcp_hdr(skb);
1009 struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1010 const __u8 *md5_hash_location = NULL;
1011 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1012 bool allocated_traffic_key = false;
1013 #endif
1014 const struct tcp_ao_hdr *aoh;
1015 struct tcp_key key = {};
1016 u32 seq = 0, ack_seq = 0;
1017 __be32 label = 0;
1018 u32 priority = 0;
1019 struct net *net;
1020 u32 txhash = 0;
1021 int oif = 0;
1022 #ifdef CONFIG_TCP_MD5SIG
1023 unsigned char newhash[16];
1024 struct sock *sk1 = NULL;
1025 #endif
1026
1027 if (th->rst)
1028 return;
1029
1030 /* If sk not NULL, it means we did a successful lookup and incoming
1031 * route had to be correct. prequeue might have dropped our dst.
1032 */
1033 if (!sk && !ipv6_unicast_destination(skb))
1034 return;
1035
1036 net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
1037 /* Invalid TCP option size or twice included auth */
1038 if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
1039 return;
1040 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1041 rcu_read_lock();
1042 #endif
1043 #ifdef CONFIG_TCP_MD5SIG
1044 if (sk && sk_fullsock(sk)) {
1045 int l3index;
1046
1047 /* sdif set, means packet ingressed via a device
1048 * in an L3 domain and inet_iif is set to it.
1049 */
1050 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1051 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1052 if (key.md5_key)
1053 key.type = TCP_KEY_MD5;
1054 } else if (md5_hash_location) {
1055 int dif = tcp_v6_iif_l3_slave(skb);
1056 int sdif = tcp_v6_sdif(skb);
1057 int l3index;
1058
1059 /*
1060 * active side is lost. Try to find listening socket through
1061 * source port, and then find md5 key through listening socket.
1062 * we are not loose security here:
1063 * Incoming packet is checked with md5 hash with finding key,
1064 * no RST generated if md5 hash doesn't match.
1065 */
1066 sk1 = inet6_lookup_listener(net, NULL, 0, &ipv6h->saddr, th->source,
1067 &ipv6h->daddr, ntohs(th->source),
1068 dif, sdif);
1069 if (!sk1)
1070 goto out;
1071
1072 /* sdif set, means packet ingressed via a device
1073 * in an L3 domain and dif is set to it.
1074 */
1075 l3index = tcp_v6_sdif(skb) ? dif : 0;
1076
1077 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1078 if (!key.md5_key)
1079 goto out;
1080 key.type = TCP_KEY_MD5;
1081
1082 tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
1083 if (crypto_memneq(md5_hash_location, newhash, 16))
1084 goto out;
1085 }
1086 #endif
1087
1088 if (th->ack)
1089 seq = ntohl(th->ack_seq);
1090 else
1091 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1092 (th->doff << 2);
1093
1094 #ifdef CONFIG_TCP_AO
1095 if (aoh) {
1096 int l3index;
1097
1098 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1099 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1100 &key.ao_key, &key.traffic_key,
1101 &allocated_traffic_key,
1102 &key.rcv_next, &key.sne))
1103 goto out;
1104 key.type = TCP_KEY_AO;
1105 }
1106 #endif
1107
1108 if (sk) {
1109 oif = sk->sk_bound_dev_if;
1110 if (sk_fullsock(sk)) {
1111 if (inet6_test_bit(REPFLOW, sk))
1112 label = ip6_flowlabel(ipv6h);
1113 priority = READ_ONCE(sk->sk_priority);
1114 txhash = sk->sk_txhash;
1115 }
1116 if (sk->sk_state == TCP_TIME_WAIT) {
1117 label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1118 priority = inet_twsk(sk)->tw_priority;
1119 txhash = inet_twsk(sk)->tw_txhash;
1120 }
1121 } else {
1122 if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) &
1123 FLOWLABEL_REFLECT_TCP_RESET)
1124 label = ip6_flowlabel(ipv6h);
1125 }
1126
1127 trace_tcp_send_reset(sk, skb, reason);
1128
1129 tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
1130 ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK,
1131 label, priority, txhash,
1132 &key);
1133
1134 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1135 out:
1136 if (allocated_traffic_key)
1137 kfree(key.traffic_key);
1138 rcu_read_unlock();
1139 #endif
1140 }
1141
tcp_v6_send_ack(const struct sock * sk,struct sk_buff * skb,u32 seq,u32 ack,u32 win,u32 tsval,u32 tsecr,int oif,struct tcp_key * key,u8 tclass,__be32 label,u32 priority,u32 txhash)1142 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1143 u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1144 struct tcp_key *key, u8 tclass,
1145 __be32 label, u32 priority, u32 txhash)
1146 {
1147 tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
1148 tclass, label, priority, txhash, key);
1149 }
1150
tcp_v6_timewait_ack(struct sock * sk,struct sk_buff * skb,enum tcp_tw_status tw_status)1151 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb,
1152 enum tcp_tw_status tw_status)
1153 {
1154 struct inet_timewait_sock *tw = inet_twsk(sk);
1155 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1156 u8 tclass = tw->tw_tclass;
1157 struct tcp_key key = {};
1158
1159 if (tw_status == TCP_TW_ACK_OOW)
1160 tclass &= ~INET_ECN_MASK;
1161 #ifdef CONFIG_TCP_AO
1162 struct tcp_ao_info *ao_info;
1163
1164 if (static_branch_unlikely(&tcp_ao_needed.key)) {
1165
1166 /* FIXME: the segment to-be-acked is not verified yet */
1167 ao_info = rcu_dereference(tcptw->ao_info);
1168 if (ao_info) {
1169 const struct tcp_ao_hdr *aoh;
1170
1171 /* Invalid TCP option size or twice included auth */
1172 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1173 goto out;
1174 if (aoh)
1175 key.ao_key = tcp_ao_established_key(sk, ao_info,
1176 aoh->rnext_keyid, -1);
1177 }
1178 }
1179 if (key.ao_key) {
1180 struct tcp_ao_key *rnext_key;
1181
1182 key.traffic_key = snd_other_key(key.ao_key);
1183 /* rcv_next switches to our rcv_next */
1184 rnext_key = READ_ONCE(ao_info->rnext_key);
1185 key.rcv_next = rnext_key->rcvid;
1186 key.sne = READ_ONCE(ao_info->snd_sne);
1187 key.type = TCP_KEY_AO;
1188 #else
1189 if (0) {
1190 #endif
1191 #ifdef CONFIG_TCP_MD5SIG
1192 } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1193 key.md5_key = tcp_twsk_md5_key(tcptw);
1194 if (key.md5_key)
1195 key.type = TCP_KEY_MD5;
1196 #endif
1197 }
1198
1199 tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt,
1200 READ_ONCE(tcptw->tw_rcv_nxt),
1201 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1202 tcp_tw_tsval(tcptw),
1203 READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if,
1204 &key, tclass, cpu_to_be32(tw->tw_flowlabel),
1205 tw->tw_priority, tw->tw_txhash);
1206
1207 #ifdef CONFIG_TCP_AO
1208 out:
1209 #endif
1210 inet_twsk_put(tw);
1211 }
1212
1213 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1214 struct request_sock *req)
1215 {
1216 struct tcp_key key = {};
1217
1218 #ifdef CONFIG_TCP_AO
1219 if (static_branch_unlikely(&tcp_ao_needed.key) &&
1220 tcp_rsk_used_ao(req)) {
1221 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1222 const struct tcp_ao_hdr *aoh;
1223 int l3index;
1224
1225 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1226 /* Invalid TCP option size or twice included auth */
1227 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1228 return;
1229 if (!aoh)
1230 return;
1231 key.ao_key = tcp_ao_do_lookup(sk, l3index,
1232 (union tcp_ao_addr *)addr,
1233 AF_INET6, aoh->rnext_keyid, -1);
1234 if (unlikely(!key.ao_key)) {
1235 /* Send ACK with any matching MKT for the peer */
1236 key.ao_key = tcp_ao_do_lookup(sk, l3index,
1237 (union tcp_ao_addr *)addr,
1238 AF_INET6, -1, -1);
1239 /* Matching key disappeared (user removed the key?)
1240 * let the handshake timeout.
1241 */
1242 if (!key.ao_key) {
1243 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1244 addr,
1245 ntohs(tcp_hdr(skb)->source),
1246 &ipv6_hdr(skb)->daddr,
1247 ntohs(tcp_hdr(skb)->dest));
1248 return;
1249 }
1250 }
1251 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1252 if (!key.traffic_key)
1253 return;
1254
1255 key.type = TCP_KEY_AO;
1256 key.rcv_next = aoh->keyid;
1257 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1258 #else
1259 if (0) {
1260 #endif
1261 #ifdef CONFIG_TCP_MD5SIG
1262 } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1263 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1264
1265 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
1266 l3index);
1267 if (key.md5_key)
1268 key.type = TCP_KEY_MD5;
1269 #endif
1270 }
1271
1272 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1273 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1274 */
1275 tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1276 tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1277 tcp_rsk(req)->rcv_nxt,
1278 tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
1279 tcp_rsk_tsval(tcp_rsk(req)),
1280 req->ts_recent, sk->sk_bound_dev_if,
1281 &key, ipv6_get_dsfield(ipv6_hdr(skb)) & ~INET_ECN_MASK,
1282 0,
1283 READ_ONCE(sk->sk_priority),
1284 READ_ONCE(tcp_rsk(req)->txhash));
1285 if (tcp_key_is_ao(&key))
1286 kfree(key.traffic_key);
1287 }
1288
1289
1290 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1291 {
1292 #ifdef CONFIG_SYN_COOKIES
1293 const struct tcphdr *th = tcp_hdr(skb);
1294
1295 if (!th->syn)
1296 sk = cookie_v6_check(sk, skb);
1297 #endif
1298 return sk;
1299 }
1300
1301 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1302 struct tcphdr *th, u32 *cookie)
1303 {
1304 u16 mss = 0;
1305 #ifdef CONFIG_SYN_COOKIES
1306 mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1307 &tcp_request_sock_ipv6_ops, sk, th);
1308 if (mss) {
1309 *cookie = __cookie_v6_init_sequence(iph, th, &mss);
1310 tcp_synq_overflow(sk);
1311 }
1312 #endif
1313 return mss;
1314 }
1315
1316 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1317 {
1318 if (skb->protocol == htons(ETH_P_IP))
1319 return tcp_v4_conn_request(sk, skb);
1320
1321 if (!ipv6_unicast_destination(skb))
1322 goto drop;
1323
1324 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1325 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1326 return 0;
1327 }
1328
1329 return tcp_conn_request(&tcp6_request_sock_ops,
1330 &tcp_request_sock_ipv6_ops, sk, skb);
1331
1332 drop:
1333 tcp_listendrop(sk);
1334 return 0; /* don't send reset */
1335 }
1336
1337 static void tcp_v6_restore_cb(struct sk_buff *skb)
1338 {
1339 /* We need to move header back to the beginning if xfrm6_policy_check()
1340 * and tcp_v6_fill_cb() are going to be called again.
1341 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1342 */
1343 memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1344 sizeof(struct inet6_skb_parm));
1345 }
1346
1347 /* Called from tcp_v4_syn_recv_sock() for v6_mapped children. */
1348 static void tcp_v6_mapped_child_init(struct sock *newsk, const struct sock *sk)
1349 {
1350 struct inet_sock *newinet = inet_sk(newsk);
1351 struct ipv6_pinfo *newnp;
1352
1353 newinet->pinet6 = newnp = tcp_inet6_sk(newsk);
1354 newinet->ipv6_fl_list = NULL;
1355
1356 memcpy(newnp, tcp_inet6_sk(sk), sizeof(struct ipv6_pinfo));
1357
1358 newnp->saddr = newsk->sk_v6_rcv_saddr;
1359
1360 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1361 if (sk_is_mptcp(newsk))
1362 mptcpv6_handle_mapped(newsk, true);
1363 newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1364 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1365 tcp_sk(newsk)->af_specific = &tcp_sock_ipv6_mapped_specific;
1366 #endif
1367
1368 newnp->ipv6_mc_list = NULL;
1369 newnp->ipv6_ac_list = NULL;
1370 newnp->pktoptions = NULL;
1371 newnp->opt = NULL;
1372
1373 /* tcp_v4_syn_recv_sock() has initialized newinet->mc_{index,ttl} */
1374 newnp->mcast_oif = newinet->mc_index;
1375 newnp->mcast_hops = newinet->mc_ttl;
1376
1377 newnp->rcv_flowinfo = 0;
1378 if (inet6_test_bit(REPFLOW, sk))
1379 newnp->flow_label = 0;
1380 }
1381
1382 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1383 struct request_sock *req,
1384 struct dst_entry *dst,
1385 struct request_sock *req_unhash,
1386 bool *own_req,
1387 void (*opt_child_init)(struct sock *newsk,
1388 const struct sock *sk))
1389 {
1390 const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1391 struct inet_request_sock *ireq;
1392 struct ipv6_txoptions *opt;
1393 struct inet_sock *newinet;
1394 bool found_dup_sk = false;
1395 struct ipv6_pinfo *newnp;
1396 struct tcp_sock *newtp;
1397 struct sock *newsk;
1398 #ifdef CONFIG_TCP_MD5SIG
1399 struct tcp_md5sig_key *key;
1400 int l3index;
1401 #endif
1402 struct flowi6 fl6;
1403
1404 if (skb->protocol == htons(ETH_P_IP))
1405 return tcp_v4_syn_recv_sock(sk, skb, req, dst,
1406 req_unhash, own_req,
1407 tcp_v6_mapped_child_init);
1408 ireq = inet_rsk(req);
1409
1410 if (sk_acceptq_is_full(sk))
1411 goto exit_overflow;
1412
1413 dst = inet6_csk_route_req(sk, dst, &fl6, req, IPPROTO_TCP);
1414 if (!dst)
1415 goto exit;
1416
1417 newsk = tcp_create_openreq_child(sk, req, skb);
1418 if (!newsk)
1419 goto exit_nonewsk;
1420
1421 /*
1422 * No need to charge this sock to the relevant IPv6 refcnt debug socks
1423 * count here, tcp_create_openreq_child now does this for us, see the
1424 * comment in that function for the gory details. -acme
1425 */
1426
1427 newsk->sk_gso_type = SKB_GSO_TCPV6;
1428 inet6_sk_rx_dst_set(newsk, skb);
1429
1430 newinet = inet_sk(newsk);
1431 newinet->cork.fl.u.ip6 = fl6;
1432 newinet->pinet6 = tcp_inet6_sk(newsk);
1433 newinet->ipv6_fl_list = NULL;
1434 newinet->inet_opt = NULL;
1435
1436 newtp = tcp_sk(newsk);
1437 newnp = tcp_inet6_sk(newsk);
1438
1439 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1440
1441 ip6_dst_store(newsk, dst, false, false);
1442
1443 newnp->saddr = ireq->ir_v6_loc_addr;
1444
1445 /* Now IPv6 options...
1446
1447 First: no IPv4 options.
1448 */
1449 newnp->ipv6_mc_list = NULL;
1450 newnp->ipv6_ac_list = NULL;
1451
1452 /* Clone RX bits */
1453 newnp->rxopt.all = np->rxopt.all;
1454
1455 newnp->pktoptions = NULL;
1456 newnp->opt = NULL;
1457 newnp->mcast_oif = tcp_v6_iif(skb);
1458 newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1459 newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1460 if (inet6_test_bit(REPFLOW, sk))
1461 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1462
1463 /* Set ToS of the new socket based upon the value of incoming SYN.
1464 * ECT bits are set later in tcp_init_transfer().
1465 */
1466 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1467 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1468
1469 /* Clone native IPv6 options from listening socket (if any)
1470
1471 Yes, keeping reference count would be much more clever,
1472 but we make one more one thing there: reattach optmem
1473 to newsk.
1474 */
1475 opt = ireq->ipv6_opt;
1476 if (!opt)
1477 opt = rcu_dereference(np->opt);
1478 if (opt) {
1479 opt = ipv6_dup_options(newsk, opt);
1480 RCU_INIT_POINTER(newnp->opt, opt);
1481 }
1482 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1483 if (opt)
1484 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1485 opt->opt_flen;
1486
1487 tcp_ca_openreq_child(newsk, dst);
1488
1489 tcp_sync_mss(newsk, dst6_mtu(dst));
1490 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst));
1491
1492 tcp_initialize_rcv_mss(newsk);
1493
1494 #ifdef CONFIG_TCP_MD5SIG
1495 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1496
1497 if (!tcp_rsk_used_ao(req)) {
1498 /* Copy over the MD5 key from the original socket */
1499 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1500 if (key) {
1501 const union tcp_md5_addr *addr;
1502
1503 addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1504 if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key))
1505 goto put_and_exit;
1506 }
1507 }
1508 #endif
1509 #ifdef CONFIG_TCP_AO
1510 /* Copy over tcp_ao_info if any */
1511 if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1512 goto put_and_exit; /* OOM */
1513 #endif
1514
1515 if (__inet_inherit_port(sk, newsk) < 0)
1516 goto put_and_exit;
1517 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1518 &found_dup_sk);
1519 if (*own_req) {
1520 tcp_move_syn(newtp, req);
1521
1522 /* Clone pktoptions received with SYN, if we own the req */
1523 if (ireq->pktopts) {
1524 newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
1525 consume_skb(ireq->pktopts);
1526 ireq->pktopts = NULL;
1527 if (newnp->pktoptions)
1528 tcp_v6_restore_cb(newnp->pktoptions);
1529 }
1530 } else {
1531 if (!req_unhash && found_dup_sk) {
1532 /* This code path should only be executed in the
1533 * syncookie case only
1534 */
1535 bh_unlock_sock(newsk);
1536 sock_put(newsk);
1537 newsk = NULL;
1538 }
1539 }
1540
1541 return newsk;
1542
1543 exit_overflow:
1544 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1545 exit_nonewsk:
1546 dst_release(dst);
1547 exit:
1548 tcp_listendrop(sk);
1549 return NULL;
1550 put_and_exit:
1551 inet_csk_prepare_forced_close(newsk);
1552 tcp_done(newsk);
1553 goto exit;
1554 }
1555
1556 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1557 u32));
1558 /* The socket must have it's spinlock held when we get
1559 * here, unless it is a TCP_LISTEN socket.
1560 *
1561 * We have a potential double-lock case here, so even when
1562 * doing backlog processing we use the BH locking scheme.
1563 * This is because we cannot sleep with the original spinlock
1564 * held.
1565 */
1566 INDIRECT_CALLABLE_SCOPE
1567 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1568 {
1569 struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1570 struct sk_buff *opt_skb = NULL;
1571 enum skb_drop_reason reason;
1572 struct tcp_sock *tp;
1573
1574 /* Imagine: socket is IPv6. IPv4 packet arrives,
1575 goes to IPv4 receive handler and backlogged.
1576 From backlog it always goes here. Kerboom...
1577 Fortunately, tcp_rcv_established and rcv_established
1578 handle them correctly, but it is not case with
1579 tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK
1580 */
1581
1582 if (skb->protocol == htons(ETH_P_IP))
1583 return tcp_v4_do_rcv(sk, skb);
1584
1585 reason = psp_sk_rx_policy_check(sk, skb);
1586 if (reason)
1587 goto err_discard;
1588
1589 /*
1590 * socket locking is here for SMP purposes as backlog rcv
1591 * is currently called with bh processing disabled.
1592 */
1593
1594 /* Do Stevens' IPV6_PKTOPTIONS.
1595
1596 Yes, guys, it is the only place in our code, where we
1597 may make it not affecting IPv4.
1598 The rest of code is protocol independent,
1599 and I do not like idea to uglify IPv4.
1600
1601 Actually, all the idea behind IPV6_PKTOPTIONS
1602 looks not very well thought. For now we latch
1603 options, received in the last packet, enqueued
1604 by tcp. Feel free to propose better solution.
1605 --ANK (980728)
1606 */
1607 if (np->rxopt.all &&
1608 !((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1609 opt_skb = skb_clone_and_charge_r(skb, sk);
1610
1611 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1612 struct dst_entry *dst;
1613
1614 dst = rcu_dereference_protected(sk->sk_rx_dst,
1615 lockdep_sock_is_held(sk));
1616
1617 sock_rps_save_rxhash(sk, skb);
1618 sk_mark_napi_id(sk, skb);
1619 if (dst && unlikely(dst != skb_dst(skb))) {
1620 if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1621 INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1622 dst, sk->sk_rx_dst_cookie) == NULL) {
1623 RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1624 dst_release(dst);
1625 }
1626 }
1627
1628 tcp_rcv_established(sk, skb);
1629 if (opt_skb)
1630 goto ipv6_pktoptions;
1631 return 0;
1632 }
1633
1634 if (tcp_checksum_complete(skb))
1635 goto csum_err;
1636
1637 if (sk->sk_state == TCP_LISTEN) {
1638 struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1639
1640 if (!nsk)
1641 return 0;
1642 if (nsk != sk) {
1643 reason = tcp_child_process(sk, nsk, skb);
1644 sock_put(nsk);
1645 if (reason)
1646 goto reset;
1647 return 0;
1648 }
1649 } else
1650 sock_rps_save_rxhash(sk, skb);
1651
1652 reason = tcp_rcv_state_process(sk, skb);
1653 if (reason)
1654 goto reset;
1655 if (opt_skb)
1656 goto ipv6_pktoptions;
1657 return 0;
1658
1659 reset:
1660 tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
1661 discard:
1662 if (opt_skb)
1663 __kfree_skb(opt_skb);
1664 sk_skb_reason_drop(sk, skb, reason);
1665 return 0;
1666 csum_err:
1667 reason = SKB_DROP_REASON_TCP_CSUM;
1668 trace_tcp_bad_csum(skb);
1669 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1670 err_discard:
1671 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1672 goto discard;
1673
1674
1675 ipv6_pktoptions:
1676 /* Do you ask, what is it?
1677
1678 1. skb was enqueued by tcp.
1679 2. skb is added to tail of read queue, rather than out of order.
1680 3. socket is not in passive state.
1681 4. Finally, it really contains options, which user wants to receive.
1682 */
1683 tp = tcp_sk(sk);
1684 if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1685 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1686 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1687 WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1688 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1689 WRITE_ONCE(np->mcast_hops,
1690 ipv6_hdr(opt_skb)->hop_limit);
1691 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1692 np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1693 if (inet6_test_bit(REPFLOW, sk))
1694 np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1695 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1696 tcp_v6_restore_cb(opt_skb);
1697 opt_skb = xchg(&np->pktoptions, opt_skb);
1698 } else {
1699 __kfree_skb(opt_skb);
1700 opt_skb = xchg(&np->pktoptions, NULL);
1701 }
1702 }
1703
1704 consume_skb(opt_skb);
1705 return 0;
1706 }
1707
1708 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1709 const struct tcphdr *th)
1710 {
1711 /* This is tricky: we move IP6CB at its correct location into
1712 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1713 * _decode_session6() uses IP6CB().
1714 * barrier() makes sure compiler won't play aliasing games.
1715 */
1716 memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1717 sizeof(struct inet6_skb_parm));
1718 barrier();
1719
1720 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1721 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1722 skb->len - th->doff*4);
1723 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1724 TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
1725 TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1726 TCP_SKB_CB(skb)->sacked = 0;
1727 TCP_SKB_CB(skb)->has_rxtstamp =
1728 skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1729 }
1730
1731 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1732 {
1733 struct net *net = dev_net_rcu(skb->dev);
1734 enum skb_drop_reason drop_reason;
1735 enum tcp_tw_status tw_status;
1736 int sdif = inet6_sdif(skb);
1737 int dif = inet6_iif(skb);
1738 const struct tcphdr *th;
1739 const struct ipv6hdr *hdr;
1740 struct sock *sk = NULL;
1741 bool refcounted;
1742 int ret;
1743 u32 isn;
1744
1745 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1746 if (skb->pkt_type != PACKET_HOST)
1747 goto discard_it;
1748
1749 /*
1750 * Count it even if it's bad.
1751 */
1752 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1753
1754 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1755 goto discard_it;
1756
1757 th = (const struct tcphdr *)skb->data;
1758
1759 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1760 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1761 goto bad_packet;
1762 }
1763 if (!pskb_may_pull(skb, th->doff*4))
1764 goto discard_it;
1765
1766 if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1767 goto csum_error;
1768
1769 th = (const struct tcphdr *)skb->data;
1770 hdr = ipv6_hdr(skb);
1771
1772 lookup:
1773 sk = __inet6_lookup_skb(skb, __tcp_hdrlen(th),
1774 th->source, th->dest, inet6_iif(skb), sdif,
1775 &refcounted);
1776 if (!sk)
1777 goto no_tcp_socket;
1778
1779 if (sk->sk_state == TCP_TIME_WAIT)
1780 goto do_time_wait;
1781
1782 if (sk->sk_state == TCP_NEW_SYN_RECV) {
1783 struct request_sock *req = inet_reqsk(sk);
1784 bool req_stolen = false;
1785 struct sock *nsk;
1786
1787 sk = req->rsk_listener;
1788 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1789 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1790 else
1791 drop_reason = tcp_inbound_hash(sk, req, skb,
1792 &hdr->saddr, &hdr->daddr,
1793 AF_INET6, dif, sdif);
1794 if (drop_reason) {
1795 sk_drops_skbadd(sk, skb);
1796 reqsk_put(req);
1797 goto discard_it;
1798 }
1799 if (tcp_checksum_complete(skb)) {
1800 reqsk_put(req);
1801 goto csum_error;
1802 }
1803 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1804 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1805 if (!nsk) {
1806 inet_csk_reqsk_queue_drop_and_put(sk, req);
1807 goto lookup;
1808 }
1809 sk = nsk;
1810 /* reuseport_migrate_sock() has already held one sk_refcnt
1811 * before returning.
1812 */
1813 } else {
1814 sock_hold(sk);
1815 }
1816 refcounted = true;
1817 nsk = NULL;
1818 drop_reason = tcp_filter(sk, skb);
1819 if (!drop_reason) {
1820 th = (const struct tcphdr *)skb->data;
1821 hdr = ipv6_hdr(skb);
1822 tcp_v6_fill_cb(skb, hdr, th);
1823 nsk = tcp_check_req(sk, skb, req, false, &req_stolen,
1824 &drop_reason);
1825 }
1826 if (!nsk) {
1827 reqsk_put(req);
1828 if (req_stolen) {
1829 /* Another cpu got exclusive access to req
1830 * and created a full blown socket.
1831 * Try to feed this packet to this socket
1832 * instead of discarding it.
1833 */
1834 tcp_v6_restore_cb(skb);
1835 sock_put(sk);
1836 goto lookup;
1837 }
1838 goto discard_and_relse;
1839 }
1840 nf_reset_ct(skb);
1841 if (nsk == sk) {
1842 reqsk_put(req);
1843 tcp_v6_restore_cb(skb);
1844 } else {
1845 drop_reason = tcp_child_process(sk, nsk, skb);
1846 if (drop_reason) {
1847 enum sk_rst_reason rst_reason;
1848
1849 rst_reason = sk_rst_convert_drop_reason(drop_reason);
1850 tcp_v6_send_reset(nsk, skb, rst_reason);
1851 sock_put(nsk);
1852 goto discard_and_relse;
1853 }
1854 sock_put(nsk);
1855 sock_put(sk);
1856 return 0;
1857 }
1858 }
1859
1860 isn = 0;
1861 process:
1862 if (static_branch_unlikely(&ip6_min_hopcount)) {
1863 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1864 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1865 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1866 drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1867 goto discard_and_relse;
1868 }
1869 }
1870
1871 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
1872 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1873 goto discard_and_relse;
1874 }
1875
1876 drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
1877 AF_INET6, dif, sdif);
1878 if (drop_reason)
1879 goto discard_and_relse;
1880
1881 nf_reset_ct(skb);
1882
1883 drop_reason = tcp_filter(sk, skb);
1884 if (drop_reason)
1885 goto discard_and_relse;
1886
1887 th = (const struct tcphdr *)skb->data;
1888 hdr = ipv6_hdr(skb);
1889 tcp_v6_fill_cb(skb, hdr, th);
1890 TCP_SKB_CB(skb)->tcp_tw_isn = isn;
1891
1892 skb->dev = NULL;
1893
1894 if (sk->sk_state == TCP_LISTEN) {
1895 ret = tcp_v6_do_rcv(sk, skb);
1896 goto put_and_return;
1897 }
1898
1899 sk_incoming_cpu_update(sk);
1900
1901 bh_lock_sock_nested(sk);
1902 tcp_segs_in(tcp_sk(sk), skb);
1903 ret = 0;
1904 if (!sock_owned_by_user(sk)) {
1905 ret = tcp_v6_do_rcv(sk, skb);
1906 } else {
1907 drop_reason = tcp_add_backlog(sk, skb);
1908 if (drop_reason)
1909 goto discard_and_relse;
1910 }
1911 bh_unlock_sock(sk);
1912 put_and_return:
1913 if (refcounted)
1914 sock_put(sk);
1915 return ret ? -1 : 0;
1916
1917 no_tcp_socket:
1918 drop_reason = SKB_DROP_REASON_NO_SOCKET;
1919 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1920 goto discard_it;
1921
1922 tcp_v6_fill_cb(skb, hdr, th);
1923
1924 if (tcp_checksum_complete(skb)) {
1925 csum_error:
1926 drop_reason = SKB_DROP_REASON_TCP_CSUM;
1927 trace_tcp_bad_csum(skb);
1928 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1929 bad_packet:
1930 __TCP_INC_STATS(net, TCP_MIB_INERRS);
1931 } else {
1932 tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
1933 }
1934
1935 discard_it:
1936 SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1937 sk_skb_reason_drop(sk, skb, drop_reason);
1938 return 0;
1939
1940 discard_and_relse:
1941 sk_drops_skbadd(sk, skb);
1942 if (refcounted)
1943 sock_put(sk);
1944 goto discard_it;
1945
1946 do_time_wait:
1947 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1948 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1949 inet_twsk_put(inet_twsk(sk));
1950 goto discard_it;
1951 }
1952
1953 tcp_v6_fill_cb(skb, hdr, th);
1954
1955 if (tcp_checksum_complete(skb)) {
1956 inet_twsk_put(inet_twsk(sk));
1957 goto csum_error;
1958 }
1959
1960 tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn,
1961 &drop_reason);
1962 switch (tw_status) {
1963 case TCP_TW_SYN:
1964 {
1965 struct sock *sk2;
1966
1967 sk2 = inet6_lookup_listener(net, skb, __tcp_hdrlen(th),
1968 &ipv6_hdr(skb)->saddr, th->source,
1969 &ipv6_hdr(skb)->daddr,
1970 ntohs(th->dest),
1971 tcp_v6_iif_l3_slave(skb),
1972 sdif);
1973 if (sk2) {
1974 struct inet_timewait_sock *tw = inet_twsk(sk);
1975 inet_twsk_deschedule_put(tw);
1976 sk = sk2;
1977 tcp_v6_restore_cb(skb);
1978 refcounted = false;
1979 goto process;
1980 }
1981
1982 drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
1983 if (drop_reason) {
1984 inet_twsk_put(inet_twsk(sk));
1985 goto discard_it;
1986 }
1987 }
1988 /* to ACK */
1989 fallthrough;
1990 case TCP_TW_ACK:
1991 case TCP_TW_ACK_OOW:
1992 tcp_v6_timewait_ack(sk, skb, tw_status);
1993 break;
1994 case TCP_TW_RST:
1995 tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
1996 inet_twsk_deschedule_put(inet_twsk(sk));
1997 goto discard_it;
1998 case TCP_TW_SUCCESS:
1999 ;
2000 }
2001 goto discard_it;
2002 }
2003
2004 static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2005 .twsk_obj_size = sizeof(struct tcp6_timewait_sock),
2006 };
2007
2008 const struct inet_connection_sock_af_ops ipv6_specific = {
2009 .queue_xmit = inet6_csk_xmit,
2010 .rebuild_header = inet6_sk_rebuild_header,
2011 .sk_rx_dst_set = inet6_sk_rx_dst_set,
2012 .conn_request = tcp_v6_conn_request,
2013 .syn_recv_sock = tcp_v6_syn_recv_sock,
2014 .net_header_len = sizeof(struct ipv6hdr),
2015 .setsockopt = ipv6_setsockopt,
2016 .getsockopt = ipv6_getsockopt,
2017 .mtu_reduced = tcp_v6_mtu_reduced,
2018 };
2019
2020 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2021 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2022 #ifdef CONFIG_TCP_MD5SIG
2023 .md5_lookup = tcp_v6_md5_lookup,
2024 .calc_md5_hash = tcp_v6_md5_hash_skb,
2025 .md5_parse = tcp_v6_parse_md5_keys,
2026 #endif
2027 #ifdef CONFIG_TCP_AO
2028 .ao_lookup = tcp_v6_ao_lookup,
2029 .calc_ao_hash = tcp_v6_ao_hash_skb,
2030 .ao_parse = tcp_v6_parse_ao,
2031 .ao_calc_key_sk = tcp_v6_ao_calc_key_sk,
2032 #endif
2033 };
2034 #endif
2035
2036 /*
2037 * TCP over IPv4 via INET6 API
2038 */
2039 static const struct inet_connection_sock_af_ops ipv6_mapped = {
2040 .queue_xmit = ip_queue_xmit,
2041 .rebuild_header = inet_sk_rebuild_header,
2042 .sk_rx_dst_set = inet_sk_rx_dst_set,
2043 .conn_request = tcp_v6_conn_request,
2044 .syn_recv_sock = tcp_v6_syn_recv_sock,
2045 .net_header_len = sizeof(struct iphdr),
2046 .setsockopt = ipv6_setsockopt,
2047 .getsockopt = ipv6_getsockopt,
2048 .mtu_reduced = tcp_v4_mtu_reduced,
2049 };
2050
2051 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2052 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2053 #ifdef CONFIG_TCP_MD5SIG
2054 .md5_lookup = tcp_v4_md5_lookup,
2055 .calc_md5_hash = tcp_v4_md5_hash_skb,
2056 .md5_parse = tcp_v6_parse_md5_keys,
2057 #endif
2058 #ifdef CONFIG_TCP_AO
2059 .ao_lookup = tcp_v6_ao_lookup,
2060 .calc_ao_hash = tcp_v4_ao_hash_skb,
2061 .ao_parse = tcp_v6_parse_ao,
2062 .ao_calc_key_sk = tcp_v4_ao_calc_key_sk,
2063 #endif
2064 };
2065
2066 static void tcp6_destruct_sock(struct sock *sk)
2067 {
2068 tcp_md5_destruct_sock(sk);
2069 tcp_ao_destroy_sock(sk, false);
2070 inet6_sock_destruct(sk);
2071 }
2072 #endif
2073
2074 /* NOTE: A lot of things set to zero explicitly by call to
2075 * sk_alloc() so need not be done here.
2076 */
2077 static int tcp_v6_init_sock(struct sock *sk)
2078 {
2079 struct inet_connection_sock *icsk = inet_csk(sk);
2080
2081 tcp_init_sock(sk);
2082
2083 icsk->icsk_af_ops = &ipv6_specific;
2084
2085 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2086 tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2087 sk->sk_destruct = tcp6_destruct_sock;
2088 #endif
2089
2090 return 0;
2091 }
2092
2093 #ifdef CONFIG_PROC_FS
2094 /* Proc filesystem TCPv6 sock list dumping. */
2095 static void get_openreq6(struct seq_file *seq,
2096 const struct request_sock *req, int i)
2097 {
2098 long ttd = req->rsk_timer.expires - jiffies;
2099 const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
2100 const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
2101
2102 if (ttd < 0)
2103 ttd = 0;
2104
2105 seq_printf(seq,
2106 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2107 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2108 i,
2109 src->s6_addr32[0], src->s6_addr32[1],
2110 src->s6_addr32[2], src->s6_addr32[3],
2111 inet_rsk(req)->ir_num,
2112 dest->s6_addr32[0], dest->s6_addr32[1],
2113 dest->s6_addr32[2], dest->s6_addr32[3],
2114 ntohs(inet_rsk(req)->ir_rmt_port),
2115 TCP_SYN_RECV,
2116 0, 0, /* could print option size, but that is af dependent. */
2117 1, /* timers active (only the expire timer) */
2118 jiffies_to_clock_t(ttd),
2119 req->num_timeout,
2120 from_kuid_munged(seq_user_ns(seq),
2121 sk_uid(req->rsk_listener)),
2122 0, /* non standard timer */
2123 0, /* open_requests have no inode */
2124 0, req);
2125 }
2126
2127 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2128 {
2129 const struct in6_addr *dest, *src;
2130 __u16 destp, srcp;
2131 int timer_active;
2132 unsigned long timer_expires;
2133 const struct inet_sock *inet = inet_sk(sp);
2134 const struct tcp_sock *tp = tcp_sk(sp);
2135 const struct inet_connection_sock *icsk = inet_csk(sp);
2136 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2137 u8 icsk_pending;
2138 int rx_queue;
2139 int state;
2140
2141 dest = &sp->sk_v6_daddr;
2142 src = &sp->sk_v6_rcv_saddr;
2143 destp = ntohs(inet->inet_dport);
2144 srcp = ntohs(inet->inet_sport);
2145
2146 icsk_pending = smp_load_acquire(&icsk->icsk_pending);
2147 if (icsk_pending == ICSK_TIME_RETRANS ||
2148 icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2149 icsk_pending == ICSK_TIME_LOSS_PROBE) {
2150 timer_active = 1;
2151 timer_expires = tcp_timeout_expires(sp);
2152 } else if (icsk_pending == ICSK_TIME_PROBE0) {
2153 timer_active = 4;
2154 timer_expires = tcp_timeout_expires(sp);
2155 } else if (timer_pending(&icsk->icsk_keepalive_timer)) {
2156 timer_active = 2;
2157 timer_expires = icsk->icsk_keepalive_timer.expires;
2158 } else {
2159 timer_active = 0;
2160 timer_expires = jiffies;
2161 }
2162
2163 state = inet_sk_state_load(sp);
2164 if (state == TCP_LISTEN)
2165 rx_queue = READ_ONCE(sp->sk_ack_backlog);
2166 else
2167 /* Because we don't lock the socket,
2168 * we might find a transient negative value.
2169 */
2170 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2171 READ_ONCE(tp->copied_seq), 0);
2172
2173 seq_printf(seq,
2174 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2175 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d %pK %lu %lu %u %u %d\n",
2176 i,
2177 src->s6_addr32[0], src->s6_addr32[1],
2178 src->s6_addr32[2], src->s6_addr32[3], srcp,
2179 dest->s6_addr32[0], dest->s6_addr32[1],
2180 dest->s6_addr32[2], dest->s6_addr32[3], destp,
2181 state,
2182 READ_ONCE(tp->write_seq) - tp->snd_una,
2183 rx_queue,
2184 timer_active,
2185 jiffies_delta_to_clock_t(timer_expires - jiffies),
2186 READ_ONCE(icsk->icsk_retransmits),
2187 from_kuid_munged(seq_user_ns(seq), sk_uid(sp)),
2188 READ_ONCE(icsk->icsk_probes_out),
2189 sock_i_ino(sp),
2190 refcount_read(&sp->sk_refcnt), sp,
2191 jiffies_to_clock_t(icsk->icsk_rto),
2192 jiffies_to_clock_t(icsk->icsk_ack.ato),
2193 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2194 tcp_snd_cwnd(tp),
2195 state == TCP_LISTEN ?
2196 fastopenq->max_qlen :
2197 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2198 );
2199 }
2200
2201 static void get_timewait6_sock(struct seq_file *seq,
2202 struct inet_timewait_sock *tw, int i)
2203 {
2204 long delta = tw->tw_timer.expires - jiffies;
2205 const struct in6_addr *dest, *src;
2206 __u16 destp, srcp;
2207
2208 dest = &tw->tw_v6_daddr;
2209 src = &tw->tw_v6_rcv_saddr;
2210 destp = ntohs(tw->tw_dport);
2211 srcp = ntohs(tw->tw_sport);
2212
2213 seq_printf(seq,
2214 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2215 "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2216 i,
2217 src->s6_addr32[0], src->s6_addr32[1],
2218 src->s6_addr32[2], src->s6_addr32[3], srcp,
2219 dest->s6_addr32[0], dest->s6_addr32[1],
2220 dest->s6_addr32[2], dest->s6_addr32[3], destp,
2221 READ_ONCE(tw->tw_substate), 0, 0,
2222 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2223 refcount_read(&tw->tw_refcnt), tw);
2224 }
2225
2226 static int tcp6_seq_show(struct seq_file *seq, void *v)
2227 {
2228 struct tcp_iter_state *st;
2229 struct sock *sk = v;
2230
2231 if (v == SEQ_START_TOKEN) {
2232 seq_puts(seq,
2233 " sl "
2234 "local_address "
2235 "remote_address "
2236 "st tx_queue rx_queue tr tm->when retrnsmt"
2237 " uid timeout inode\n");
2238 goto out;
2239 }
2240 st = seq->private;
2241
2242 if (sk->sk_state == TCP_TIME_WAIT)
2243 get_timewait6_sock(seq, v, st->num);
2244 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2245 get_openreq6(seq, v, st->num);
2246 else
2247 get_tcp6_sock(seq, v, st->num);
2248 out:
2249 return 0;
2250 }
2251
2252 static const struct seq_operations tcp6_seq_ops = {
2253 .show = tcp6_seq_show,
2254 .start = tcp_seq_start,
2255 .next = tcp_seq_next,
2256 .stop = tcp_seq_stop,
2257 };
2258
2259 static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2260 .family = AF_INET6,
2261 };
2262
2263 int __net_init tcp6_proc_init(struct net *net)
2264 {
2265 if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2266 sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2267 return -ENOMEM;
2268 return 0;
2269 }
2270
2271 void tcp6_proc_exit(struct net *net)
2272 {
2273 remove_proc_entry("tcp6", net->proc_net);
2274 }
2275 #endif
2276
2277 struct proto tcpv6_prot = {
2278 .name = "TCPv6",
2279 .owner = THIS_MODULE,
2280 .close = tcp_close,
2281 .pre_connect = tcp_v6_pre_connect,
2282 .connect = tcp_v6_connect,
2283 .disconnect = tcp_disconnect,
2284 .accept = inet_csk_accept,
2285 .ioctl = tcp_ioctl,
2286 .init = tcp_v6_init_sock,
2287 .destroy = tcp_v4_destroy_sock,
2288 .shutdown = tcp_shutdown,
2289 .setsockopt = tcp_setsockopt,
2290 .getsockopt = tcp_getsockopt,
2291 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
2292 .keepalive = tcp_set_keepalive,
2293 .recvmsg = tcp_recvmsg,
2294 .sendmsg = tcp_sendmsg,
2295 .splice_eof = tcp_splice_eof,
2296 .backlog_rcv = tcp_v6_do_rcv,
2297 .release_cb = tcp_release_cb,
2298 .hash = inet_hash,
2299 .unhash = inet_unhash,
2300 .get_port = inet_csk_get_port,
2301 .put_port = inet_put_port,
2302 #ifdef CONFIG_BPF_SYSCALL
2303 .psock_update_sk_prot = tcp_bpf_update_proto,
2304 #endif
2305 .enter_memory_pressure = tcp_enter_memory_pressure,
2306 .leave_memory_pressure = tcp_leave_memory_pressure,
2307 .stream_memory_free = tcp_stream_memory_free,
2308 .sockets_allocated = &tcp_sockets_allocated,
2309
2310 .memory_allocated = &net_aligned_data.tcp_memory_allocated,
2311 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
2312
2313 .memory_pressure = &tcp_memory_pressure,
2314 .sysctl_mem = sysctl_tcp_mem,
2315 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2316 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2317 .max_header = MAX_TCP_HEADER,
2318 .obj_size = sizeof(struct tcp6_sock),
2319 .freeptr_offset = offsetof(struct tcp6_sock,
2320 tcp.inet_conn.icsk_inet.sk.sk_freeptr),
2321 .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2322 .slab_flags = SLAB_TYPESAFE_BY_RCU,
2323 .twsk_prot = &tcp6_timewait_sock_ops,
2324 .rsk_prot = &tcp6_request_sock_ops,
2325 .h.hashinfo = NULL,
2326 .no_autobind = true,
2327 .diag_destroy = tcp_abort,
2328 };
2329 EXPORT_SYMBOL_GPL(tcpv6_prot);
2330
2331
2332 static struct inet_protosw tcpv6_protosw = {
2333 .type = SOCK_STREAM,
2334 .protocol = IPPROTO_TCP,
2335 .prot = &tcpv6_prot,
2336 .ops = &inet6_stream_ops,
2337 .flags = INET_PROTOSW_PERMANENT |
2338 INET_PROTOSW_ICSK,
2339 };
2340
2341 static int __net_init tcpv6_net_init(struct net *net)
2342 {
2343 int res;
2344
2345 res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2346 SOCK_RAW, IPPROTO_TCP, net);
2347 if (!res)
2348 net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC;
2349
2350 return res;
2351 }
2352
2353 static void __net_exit tcpv6_net_exit(struct net *net)
2354 {
2355 inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2356 }
2357
2358 static struct pernet_operations tcpv6_net_ops = {
2359 .init = tcpv6_net_init,
2360 .exit = tcpv6_net_exit,
2361 };
2362
2363 int __init tcpv6_init(void)
2364 {
2365 int ret;
2366
2367 net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2368 .handler = tcp_v6_rcv,
2369 .err_handler = tcp_v6_err,
2370 .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2371 };
2372 ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2373 if (ret)
2374 goto out;
2375
2376 /* register inet6 protocol */
2377 ret = inet6_register_protosw(&tcpv6_protosw);
2378 if (ret)
2379 goto out_tcpv6_protocol;
2380
2381 ret = register_pernet_subsys(&tcpv6_net_ops);
2382 if (ret)
2383 goto out_tcpv6_protosw;
2384
2385 ret = mptcpv6_init();
2386 if (ret)
2387 goto out_tcpv6_pernet_subsys;
2388
2389 out:
2390 return ret;
2391
2392 out_tcpv6_pernet_subsys:
2393 unregister_pernet_subsys(&tcpv6_net_ops);
2394 out_tcpv6_protosw:
2395 inet6_unregister_protosw(&tcpv6_protosw);
2396 out_tcpv6_protocol:
2397 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2398 goto out;
2399 }
2400
2401 void tcpv6_exit(void)
2402 {
2403 unregister_pernet_subsys(&tcpv6_net_ops);
2404 inet6_unregister_protosw(&tcpv6_protosw);
2405 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2406 }
2407