1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TCP over IPv6 4 * Linux INET6 implementation 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 * 9 * Based on: 10 * linux/net/ipv4/tcp.c 11 * linux/net/ipv4/tcp_input.c 12 * linux/net/ipv4/tcp_output.c 13 * 14 * Fixes: 15 * Hideaki YOSHIFUJI : sin6_scope_id support 16 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which 17 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind 18 * a single port at the same time. 19 * YOSHIFUJI Hideaki @USAGI: convert /proc/net/tcp6 to seq_file. 20 */ 21 22 #include <linux/bottom_half.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/types.h> 26 #include <linux/socket.h> 27 #include <linux/sockios.h> 28 #include <linux/net.h> 29 #include <linux/jiffies.h> 30 #include <linux/in.h> 31 #include <linux/in6.h> 32 #include <linux/netdevice.h> 33 #include <linux/init.h> 34 #include <linux/jhash.h> 35 #include <linux/ipsec.h> 36 #include <linux/times.h> 37 #include <linux/slab.h> 38 #include <linux/uaccess.h> 39 #include <linux/ipv6.h> 40 #include <linux/icmpv6.h> 41 #include <linux/random.h> 42 #include <linux/indirect_call_wrapper.h> 43 44 #include <net/aligned_data.h> 45 #include <net/tcp.h> 46 #include <net/ndisc.h> 47 #include <net/inet6_hashtables.h> 48 #include <net/inet6_connection_sock.h> 49 #include <net/ipv6.h> 50 #include <net/transp_v6.h> 51 #include <net/addrconf.h> 52 #include <net/ip6_route.h> 53 #include <net/ip6_checksum.h> 54 #include <net/inet_ecn.h> 55 #include <net/protocol.h> 56 #include <net/xfrm.h> 57 #include <net/snmp.h> 58 #include <net/dsfield.h> 59 #include <net/timewait_sock.h> 60 #include <net/inet_common.h> 61 #include <net/secure_seq.h> 62 #include <net/hotdata.h> 63 #include <net/busy_poll.h> 64 #include <net/rstreason.h> 65 66 #include <linux/proc_fs.h> 67 #include <linux/seq_file.h> 68 69 #include <crypto/hash.h> 70 #include <linux/scatterlist.h> 71 72 #include <trace/events/tcp.h> 73 74 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 75 enum sk_rst_reason reason); 76 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 77 struct request_sock *req); 78 79 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb); 80 81 static const struct inet_connection_sock_af_ops ipv6_mapped; 82 const struct inet_connection_sock_af_ops ipv6_specific; 83 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 84 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific; 85 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific; 86 #endif 87 88 /* Helper returning the inet6 address from a given tcp socket. 89 * It can be used in TCP stack instead of inet6_sk(sk). 90 * This avoids a dereference and allow compiler optimizations. 91 * It is a specialized version of inet6_sk_generic(). 92 */ 93 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \ 94 struct tcp6_sock, tcp)->inet6) 95 96 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 97 { 98 struct dst_entry *dst = skb_dst(skb); 99 100 if (dst && dst_hold_safe(dst)) { 101 rcu_assign_pointer(sk->sk_rx_dst, dst); 102 sk->sk_rx_dst_ifindex = skb->skb_iif; 103 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst)); 104 } 105 } 106 107 static u32 tcp_v6_init_seq(const struct sk_buff *skb) 108 { 109 return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32, 110 ipv6_hdr(skb)->saddr.s6_addr32, 111 tcp_hdr(skb)->dest, 112 tcp_hdr(skb)->source); 113 } 114 115 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb) 116 { 117 return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32, 118 ipv6_hdr(skb)->saddr.s6_addr32); 119 } 120 121 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr, 122 int addr_len) 123 { 124 /* This check is replicated from tcp_v6_connect() and intended to 125 * prevent BPF program called below from accessing bytes that are out 126 * of the bound specified by user in addr_len. 127 */ 128 if (addr_len < SIN6_LEN_RFC2133) 129 return -EINVAL; 130 131 sock_owned_by_me(sk); 132 133 return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len); 134 } 135 136 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, 137 int addr_len) 138 { 139 struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr; 140 struct inet_connection_sock *icsk = inet_csk(sk); 141 struct in6_addr *saddr = NULL, *final_p, final; 142 struct inet_timewait_death_row *tcp_death_row; 143 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 144 struct inet_sock *inet = inet_sk(sk); 145 struct tcp_sock *tp = tcp_sk(sk); 146 struct net *net = sock_net(sk); 147 struct ipv6_txoptions *opt; 148 struct dst_entry *dst; 149 struct flowi6 fl6; 150 int addr_type; 151 int err; 152 153 if (addr_len < SIN6_LEN_RFC2133) 154 return -EINVAL; 155 156 if (usin->sin6_family != AF_INET6) 157 return -EAFNOSUPPORT; 158 159 memset(&fl6, 0, sizeof(fl6)); 160 161 if (inet6_test_bit(SNDFLOW, sk)) { 162 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK; 163 IP6_ECN_flow_init(fl6.flowlabel); 164 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) { 165 struct ip6_flowlabel *flowlabel; 166 flowlabel = fl6_sock_lookup(sk, fl6.flowlabel); 167 if (IS_ERR(flowlabel)) 168 return -EINVAL; 169 fl6_sock_release(flowlabel); 170 } 171 } 172 173 /* 174 * connect() to INADDR_ANY means loopback (BSD'ism). 175 */ 176 177 if (ipv6_addr_any(&usin->sin6_addr)) { 178 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) 179 ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK), 180 &usin->sin6_addr); 181 else 182 usin->sin6_addr = in6addr_loopback; 183 } 184 185 addr_type = ipv6_addr_type(&usin->sin6_addr); 186 187 if (addr_type & IPV6_ADDR_MULTICAST) 188 return -ENETUNREACH; 189 190 if (addr_type&IPV6_ADDR_LINKLOCAL) { 191 if (addr_len >= sizeof(struct sockaddr_in6) && 192 usin->sin6_scope_id) { 193 /* If interface is set while binding, indices 194 * must coincide. 195 */ 196 if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id)) 197 return -EINVAL; 198 199 sk->sk_bound_dev_if = usin->sin6_scope_id; 200 } 201 202 /* Connect to link-local address requires an interface */ 203 if (!sk->sk_bound_dev_if) 204 return -EINVAL; 205 } 206 207 if (tp->rx_opt.ts_recent_stamp && 208 !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) { 209 tp->rx_opt.ts_recent = 0; 210 tp->rx_opt.ts_recent_stamp = 0; 211 WRITE_ONCE(tp->write_seq, 0); 212 } 213 214 sk->sk_v6_daddr = usin->sin6_addr; 215 np->flow_label = fl6.flowlabel; 216 217 /* 218 * TCP over IPv4 219 */ 220 221 if (addr_type & IPV6_ADDR_MAPPED) { 222 u32 exthdrlen = icsk->icsk_ext_hdr_len; 223 struct sockaddr_in sin; 224 225 if (ipv6_only_sock(sk)) 226 return -ENETUNREACH; 227 228 sin.sin_family = AF_INET; 229 sin.sin_port = usin->sin6_port; 230 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3]; 231 232 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 233 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped); 234 if (sk_is_mptcp(sk)) 235 mptcpv6_handle_mapped(sk, true); 236 sk->sk_backlog_rcv = tcp_v4_do_rcv; 237 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 238 tp->af_specific = &tcp_sock_ipv6_mapped_specific; 239 #endif 240 241 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin)); 242 243 if (err) { 244 icsk->icsk_ext_hdr_len = exthdrlen; 245 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 246 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific); 247 if (sk_is_mptcp(sk)) 248 mptcpv6_handle_mapped(sk, false); 249 sk->sk_backlog_rcv = tcp_v6_do_rcv; 250 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 251 tp->af_specific = &tcp_sock_ipv6_specific; 252 #endif 253 goto failure; 254 } 255 np->saddr = sk->sk_v6_rcv_saddr; 256 257 return err; 258 } 259 260 if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) 261 saddr = &sk->sk_v6_rcv_saddr; 262 263 fl6.flowi6_proto = IPPROTO_TCP; 264 fl6.daddr = sk->sk_v6_daddr; 265 fl6.saddr = saddr ? *saddr : np->saddr; 266 fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label); 267 fl6.flowi6_oif = sk->sk_bound_dev_if; 268 fl6.flowi6_mark = sk->sk_mark; 269 fl6.fl6_dport = usin->sin6_port; 270 fl6.fl6_sport = inet->inet_sport; 271 if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6.fl6_sport) 272 fl6.flowi6_flags = FLOWI_FLAG_ANY_SPORT; 273 fl6.flowi6_uid = sk_uid(sk); 274 275 opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk)); 276 final_p = fl6_update_dst(&fl6, opt, &final); 277 278 security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6)); 279 280 dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p); 281 if (IS_ERR(dst)) { 282 err = PTR_ERR(dst); 283 goto failure; 284 } 285 286 tp->tcp_usec_ts = dst_tcp_usec_ts(dst); 287 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row; 288 289 if (!saddr) { 290 saddr = &fl6.saddr; 291 292 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6); 293 if (err) 294 goto failure; 295 } 296 297 /* set the source address */ 298 np->saddr = *saddr; 299 inet->inet_rcv_saddr = LOOPBACK4_IPV6; 300 301 sk->sk_gso_type = SKB_GSO_TCPV6; 302 ip6_dst_store(sk, dst, NULL, NULL); 303 304 icsk->icsk_ext_hdr_len = 0; 305 if (opt) 306 icsk->icsk_ext_hdr_len = opt->opt_flen + 307 opt->opt_nflen; 308 309 tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr); 310 311 inet->inet_dport = usin->sin6_port; 312 313 tcp_set_state(sk, TCP_SYN_SENT); 314 err = inet6_hash_connect(tcp_death_row, sk); 315 if (err) 316 goto late_failure; 317 318 sk_set_txhash(sk); 319 320 if (likely(!tp->repair)) { 321 if (!tp->write_seq) 322 WRITE_ONCE(tp->write_seq, 323 secure_tcpv6_seq(np->saddr.s6_addr32, 324 sk->sk_v6_daddr.s6_addr32, 325 inet->inet_sport, 326 inet->inet_dport)); 327 tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, 328 sk->sk_v6_daddr.s6_addr32); 329 } 330 331 if (tcp_fastopen_defer_connect(sk, &err)) 332 return err; 333 if (err) 334 goto late_failure; 335 336 err = tcp_connect(sk); 337 if (err) 338 goto late_failure; 339 340 return 0; 341 342 late_failure: 343 tcp_set_state(sk, TCP_CLOSE); 344 inet_bhash2_reset_saddr(sk); 345 failure: 346 inet->inet_dport = 0; 347 sk->sk_route_caps = 0; 348 return err; 349 } 350 351 static void tcp_v6_mtu_reduced(struct sock *sk) 352 { 353 struct dst_entry *dst; 354 u32 mtu; 355 356 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 357 return; 358 359 mtu = READ_ONCE(tcp_sk(sk)->mtu_info); 360 361 /* Drop requests trying to increase our current mss. 362 * Check done in __ip6_rt_update_pmtu() is too late. 363 */ 364 if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache) 365 return; 366 367 dst = inet6_csk_update_pmtu(sk, mtu); 368 if (!dst) 369 return; 370 371 if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) { 372 tcp_sync_mss(sk, dst_mtu(dst)); 373 tcp_simple_retransmit(sk); 374 } 375 } 376 377 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 378 u8 type, u8 code, int offset, __be32 info) 379 { 380 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; 381 const struct tcphdr *th = (struct tcphdr *)(skb->data+offset); 382 struct net *net = dev_net_rcu(skb->dev); 383 struct request_sock *fastopen; 384 struct ipv6_pinfo *np; 385 struct tcp_sock *tp; 386 __u32 seq, snd_una; 387 struct sock *sk; 388 bool fatal; 389 int err; 390 391 sk = __inet6_lookup_established(net, &hdr->daddr, th->dest, 392 &hdr->saddr, ntohs(th->source), 393 skb->dev->ifindex, inet6_sdif(skb)); 394 395 if (!sk) { 396 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), 397 ICMP6_MIB_INERRORS); 398 return -ENOENT; 399 } 400 401 if (sk->sk_state == TCP_TIME_WAIT) { 402 /* To increase the counter of ignored icmps for TCP-AO */ 403 tcp_ao_ignore_icmp(sk, AF_INET6, type, code); 404 inet_twsk_put(inet_twsk(sk)); 405 return 0; 406 } 407 seq = ntohl(th->seq); 408 fatal = icmpv6_err_convert(type, code, &err); 409 if (sk->sk_state == TCP_NEW_SYN_RECV) { 410 tcp_req_err(sk, seq, fatal); 411 return 0; 412 } 413 414 if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) { 415 sock_put(sk); 416 return 0; 417 } 418 419 bh_lock_sock(sk); 420 if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG) 421 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS); 422 423 if (sk->sk_state == TCP_CLOSE) 424 goto out; 425 426 if (static_branch_unlikely(&ip6_min_hopcount)) { 427 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 428 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) { 429 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 430 goto out; 431 } 432 } 433 434 tp = tcp_sk(sk); 435 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */ 436 fastopen = rcu_dereference(tp->fastopen_rsk); 437 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una; 438 if (sk->sk_state != TCP_LISTEN && 439 !between(seq, snd_una, tp->snd_nxt)) { 440 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS); 441 goto out; 442 } 443 444 np = tcp_inet6_sk(sk); 445 446 if (type == NDISC_REDIRECT) { 447 if (!sock_owned_by_user(sk)) { 448 struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie); 449 450 if (dst) 451 dst->ops->redirect(dst, sk, skb); 452 } 453 goto out; 454 } 455 456 if (type == ICMPV6_PKT_TOOBIG) { 457 u32 mtu = ntohl(info); 458 459 /* We are not interested in TCP_LISTEN and open_requests 460 * (SYN-ACKs send out by Linux are always <576bytes so 461 * they should go through unfragmented). 462 */ 463 if (sk->sk_state == TCP_LISTEN) 464 goto out; 465 466 if (!ip6_sk_accept_pmtu(sk)) 467 goto out; 468 469 if (mtu < IPV6_MIN_MTU) 470 goto out; 471 472 WRITE_ONCE(tp->mtu_info, mtu); 473 474 if (!sock_owned_by_user(sk)) 475 tcp_v6_mtu_reduced(sk); 476 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, 477 &sk->sk_tsq_flags)) 478 sock_hold(sk); 479 goto out; 480 } 481 482 483 /* Might be for an request_sock */ 484 switch (sk->sk_state) { 485 case TCP_SYN_SENT: 486 case TCP_SYN_RECV: 487 /* Only in fast or simultaneous open. If a fast open socket is 488 * already accepted it is treated as a connected one below. 489 */ 490 if (fastopen && !fastopen->sk) 491 break; 492 493 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th); 494 495 if (!sock_owned_by_user(sk)) 496 tcp_done_with_error(sk, err); 497 else 498 WRITE_ONCE(sk->sk_err_soft, err); 499 goto out; 500 case TCP_LISTEN: 501 break; 502 default: 503 /* check if this ICMP message allows revert of backoff. 504 * (see RFC 6069) 505 */ 506 if (!fastopen && type == ICMPV6_DEST_UNREACH && 507 code == ICMPV6_NOROUTE) 508 tcp_ld_RTO_revert(sk, seq); 509 } 510 511 if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) { 512 WRITE_ONCE(sk->sk_err, err); 513 sk_error_report(sk); 514 } else { 515 WRITE_ONCE(sk->sk_err_soft, err); 516 } 517 out: 518 bh_unlock_sock(sk); 519 sock_put(sk); 520 return 0; 521 } 522 523 524 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 525 struct flowi *fl, 526 struct request_sock *req, 527 struct tcp_fastopen_cookie *foc, 528 enum tcp_synack_type synack_type, 529 struct sk_buff *syn_skb) 530 { 531 struct inet_request_sock *ireq = inet_rsk(req); 532 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 533 struct ipv6_txoptions *opt; 534 struct flowi6 *fl6 = &fl->u.ip6; 535 struct sk_buff *skb; 536 int err = -ENOMEM; 537 u8 tclass; 538 539 /* First, grab a route. */ 540 if (!dst && (dst = inet6_csk_route_req(sk, fl6, req, 541 IPPROTO_TCP)) == NULL) 542 goto done; 543 544 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb); 545 546 if (skb) { 547 tcp_rsk(req)->syn_ect_snt = np->tclass & INET_ECN_MASK; 548 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr, 549 &ireq->ir_v6_rmt_addr); 550 551 fl6->daddr = ireq->ir_v6_rmt_addr; 552 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts) 553 fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts)); 554 555 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ? 556 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) | 557 (np->tclass & INET_ECN_MASK) : 558 np->tclass; 559 560 if (!INET_ECN_is_capable(tclass) && 561 tcp_bpf_ca_needs_ecn((struct sock *)req)) 562 tclass |= INET_ECN_ECT_0; 563 564 rcu_read_lock(); 565 opt = ireq->ipv6_opt; 566 if (!opt) 567 opt = rcu_dereference(np->opt); 568 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark), 569 opt, tclass, READ_ONCE(sk->sk_priority)); 570 rcu_read_unlock(); 571 err = net_xmit_eval(err); 572 } 573 574 done: 575 return err; 576 } 577 578 579 static void tcp_v6_reqsk_destructor(struct request_sock *req) 580 { 581 kfree(inet_rsk(req)->ipv6_opt); 582 consume_skb(inet_rsk(req)->pktopts); 583 } 584 585 #ifdef CONFIG_TCP_MD5SIG 586 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk, 587 const struct in6_addr *addr, 588 int l3index) 589 { 590 return tcp_md5_do_lookup(sk, l3index, 591 (union tcp_md5_addr *)addr, AF_INET6); 592 } 593 594 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk, 595 const struct sock *addr_sk) 596 { 597 int l3index; 598 599 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), 600 addr_sk->sk_bound_dev_if); 601 return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr, 602 l3index); 603 } 604 605 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname, 606 sockptr_t optval, int optlen) 607 { 608 struct tcp_md5sig cmd; 609 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr; 610 union tcp_ao_addr *addr; 611 int l3index = 0; 612 u8 prefixlen; 613 bool l3flag; 614 u8 flags; 615 616 if (optlen < sizeof(cmd)) 617 return -EINVAL; 618 619 if (copy_from_sockptr(&cmd, optval, sizeof(cmd))) 620 return -EFAULT; 621 622 if (sin6->sin6_family != AF_INET6) 623 return -EINVAL; 624 625 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 626 l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 627 628 if (optname == TCP_MD5SIG_EXT && 629 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) { 630 prefixlen = cmd.tcpm_prefixlen; 631 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) && 632 prefixlen > 32)) 633 return -EINVAL; 634 } else { 635 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128; 636 } 637 638 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex && 639 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) { 640 struct net_device *dev; 641 642 rcu_read_lock(); 643 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex); 644 if (dev && netif_is_l3_master(dev)) 645 l3index = dev->ifindex; 646 rcu_read_unlock(); 647 648 /* ok to reference set/not set outside of rcu; 649 * right now device MUST be an L3 master 650 */ 651 if (!dev || !l3index) 652 return -EINVAL; 653 } 654 655 if (!cmd.tcpm_keylen) { 656 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) 657 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3], 658 AF_INET, prefixlen, 659 l3index, flags); 660 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr, 661 AF_INET6, prefixlen, l3index, flags); 662 } 663 664 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN) 665 return -EINVAL; 666 667 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) { 668 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3]; 669 670 /* Don't allow keys for peers that have a matching TCP-AO key. 671 * See the comment in tcp_ao_add_cmd() 672 */ 673 if (tcp_ao_required(sk, addr, AF_INET, 674 l3flag ? l3index : -1, false)) 675 return -EKEYREJECTED; 676 return tcp_md5_do_add(sk, addr, 677 AF_INET, prefixlen, l3index, flags, 678 cmd.tcpm_key, cmd.tcpm_keylen); 679 } 680 681 addr = (union tcp_md5_addr *)&sin6->sin6_addr; 682 683 /* Don't allow keys for peers that have a matching TCP-AO key. 684 * See the comment in tcp_ao_add_cmd() 685 */ 686 if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false)) 687 return -EKEYREJECTED; 688 689 return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags, 690 cmd.tcpm_key, cmd.tcpm_keylen); 691 } 692 693 static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp, 694 const struct in6_addr *daddr, 695 const struct in6_addr *saddr, 696 const struct tcphdr *th, int nbytes) 697 { 698 struct tcp6_pseudohdr *bp; 699 struct scatterlist sg; 700 struct tcphdr *_th; 701 702 bp = hp->scratch; 703 /* 1. TCP pseudo-header (RFC2460) */ 704 bp->saddr = *saddr; 705 bp->daddr = *daddr; 706 bp->protocol = cpu_to_be32(IPPROTO_TCP); 707 bp->len = cpu_to_be32(nbytes); 708 709 _th = (struct tcphdr *)(bp + 1); 710 memcpy(_th, th, sizeof(*th)); 711 _th->check = 0; 712 713 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); 714 ahash_request_set_crypt(hp->req, &sg, NULL, 715 sizeof(*bp) + sizeof(*th)); 716 return crypto_ahash_update(hp->req); 717 } 718 719 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key, 720 const struct in6_addr *daddr, struct in6_addr *saddr, 721 const struct tcphdr *th) 722 { 723 struct tcp_sigpool hp; 724 725 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 726 goto clear_hash_nostart; 727 728 if (crypto_ahash_init(hp.req)) 729 goto clear_hash; 730 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) 731 goto clear_hash; 732 if (tcp_md5_hash_key(&hp, key)) 733 goto clear_hash; 734 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 735 if (crypto_ahash_final(hp.req)) 736 goto clear_hash; 737 738 tcp_sigpool_end(&hp); 739 return 0; 740 741 clear_hash: 742 tcp_sigpool_end(&hp); 743 clear_hash_nostart: 744 memset(md5_hash, 0, 16); 745 return 1; 746 } 747 748 static int tcp_v6_md5_hash_skb(char *md5_hash, 749 const struct tcp_md5sig_key *key, 750 const struct sock *sk, 751 const struct sk_buff *skb) 752 { 753 const struct tcphdr *th = tcp_hdr(skb); 754 const struct in6_addr *saddr, *daddr; 755 struct tcp_sigpool hp; 756 757 if (sk) { /* valid for establish/request sockets */ 758 saddr = &sk->sk_v6_rcv_saddr; 759 daddr = &sk->sk_v6_daddr; 760 } else { 761 const struct ipv6hdr *ip6h = ipv6_hdr(skb); 762 saddr = &ip6h->saddr; 763 daddr = &ip6h->daddr; 764 } 765 766 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 767 goto clear_hash_nostart; 768 769 if (crypto_ahash_init(hp.req)) 770 goto clear_hash; 771 772 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) 773 goto clear_hash; 774 if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2)) 775 goto clear_hash; 776 if (tcp_md5_hash_key(&hp, key)) 777 goto clear_hash; 778 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 779 if (crypto_ahash_final(hp.req)) 780 goto clear_hash; 781 782 tcp_sigpool_end(&hp); 783 return 0; 784 785 clear_hash: 786 tcp_sigpool_end(&hp); 787 clear_hash_nostart: 788 memset(md5_hash, 0, 16); 789 return 1; 790 } 791 #endif 792 793 static void tcp_v6_init_req(struct request_sock *req, 794 const struct sock *sk_listener, 795 struct sk_buff *skb, 796 u32 tw_isn) 797 { 798 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 799 struct inet_request_sock *ireq = inet_rsk(req); 800 const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener); 801 802 ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; 803 ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; 804 ireq->ir_rmt_addr = LOOPBACK4_IPV6; 805 ireq->ir_loc_addr = LOOPBACK4_IPV6; 806 807 /* So that link locals have meaning */ 808 if ((!sk_listener->sk_bound_dev_if || l3_slave) && 809 ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL) 810 ireq->ir_iif = tcp_v6_iif(skb); 811 812 if (!tw_isn && 813 (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) || 814 np->rxopt.bits.rxinfo || 815 np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim || 816 np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) { 817 refcount_inc(&skb->users); 818 ireq->pktopts = skb; 819 } 820 } 821 822 static struct dst_entry *tcp_v6_route_req(const struct sock *sk, 823 struct sk_buff *skb, 824 struct flowi *fl, 825 struct request_sock *req, 826 u32 tw_isn) 827 { 828 tcp_v6_init_req(req, sk, skb, tw_isn); 829 830 if (security_inet_conn_request(sk, skb, req)) 831 return NULL; 832 833 return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP); 834 } 835 836 struct request_sock_ops tcp6_request_sock_ops __read_mostly = { 837 .family = AF_INET6, 838 .obj_size = sizeof(struct tcp6_request_sock), 839 .send_ack = tcp_v6_reqsk_send_ack, 840 .destructor = tcp_v6_reqsk_destructor, 841 .send_reset = tcp_v6_send_reset, 842 .syn_ack_timeout = tcp_syn_ack_timeout, 843 }; 844 845 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { 846 .mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - 847 sizeof(struct ipv6hdr), 848 #ifdef CONFIG_TCP_MD5SIG 849 .req_md5_lookup = tcp_v6_md5_lookup, 850 .calc_md5_hash = tcp_v6_md5_hash_skb, 851 #endif 852 #ifdef CONFIG_TCP_AO 853 .ao_lookup = tcp_v6_ao_lookup_rsk, 854 .ao_calc_key = tcp_v6_ao_calc_key_rsk, 855 .ao_synack_hash = tcp_v6_ao_synack_hash, 856 #endif 857 #ifdef CONFIG_SYN_COOKIES 858 .cookie_init_seq = cookie_v6_init_sequence, 859 #endif 860 .route_req = tcp_v6_route_req, 861 .init_seq = tcp_v6_init_seq, 862 .init_ts_off = tcp_v6_init_ts_off, 863 .send_synack = tcp_v6_send_synack, 864 }; 865 866 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq, 867 u32 ack, u32 win, u32 tsval, u32 tsecr, 868 int oif, int rst, u8 tclass, __be32 label, 869 u32 priority, u32 txhash, struct tcp_key *key) 870 { 871 struct net *net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb); 872 unsigned int tot_len = sizeof(struct tcphdr); 873 struct sock *ctl_sk = net->ipv6.tcp_sk; 874 const struct tcphdr *th = tcp_hdr(skb); 875 __be32 mrst = 0, *topt; 876 struct dst_entry *dst; 877 struct sk_buff *buff; 878 struct tcphdr *t1; 879 struct flowi6 fl6; 880 u32 mark = 0; 881 882 if (tsecr) 883 tot_len += TCPOLEN_TSTAMP_ALIGNED; 884 if (tcp_key_is_md5(key)) 885 tot_len += TCPOLEN_MD5SIG_ALIGNED; 886 if (tcp_key_is_ao(key)) 887 tot_len += tcp_ao_len_aligned(key->ao_key); 888 889 #ifdef CONFIG_MPTCP 890 if (rst && !tcp_key_is_md5(key)) { 891 mrst = mptcp_reset_option(skb); 892 893 if (mrst) 894 tot_len += sizeof(__be32); 895 } 896 #endif 897 898 buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 899 if (!buff) 900 return; 901 902 skb_reserve(buff, MAX_TCP_HEADER); 903 904 t1 = skb_push(buff, tot_len); 905 skb_reset_transport_header(buff); 906 907 /* Swap the send and the receive. */ 908 memset(t1, 0, sizeof(*t1)); 909 t1->dest = th->source; 910 t1->source = th->dest; 911 t1->doff = tot_len / 4; 912 t1->seq = htonl(seq); 913 t1->ack_seq = htonl(ack); 914 t1->ack = !rst || !th->ack; 915 t1->rst = rst; 916 t1->window = htons(win); 917 918 topt = (__be32 *)(t1 + 1); 919 920 if (tsecr) { 921 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 922 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP); 923 *topt++ = htonl(tsval); 924 *topt++ = htonl(tsecr); 925 } 926 927 if (mrst) 928 *topt++ = mrst; 929 930 #ifdef CONFIG_TCP_MD5SIG 931 if (tcp_key_is_md5(key)) { 932 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 933 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG); 934 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key, 935 &ipv6_hdr(skb)->saddr, 936 &ipv6_hdr(skb)->daddr, t1); 937 } 938 #endif 939 #ifdef CONFIG_TCP_AO 940 if (tcp_key_is_ao(key)) { 941 *topt++ = htonl((TCPOPT_AO << 24) | 942 (tcp_ao_len(key->ao_key) << 16) | 943 (key->ao_key->sndid << 8) | 944 (key->rcv_next)); 945 946 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key, 947 key->traffic_key, 948 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr, 949 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr, 950 t1, key->sne); 951 } 952 #endif 953 954 memset(&fl6, 0, sizeof(fl6)); 955 fl6.daddr = ipv6_hdr(skb)->saddr; 956 fl6.saddr = ipv6_hdr(skb)->daddr; 957 fl6.flowlabel = label; 958 959 buff->ip_summed = CHECKSUM_PARTIAL; 960 961 __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr); 962 963 fl6.flowi6_proto = IPPROTO_TCP; 964 if (rt6_need_strict(&fl6.daddr) && !oif) 965 fl6.flowi6_oif = tcp_v6_iif(skb); 966 else { 967 if (!oif && netif_index_is_l3_master(net, skb->skb_iif)) 968 oif = skb->skb_iif; 969 970 fl6.flowi6_oif = oif; 971 } 972 973 if (sk) { 974 /* unconstify the socket only to attach it to buff with care. */ 975 skb_set_owner_edemux(buff, (struct sock *)sk); 976 977 if (sk->sk_state == TCP_TIME_WAIT) 978 mark = inet_twsk(sk)->tw_mark; 979 else 980 mark = READ_ONCE(sk->sk_mark); 981 skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC); 982 } 983 if (txhash) { 984 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */ 985 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4); 986 } 987 fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark; 988 fl6.fl6_dport = t1->dest; 989 fl6.fl6_sport = t1->source; 990 fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL); 991 security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6)); 992 993 /* Pass a socket to ip6_dst_lookup either it is for RST 994 * Underlying function will use this to retrieve the network 995 * namespace 996 */ 997 if (sk && sk->sk_state != TCP_TIME_WAIT) 998 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/ 999 else 1000 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL); 1001 if (!IS_ERR(dst)) { 1002 skb_dst_set(buff, dst); 1003 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL, 1004 tclass, priority); 1005 TCP_INC_STATS(net, TCP_MIB_OUTSEGS); 1006 if (rst) 1007 TCP_INC_STATS(net, TCP_MIB_OUTRSTS); 1008 return; 1009 } 1010 1011 kfree_skb(buff); 1012 } 1013 1014 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 1015 enum sk_rst_reason reason) 1016 { 1017 const struct tcphdr *th = tcp_hdr(skb); 1018 struct ipv6hdr *ipv6h = ipv6_hdr(skb); 1019 const __u8 *md5_hash_location = NULL; 1020 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1021 bool allocated_traffic_key = false; 1022 #endif 1023 const struct tcp_ao_hdr *aoh; 1024 struct tcp_key key = {}; 1025 u32 seq = 0, ack_seq = 0; 1026 __be32 label = 0; 1027 u32 priority = 0; 1028 struct net *net; 1029 u32 txhash = 0; 1030 int oif = 0; 1031 #ifdef CONFIG_TCP_MD5SIG 1032 unsigned char newhash[16]; 1033 int genhash; 1034 struct sock *sk1 = NULL; 1035 #endif 1036 1037 if (th->rst) 1038 return; 1039 1040 /* If sk not NULL, it means we did a successful lookup and incoming 1041 * route had to be correct. prequeue might have dropped our dst. 1042 */ 1043 if (!sk && !ipv6_unicast_destination(skb)) 1044 return; 1045 1046 net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb); 1047 /* Invalid TCP option size or twice included auth */ 1048 if (tcp_parse_auth_options(th, &md5_hash_location, &aoh)) 1049 return; 1050 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1051 rcu_read_lock(); 1052 #endif 1053 #ifdef CONFIG_TCP_MD5SIG 1054 if (sk && sk_fullsock(sk)) { 1055 int l3index; 1056 1057 /* sdif set, means packet ingressed via a device 1058 * in an L3 domain and inet_iif is set to it. 1059 */ 1060 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1061 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index); 1062 if (key.md5_key) 1063 key.type = TCP_KEY_MD5; 1064 } else if (md5_hash_location) { 1065 int dif = tcp_v6_iif_l3_slave(skb); 1066 int sdif = tcp_v6_sdif(skb); 1067 int l3index; 1068 1069 /* 1070 * active side is lost. Try to find listening socket through 1071 * source port, and then find md5 key through listening socket. 1072 * we are not loose security here: 1073 * Incoming packet is checked with md5 hash with finding key, 1074 * no RST generated if md5 hash doesn't match. 1075 */ 1076 sk1 = inet6_lookup_listener(net, NULL, 0, &ipv6h->saddr, th->source, 1077 &ipv6h->daddr, ntohs(th->source), 1078 dif, sdif); 1079 if (!sk1) 1080 goto out; 1081 1082 /* sdif set, means packet ingressed via a device 1083 * in an L3 domain and dif is set to it. 1084 */ 1085 l3index = tcp_v6_sdif(skb) ? dif : 0; 1086 1087 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index); 1088 if (!key.md5_key) 1089 goto out; 1090 key.type = TCP_KEY_MD5; 1091 1092 genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb); 1093 if (genhash || memcmp(md5_hash_location, newhash, 16) != 0) 1094 goto out; 1095 } 1096 #endif 1097 1098 if (th->ack) 1099 seq = ntohl(th->ack_seq); 1100 else 1101 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len - 1102 (th->doff << 2); 1103 1104 #ifdef CONFIG_TCP_AO 1105 if (aoh) { 1106 int l3index; 1107 1108 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1109 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq, 1110 &key.ao_key, &key.traffic_key, 1111 &allocated_traffic_key, 1112 &key.rcv_next, &key.sne)) 1113 goto out; 1114 key.type = TCP_KEY_AO; 1115 } 1116 #endif 1117 1118 if (sk) { 1119 oif = sk->sk_bound_dev_if; 1120 if (sk_fullsock(sk)) { 1121 if (inet6_test_bit(REPFLOW, sk)) 1122 label = ip6_flowlabel(ipv6h); 1123 priority = READ_ONCE(sk->sk_priority); 1124 txhash = sk->sk_txhash; 1125 } 1126 if (sk->sk_state == TCP_TIME_WAIT) { 1127 label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel); 1128 priority = inet_twsk(sk)->tw_priority; 1129 txhash = inet_twsk(sk)->tw_txhash; 1130 } 1131 } else { 1132 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) 1133 label = ip6_flowlabel(ipv6h); 1134 } 1135 1136 trace_tcp_send_reset(sk, skb, reason); 1137 1138 tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1, 1139 ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK, 1140 label, priority, txhash, 1141 &key); 1142 1143 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1144 out: 1145 if (allocated_traffic_key) 1146 kfree(key.traffic_key); 1147 rcu_read_unlock(); 1148 #endif 1149 } 1150 1151 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq, 1152 u32 ack, u32 win, u32 tsval, u32 tsecr, int oif, 1153 struct tcp_key *key, u8 tclass, 1154 __be32 label, u32 priority, u32 txhash) 1155 { 1156 tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0, 1157 tclass, label, priority, txhash, key); 1158 } 1159 1160 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb, 1161 enum tcp_tw_status tw_status) 1162 { 1163 struct inet_timewait_sock *tw = inet_twsk(sk); 1164 struct tcp_timewait_sock *tcptw = tcp_twsk(sk); 1165 u8 tclass = tw->tw_tclass; 1166 struct tcp_key key = {}; 1167 1168 if (tw_status == TCP_TW_ACK_OOW) 1169 tclass &= ~INET_ECN_MASK; 1170 #ifdef CONFIG_TCP_AO 1171 struct tcp_ao_info *ao_info; 1172 1173 if (static_branch_unlikely(&tcp_ao_needed.key)) { 1174 1175 /* FIXME: the segment to-be-acked is not verified yet */ 1176 ao_info = rcu_dereference(tcptw->ao_info); 1177 if (ao_info) { 1178 const struct tcp_ao_hdr *aoh; 1179 1180 /* Invalid TCP option size or twice included auth */ 1181 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1182 goto out; 1183 if (aoh) 1184 key.ao_key = tcp_ao_established_key(sk, ao_info, 1185 aoh->rnext_keyid, -1); 1186 } 1187 } 1188 if (key.ao_key) { 1189 struct tcp_ao_key *rnext_key; 1190 1191 key.traffic_key = snd_other_key(key.ao_key); 1192 /* rcv_next switches to our rcv_next */ 1193 rnext_key = READ_ONCE(ao_info->rnext_key); 1194 key.rcv_next = rnext_key->rcvid; 1195 key.sne = READ_ONCE(ao_info->snd_sne); 1196 key.type = TCP_KEY_AO; 1197 #else 1198 if (0) { 1199 #endif 1200 #ifdef CONFIG_TCP_MD5SIG 1201 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1202 key.md5_key = tcp_twsk_md5_key(tcptw); 1203 if (key.md5_key) 1204 key.type = TCP_KEY_MD5; 1205 #endif 1206 } 1207 1208 tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, 1209 READ_ONCE(tcptw->tw_rcv_nxt), 1210 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale, 1211 tcp_tw_tsval(tcptw), 1212 READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if, 1213 &key, tclass, cpu_to_be32(tw->tw_flowlabel), 1214 tw->tw_priority, tw->tw_txhash); 1215 1216 #ifdef CONFIG_TCP_AO 1217 out: 1218 #endif 1219 inet_twsk_put(tw); 1220 } 1221 1222 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 1223 struct request_sock *req) 1224 { 1225 struct tcp_key key = {}; 1226 1227 #ifdef CONFIG_TCP_AO 1228 if (static_branch_unlikely(&tcp_ao_needed.key) && 1229 tcp_rsk_used_ao(req)) { 1230 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr; 1231 const struct tcp_ao_hdr *aoh; 1232 int l3index; 1233 1234 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1235 /* Invalid TCP option size or twice included auth */ 1236 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1237 return; 1238 if (!aoh) 1239 return; 1240 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1241 (union tcp_ao_addr *)addr, 1242 AF_INET6, aoh->rnext_keyid, -1); 1243 if (unlikely(!key.ao_key)) { 1244 /* Send ACK with any matching MKT for the peer */ 1245 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1246 (union tcp_ao_addr *)addr, 1247 AF_INET6, -1, -1); 1248 /* Matching key disappeared (user removed the key?) 1249 * let the handshake timeout. 1250 */ 1251 if (!key.ao_key) { 1252 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n", 1253 addr, 1254 ntohs(tcp_hdr(skb)->source), 1255 &ipv6_hdr(skb)->daddr, 1256 ntohs(tcp_hdr(skb)->dest)); 1257 return; 1258 } 1259 } 1260 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC); 1261 if (!key.traffic_key) 1262 return; 1263 1264 key.type = TCP_KEY_AO; 1265 key.rcv_next = aoh->keyid; 1266 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req); 1267 #else 1268 if (0) { 1269 #endif 1270 #ifdef CONFIG_TCP_MD5SIG 1271 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1272 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1273 1274 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, 1275 l3index); 1276 if (key.md5_key) 1277 key.type = TCP_KEY_MD5; 1278 #endif 1279 } 1280 1281 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV 1282 * sk->sk_state == TCP_SYN_RECV -> for Fast Open. 1283 */ 1284 tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ? 1285 tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt, 1286 tcp_rsk(req)->rcv_nxt, 1287 tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale, 1288 tcp_rsk_tsval(tcp_rsk(req)), 1289 req->ts_recent, sk->sk_bound_dev_if, 1290 &key, ipv6_get_dsfield(ipv6_hdr(skb)) & ~INET_ECN_MASK, 1291 0, 1292 READ_ONCE(sk->sk_priority), 1293 READ_ONCE(tcp_rsk(req)->txhash)); 1294 if (tcp_key_is_ao(&key)) 1295 kfree(key.traffic_key); 1296 } 1297 1298 1299 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb) 1300 { 1301 #ifdef CONFIG_SYN_COOKIES 1302 const struct tcphdr *th = tcp_hdr(skb); 1303 1304 if (!th->syn) 1305 sk = cookie_v6_check(sk, skb); 1306 #endif 1307 return sk; 1308 } 1309 1310 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 1311 struct tcphdr *th, u32 *cookie) 1312 { 1313 u16 mss = 0; 1314 #ifdef CONFIG_SYN_COOKIES 1315 mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops, 1316 &tcp_request_sock_ipv6_ops, sk, th); 1317 if (mss) { 1318 *cookie = __cookie_v6_init_sequence(iph, th, &mss); 1319 tcp_synq_overflow(sk); 1320 } 1321 #endif 1322 return mss; 1323 } 1324 1325 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb) 1326 { 1327 if (skb->protocol == htons(ETH_P_IP)) 1328 return tcp_v4_conn_request(sk, skb); 1329 1330 if (!ipv6_unicast_destination(skb)) 1331 goto drop; 1332 1333 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 1334 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 1335 return 0; 1336 } 1337 1338 return tcp_conn_request(&tcp6_request_sock_ops, 1339 &tcp_request_sock_ipv6_ops, sk, skb); 1340 1341 drop: 1342 tcp_listendrop(sk); 1343 return 0; /* don't send reset */ 1344 } 1345 1346 static void tcp_v6_restore_cb(struct sk_buff *skb) 1347 { 1348 /* We need to move header back to the beginning if xfrm6_policy_check() 1349 * and tcp_v6_fill_cb() are going to be called again. 1350 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there. 1351 */ 1352 memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6, 1353 sizeof(struct inet6_skb_parm)); 1354 } 1355 1356 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 1357 struct request_sock *req, 1358 struct dst_entry *dst, 1359 struct request_sock *req_unhash, 1360 bool *own_req) 1361 { 1362 struct inet_request_sock *ireq; 1363 struct ipv6_pinfo *newnp; 1364 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1365 struct ipv6_txoptions *opt; 1366 struct inet_sock *newinet; 1367 bool found_dup_sk = false; 1368 struct tcp_sock *newtp; 1369 struct sock *newsk; 1370 #ifdef CONFIG_TCP_MD5SIG 1371 struct tcp_md5sig_key *key; 1372 int l3index; 1373 #endif 1374 struct flowi6 fl6; 1375 1376 if (skb->protocol == htons(ETH_P_IP)) { 1377 /* 1378 * v6 mapped 1379 */ 1380 1381 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst, 1382 req_unhash, own_req); 1383 1384 if (!newsk) 1385 return NULL; 1386 1387 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1388 1389 newnp = tcp_inet6_sk(newsk); 1390 newtp = tcp_sk(newsk); 1391 1392 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1393 1394 newnp->saddr = newsk->sk_v6_rcv_saddr; 1395 1396 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; 1397 if (sk_is_mptcp(newsk)) 1398 mptcpv6_handle_mapped(newsk, true); 1399 newsk->sk_backlog_rcv = tcp_v4_do_rcv; 1400 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1401 newtp->af_specific = &tcp_sock_ipv6_mapped_specific; 1402 #endif 1403 1404 newnp->ipv6_mc_list = NULL; 1405 newnp->ipv6_ac_list = NULL; 1406 newnp->ipv6_fl_list = NULL; 1407 newnp->pktoptions = NULL; 1408 newnp->opt = NULL; 1409 newnp->mcast_oif = inet_iif(skb); 1410 newnp->mcast_hops = ip_hdr(skb)->ttl; 1411 newnp->rcv_flowinfo = 0; 1412 if (inet6_test_bit(REPFLOW, sk)) 1413 newnp->flow_label = 0; 1414 1415 /* 1416 * No need to charge this sock to the relevant IPv6 refcnt debug socks count 1417 * here, tcp_create_openreq_child now does this for us, see the comment in 1418 * that function for the gory details. -acme 1419 */ 1420 1421 /* It is tricky place. Until this moment IPv4 tcp 1422 worked with IPv6 icsk.icsk_af_ops. 1423 Sync it now. 1424 */ 1425 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie); 1426 1427 return newsk; 1428 } 1429 1430 ireq = inet_rsk(req); 1431 1432 if (sk_acceptq_is_full(sk)) 1433 goto exit_overflow; 1434 1435 if (!dst) { 1436 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP); 1437 if (!dst) 1438 goto exit; 1439 } 1440 1441 newsk = tcp_create_openreq_child(sk, req, skb); 1442 if (!newsk) 1443 goto exit_nonewsk; 1444 1445 /* 1446 * No need to charge this sock to the relevant IPv6 refcnt debug socks 1447 * count here, tcp_create_openreq_child now does this for us, see the 1448 * comment in that function for the gory details. -acme 1449 */ 1450 1451 newsk->sk_gso_type = SKB_GSO_TCPV6; 1452 inet6_sk_rx_dst_set(newsk, skb); 1453 1454 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1455 1456 newtp = tcp_sk(newsk); 1457 newinet = inet_sk(newsk); 1458 newnp = tcp_inet6_sk(newsk); 1459 1460 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1461 1462 ip6_dst_store(newsk, dst, NULL, NULL); 1463 1464 newnp->saddr = ireq->ir_v6_loc_addr; 1465 1466 /* Now IPv6 options... 1467 1468 First: no IPv4 options. 1469 */ 1470 newinet->inet_opt = NULL; 1471 newnp->ipv6_mc_list = NULL; 1472 newnp->ipv6_ac_list = NULL; 1473 newnp->ipv6_fl_list = NULL; 1474 1475 /* Clone RX bits */ 1476 newnp->rxopt.all = np->rxopt.all; 1477 1478 newnp->pktoptions = NULL; 1479 newnp->opt = NULL; 1480 newnp->mcast_oif = tcp_v6_iif(skb); 1481 newnp->mcast_hops = ipv6_hdr(skb)->hop_limit; 1482 newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb)); 1483 if (inet6_test_bit(REPFLOW, sk)) 1484 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb)); 1485 1486 /* Set ToS of the new socket based upon the value of incoming SYN. 1487 * ECT bits are set later in tcp_init_transfer(). 1488 */ 1489 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)) 1490 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK; 1491 1492 /* Clone native IPv6 options from listening socket (if any) 1493 1494 Yes, keeping reference count would be much more clever, 1495 but we make one more one thing there: reattach optmem 1496 to newsk. 1497 */ 1498 opt = ireq->ipv6_opt; 1499 if (!opt) 1500 opt = rcu_dereference(np->opt); 1501 if (opt) { 1502 opt = ipv6_dup_options(newsk, opt); 1503 RCU_INIT_POINTER(newnp->opt, opt); 1504 } 1505 inet_csk(newsk)->icsk_ext_hdr_len = 0; 1506 if (opt) 1507 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen + 1508 opt->opt_flen; 1509 1510 tcp_ca_openreq_child(newsk, dst); 1511 1512 tcp_sync_mss(newsk, dst_mtu(dst)); 1513 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); 1514 1515 tcp_initialize_rcv_mss(newsk); 1516 1517 #ifdef CONFIG_TCP_MD5SIG 1518 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif); 1519 1520 if (!tcp_rsk_used_ao(req)) { 1521 /* Copy over the MD5 key from the original socket */ 1522 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index); 1523 if (key) { 1524 const union tcp_md5_addr *addr; 1525 1526 addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr; 1527 if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) 1528 goto put_and_exit; 1529 } 1530 } 1531 #endif 1532 #ifdef CONFIG_TCP_AO 1533 /* Copy over tcp_ao_info if any */ 1534 if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6)) 1535 goto put_and_exit; /* OOM */ 1536 #endif 1537 1538 if (__inet_inherit_port(sk, newsk) < 0) 1539 goto put_and_exit; 1540 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash), 1541 &found_dup_sk); 1542 if (*own_req) { 1543 tcp_move_syn(newtp, req); 1544 1545 /* Clone pktoptions received with SYN, if we own the req */ 1546 if (ireq->pktopts) { 1547 newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk); 1548 consume_skb(ireq->pktopts); 1549 ireq->pktopts = NULL; 1550 if (newnp->pktoptions) 1551 tcp_v6_restore_cb(newnp->pktoptions); 1552 } 1553 } else { 1554 if (!req_unhash && found_dup_sk) { 1555 /* This code path should only be executed in the 1556 * syncookie case only 1557 */ 1558 bh_unlock_sock(newsk); 1559 sock_put(newsk); 1560 newsk = NULL; 1561 } 1562 } 1563 1564 return newsk; 1565 1566 exit_overflow: 1567 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS); 1568 exit_nonewsk: 1569 dst_release(dst); 1570 exit: 1571 tcp_listendrop(sk); 1572 return NULL; 1573 put_and_exit: 1574 inet_csk_prepare_forced_close(newsk); 1575 tcp_done(newsk); 1576 goto exit; 1577 } 1578 1579 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 1580 u32)); 1581 /* The socket must have it's spinlock held when we get 1582 * here, unless it is a TCP_LISTEN socket. 1583 * 1584 * We have a potential double-lock case here, so even when 1585 * doing backlog processing we use the BH locking scheme. 1586 * This is because we cannot sleep with the original spinlock 1587 * held. 1588 */ 1589 INDIRECT_CALLABLE_SCOPE 1590 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) 1591 { 1592 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1593 struct sk_buff *opt_skb = NULL; 1594 enum skb_drop_reason reason; 1595 struct tcp_sock *tp; 1596 1597 /* Imagine: socket is IPv6. IPv4 packet arrives, 1598 goes to IPv4 receive handler and backlogged. 1599 From backlog it always goes here. Kerboom... 1600 Fortunately, tcp_rcv_established and rcv_established 1601 handle them correctly, but it is not case with 1602 tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK 1603 */ 1604 1605 if (skb->protocol == htons(ETH_P_IP)) 1606 return tcp_v4_do_rcv(sk, skb); 1607 1608 /* 1609 * socket locking is here for SMP purposes as backlog rcv 1610 * is currently called with bh processing disabled. 1611 */ 1612 1613 /* Do Stevens' IPV6_PKTOPTIONS. 1614 1615 Yes, guys, it is the only place in our code, where we 1616 may make it not affecting IPv4. 1617 The rest of code is protocol independent, 1618 and I do not like idea to uglify IPv4. 1619 1620 Actually, all the idea behind IPV6_PKTOPTIONS 1621 looks not very well thought. For now we latch 1622 options, received in the last packet, enqueued 1623 by tcp. Feel free to propose better solution. 1624 --ANK (980728) 1625 */ 1626 if (np->rxopt.all && sk->sk_state != TCP_LISTEN) 1627 opt_skb = skb_clone_and_charge_r(skb, sk); 1628 1629 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */ 1630 struct dst_entry *dst; 1631 1632 dst = rcu_dereference_protected(sk->sk_rx_dst, 1633 lockdep_sock_is_held(sk)); 1634 1635 sock_rps_save_rxhash(sk, skb); 1636 sk_mark_napi_id(sk, skb); 1637 if (dst) { 1638 if (sk->sk_rx_dst_ifindex != skb->skb_iif || 1639 INDIRECT_CALL_1(dst->ops->check, ip6_dst_check, 1640 dst, sk->sk_rx_dst_cookie) == NULL) { 1641 RCU_INIT_POINTER(sk->sk_rx_dst, NULL); 1642 dst_release(dst); 1643 } 1644 } 1645 1646 tcp_rcv_established(sk, skb); 1647 if (opt_skb) 1648 goto ipv6_pktoptions; 1649 return 0; 1650 } 1651 1652 if (tcp_checksum_complete(skb)) 1653 goto csum_err; 1654 1655 if (sk->sk_state == TCP_LISTEN) { 1656 struct sock *nsk = tcp_v6_cookie_check(sk, skb); 1657 1658 if (nsk != sk) { 1659 if (nsk) { 1660 reason = tcp_child_process(sk, nsk, skb); 1661 if (reason) 1662 goto reset; 1663 } 1664 return 0; 1665 } 1666 } else 1667 sock_rps_save_rxhash(sk, skb); 1668 1669 reason = tcp_rcv_state_process(sk, skb); 1670 if (reason) 1671 goto reset; 1672 if (opt_skb) 1673 goto ipv6_pktoptions; 1674 return 0; 1675 1676 reset: 1677 tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason)); 1678 discard: 1679 if (opt_skb) 1680 __kfree_skb(opt_skb); 1681 sk_skb_reason_drop(sk, skb, reason); 1682 return 0; 1683 csum_err: 1684 reason = SKB_DROP_REASON_TCP_CSUM; 1685 trace_tcp_bad_csum(skb); 1686 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS); 1687 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS); 1688 goto discard; 1689 1690 1691 ipv6_pktoptions: 1692 /* Do you ask, what is it? 1693 1694 1. skb was enqueued by tcp. 1695 2. skb is added to tail of read queue, rather than out of order. 1696 3. socket is not in passive state. 1697 4. Finally, it really contains options, which user wants to receive. 1698 */ 1699 tp = tcp_sk(sk); 1700 if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt && 1701 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 1702 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo) 1703 WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb)); 1704 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim) 1705 WRITE_ONCE(np->mcast_hops, 1706 ipv6_hdr(opt_skb)->hop_limit); 1707 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass) 1708 np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb)); 1709 if (inet6_test_bit(REPFLOW, sk)) 1710 np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb)); 1711 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) { 1712 tcp_v6_restore_cb(opt_skb); 1713 opt_skb = xchg(&np->pktoptions, opt_skb); 1714 } else { 1715 __kfree_skb(opt_skb); 1716 opt_skb = xchg(&np->pktoptions, NULL); 1717 } 1718 } 1719 1720 consume_skb(opt_skb); 1721 return 0; 1722 } 1723 1724 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr, 1725 const struct tcphdr *th) 1726 { 1727 /* This is tricky: we move IP6CB at its correct location into 1728 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because 1729 * _decode_session6() uses IP6CB(). 1730 * barrier() makes sure compiler won't play aliasing games. 1731 */ 1732 memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb), 1733 sizeof(struct inet6_skb_parm)); 1734 barrier(); 1735 1736 TCP_SKB_CB(skb)->seq = ntohl(th->seq); 1737 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin + 1738 skb->len - th->doff*4); 1739 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq); 1740 TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th); 1741 TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr); 1742 TCP_SKB_CB(skb)->sacked = 0; 1743 TCP_SKB_CB(skb)->has_rxtstamp = 1744 skb->tstamp || skb_hwtstamps(skb)->hwtstamp; 1745 } 1746 1747 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb) 1748 { 1749 struct net *net = dev_net_rcu(skb->dev); 1750 enum skb_drop_reason drop_reason; 1751 enum tcp_tw_status tw_status; 1752 int sdif = inet6_sdif(skb); 1753 int dif = inet6_iif(skb); 1754 const struct tcphdr *th; 1755 const struct ipv6hdr *hdr; 1756 struct sock *sk = NULL; 1757 bool refcounted; 1758 int ret; 1759 u32 isn; 1760 1761 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 1762 if (skb->pkt_type != PACKET_HOST) 1763 goto discard_it; 1764 1765 /* 1766 * Count it even if it's bad. 1767 */ 1768 __TCP_INC_STATS(net, TCP_MIB_INSEGS); 1769 1770 if (!pskb_may_pull(skb, sizeof(struct tcphdr))) 1771 goto discard_it; 1772 1773 th = (const struct tcphdr *)skb->data; 1774 1775 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) { 1776 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL; 1777 goto bad_packet; 1778 } 1779 if (!pskb_may_pull(skb, th->doff*4)) 1780 goto discard_it; 1781 1782 if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo)) 1783 goto csum_error; 1784 1785 th = (const struct tcphdr *)skb->data; 1786 hdr = ipv6_hdr(skb); 1787 1788 lookup: 1789 sk = __inet6_lookup_skb(skb, __tcp_hdrlen(th), 1790 th->source, th->dest, inet6_iif(skb), sdif, 1791 &refcounted); 1792 if (!sk) 1793 goto no_tcp_socket; 1794 1795 if (sk->sk_state == TCP_TIME_WAIT) 1796 goto do_time_wait; 1797 1798 if (sk->sk_state == TCP_NEW_SYN_RECV) { 1799 struct request_sock *req = inet_reqsk(sk); 1800 bool req_stolen = false; 1801 struct sock *nsk; 1802 1803 sk = req->rsk_listener; 1804 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) 1805 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1806 else 1807 drop_reason = tcp_inbound_hash(sk, req, skb, 1808 &hdr->saddr, &hdr->daddr, 1809 AF_INET6, dif, sdif); 1810 if (drop_reason) { 1811 sk_drops_skbadd(sk, skb); 1812 reqsk_put(req); 1813 goto discard_it; 1814 } 1815 if (tcp_checksum_complete(skb)) { 1816 reqsk_put(req); 1817 goto csum_error; 1818 } 1819 if (unlikely(sk->sk_state != TCP_LISTEN)) { 1820 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb); 1821 if (!nsk) { 1822 inet_csk_reqsk_queue_drop_and_put(sk, req); 1823 goto lookup; 1824 } 1825 sk = nsk; 1826 /* reuseport_migrate_sock() has already held one sk_refcnt 1827 * before returning. 1828 */ 1829 } else { 1830 sock_hold(sk); 1831 } 1832 refcounted = true; 1833 nsk = NULL; 1834 if (!tcp_filter(sk, skb, &drop_reason)) { 1835 th = (const struct tcphdr *)skb->data; 1836 hdr = ipv6_hdr(skb); 1837 tcp_v6_fill_cb(skb, hdr, th); 1838 nsk = tcp_check_req(sk, skb, req, false, &req_stolen, 1839 &drop_reason); 1840 } 1841 if (!nsk) { 1842 reqsk_put(req); 1843 if (req_stolen) { 1844 /* Another cpu got exclusive access to req 1845 * and created a full blown socket. 1846 * Try to feed this packet to this socket 1847 * instead of discarding it. 1848 */ 1849 tcp_v6_restore_cb(skb); 1850 sock_put(sk); 1851 goto lookup; 1852 } 1853 goto discard_and_relse; 1854 } 1855 nf_reset_ct(skb); 1856 if (nsk == sk) { 1857 reqsk_put(req); 1858 tcp_v6_restore_cb(skb); 1859 } else { 1860 drop_reason = tcp_child_process(sk, nsk, skb); 1861 if (drop_reason) { 1862 enum sk_rst_reason rst_reason; 1863 1864 rst_reason = sk_rst_convert_drop_reason(drop_reason); 1865 tcp_v6_send_reset(nsk, skb, rst_reason); 1866 goto discard_and_relse; 1867 } 1868 sock_put(sk); 1869 return 0; 1870 } 1871 } 1872 1873 process: 1874 if (static_branch_unlikely(&ip6_min_hopcount)) { 1875 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 1876 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) { 1877 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 1878 drop_reason = SKB_DROP_REASON_TCP_MINTTL; 1879 goto discard_and_relse; 1880 } 1881 } 1882 1883 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) { 1884 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1885 goto discard_and_relse; 1886 } 1887 1888 drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr, 1889 AF_INET6, dif, sdif); 1890 if (drop_reason) 1891 goto discard_and_relse; 1892 1893 nf_reset_ct(skb); 1894 1895 if (tcp_filter(sk, skb, &drop_reason)) 1896 goto discard_and_relse; 1897 1898 th = (const struct tcphdr *)skb->data; 1899 hdr = ipv6_hdr(skb); 1900 tcp_v6_fill_cb(skb, hdr, th); 1901 1902 skb->dev = NULL; 1903 1904 if (sk->sk_state == TCP_LISTEN) { 1905 ret = tcp_v6_do_rcv(sk, skb); 1906 goto put_and_return; 1907 } 1908 1909 sk_incoming_cpu_update(sk); 1910 1911 bh_lock_sock_nested(sk); 1912 tcp_segs_in(tcp_sk(sk), skb); 1913 ret = 0; 1914 if (!sock_owned_by_user(sk)) { 1915 ret = tcp_v6_do_rcv(sk, skb); 1916 } else { 1917 if (tcp_add_backlog(sk, skb, &drop_reason)) 1918 goto discard_and_relse; 1919 } 1920 bh_unlock_sock(sk); 1921 put_and_return: 1922 if (refcounted) 1923 sock_put(sk); 1924 return ret ? -1 : 0; 1925 1926 no_tcp_socket: 1927 drop_reason = SKB_DROP_REASON_NO_SOCKET; 1928 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) 1929 goto discard_it; 1930 1931 tcp_v6_fill_cb(skb, hdr, th); 1932 1933 if (tcp_checksum_complete(skb)) { 1934 csum_error: 1935 drop_reason = SKB_DROP_REASON_TCP_CSUM; 1936 trace_tcp_bad_csum(skb); 1937 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS); 1938 bad_packet: 1939 __TCP_INC_STATS(net, TCP_MIB_INERRS); 1940 } else { 1941 tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason)); 1942 } 1943 1944 discard_it: 1945 SKB_DR_OR(drop_reason, NOT_SPECIFIED); 1946 sk_skb_reason_drop(sk, skb, drop_reason); 1947 return 0; 1948 1949 discard_and_relse: 1950 sk_drops_skbadd(sk, skb); 1951 if (refcounted) 1952 sock_put(sk); 1953 goto discard_it; 1954 1955 do_time_wait: 1956 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) { 1957 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1958 inet_twsk_put(inet_twsk(sk)); 1959 goto discard_it; 1960 } 1961 1962 tcp_v6_fill_cb(skb, hdr, th); 1963 1964 if (tcp_checksum_complete(skb)) { 1965 inet_twsk_put(inet_twsk(sk)); 1966 goto csum_error; 1967 } 1968 1969 tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn, 1970 &drop_reason); 1971 switch (tw_status) { 1972 case TCP_TW_SYN: 1973 { 1974 struct sock *sk2; 1975 1976 sk2 = inet6_lookup_listener(net, skb, __tcp_hdrlen(th), 1977 &ipv6_hdr(skb)->saddr, th->source, 1978 &ipv6_hdr(skb)->daddr, 1979 ntohs(th->dest), 1980 tcp_v6_iif_l3_slave(skb), 1981 sdif); 1982 if (sk2) { 1983 struct inet_timewait_sock *tw = inet_twsk(sk); 1984 inet_twsk_deschedule_put(tw); 1985 sk = sk2; 1986 tcp_v6_restore_cb(skb); 1987 refcounted = false; 1988 __this_cpu_write(tcp_tw_isn, isn); 1989 goto process; 1990 } 1991 } 1992 /* to ACK */ 1993 fallthrough; 1994 case TCP_TW_ACK: 1995 case TCP_TW_ACK_OOW: 1996 tcp_v6_timewait_ack(sk, skb, tw_status); 1997 break; 1998 case TCP_TW_RST: 1999 tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET); 2000 inet_twsk_deschedule_put(inet_twsk(sk)); 2001 goto discard_it; 2002 case TCP_TW_SUCCESS: 2003 ; 2004 } 2005 goto discard_it; 2006 } 2007 2008 void tcp_v6_early_demux(struct sk_buff *skb) 2009 { 2010 struct net *net = dev_net_rcu(skb->dev); 2011 const struct ipv6hdr *hdr; 2012 const struct tcphdr *th; 2013 struct sock *sk; 2014 2015 if (skb->pkt_type != PACKET_HOST) 2016 return; 2017 2018 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr))) 2019 return; 2020 2021 hdr = ipv6_hdr(skb); 2022 th = tcp_hdr(skb); 2023 2024 if (th->doff < sizeof(struct tcphdr) / 4) 2025 return; 2026 2027 /* Note : We use inet6_iif() here, not tcp_v6_iif() */ 2028 sk = __inet6_lookup_established(net, &hdr->saddr, th->source, 2029 &hdr->daddr, ntohs(th->dest), 2030 inet6_iif(skb), inet6_sdif(skb)); 2031 if (sk) { 2032 skb->sk = sk; 2033 skb->destructor = sock_edemux; 2034 if (sk_fullsock(sk)) { 2035 struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst); 2036 2037 if (dst) 2038 dst = dst_check(dst, sk->sk_rx_dst_cookie); 2039 if (dst && 2040 sk->sk_rx_dst_ifindex == skb->skb_iif) 2041 skb_dst_set_noref(skb, dst); 2042 } 2043 } 2044 } 2045 2046 static struct timewait_sock_ops tcp6_timewait_sock_ops = { 2047 .twsk_obj_size = sizeof(struct tcp6_timewait_sock), 2048 }; 2049 2050 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) 2051 { 2052 __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr); 2053 } 2054 2055 const struct inet_connection_sock_af_ops ipv6_specific = { 2056 .queue_xmit = inet6_csk_xmit, 2057 .send_check = tcp_v6_send_check, 2058 .rebuild_header = inet6_sk_rebuild_header, 2059 .sk_rx_dst_set = inet6_sk_rx_dst_set, 2060 .conn_request = tcp_v6_conn_request, 2061 .syn_recv_sock = tcp_v6_syn_recv_sock, 2062 .net_header_len = sizeof(struct ipv6hdr), 2063 .setsockopt = ipv6_setsockopt, 2064 .getsockopt = ipv6_getsockopt, 2065 .mtu_reduced = tcp_v6_mtu_reduced, 2066 }; 2067 2068 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2069 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = { 2070 #ifdef CONFIG_TCP_MD5SIG 2071 .md5_lookup = tcp_v6_md5_lookup, 2072 .calc_md5_hash = tcp_v6_md5_hash_skb, 2073 .md5_parse = tcp_v6_parse_md5_keys, 2074 #endif 2075 #ifdef CONFIG_TCP_AO 2076 .ao_lookup = tcp_v6_ao_lookup, 2077 .calc_ao_hash = tcp_v6_ao_hash_skb, 2078 .ao_parse = tcp_v6_parse_ao, 2079 .ao_calc_key_sk = tcp_v6_ao_calc_key_sk, 2080 #endif 2081 }; 2082 #endif 2083 2084 /* 2085 * TCP over IPv4 via INET6 API 2086 */ 2087 static const struct inet_connection_sock_af_ops ipv6_mapped = { 2088 .queue_xmit = ip_queue_xmit, 2089 .send_check = tcp_v4_send_check, 2090 .rebuild_header = inet_sk_rebuild_header, 2091 .sk_rx_dst_set = inet_sk_rx_dst_set, 2092 .conn_request = tcp_v6_conn_request, 2093 .syn_recv_sock = tcp_v6_syn_recv_sock, 2094 .net_header_len = sizeof(struct iphdr), 2095 .setsockopt = ipv6_setsockopt, 2096 .getsockopt = ipv6_getsockopt, 2097 .mtu_reduced = tcp_v4_mtu_reduced, 2098 }; 2099 2100 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2101 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = { 2102 #ifdef CONFIG_TCP_MD5SIG 2103 .md5_lookup = tcp_v4_md5_lookup, 2104 .calc_md5_hash = tcp_v4_md5_hash_skb, 2105 .md5_parse = tcp_v6_parse_md5_keys, 2106 #endif 2107 #ifdef CONFIG_TCP_AO 2108 .ao_lookup = tcp_v6_ao_lookup, 2109 .calc_ao_hash = tcp_v4_ao_hash_skb, 2110 .ao_parse = tcp_v6_parse_ao, 2111 .ao_calc_key_sk = tcp_v4_ao_calc_key_sk, 2112 #endif 2113 }; 2114 2115 static void tcp6_destruct_sock(struct sock *sk) 2116 { 2117 tcp_md5_destruct_sock(sk); 2118 tcp_ao_destroy_sock(sk, false); 2119 inet6_sock_destruct(sk); 2120 } 2121 #endif 2122 2123 /* NOTE: A lot of things set to zero explicitly by call to 2124 * sk_alloc() so need not be done here. 2125 */ 2126 static int tcp_v6_init_sock(struct sock *sk) 2127 { 2128 struct inet_connection_sock *icsk = inet_csk(sk); 2129 2130 tcp_init_sock(sk); 2131 2132 icsk->icsk_af_ops = &ipv6_specific; 2133 2134 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2135 tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific; 2136 sk->sk_destruct = tcp6_destruct_sock; 2137 #endif 2138 2139 return 0; 2140 } 2141 2142 #ifdef CONFIG_PROC_FS 2143 /* Proc filesystem TCPv6 sock list dumping. */ 2144 static void get_openreq6(struct seq_file *seq, 2145 const struct request_sock *req, int i) 2146 { 2147 long ttd = req->rsk_timer.expires - jiffies; 2148 const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr; 2149 const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr; 2150 2151 if (ttd < 0) 2152 ttd = 0; 2153 2154 seq_printf(seq, 2155 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2156 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n", 2157 i, 2158 src->s6_addr32[0], src->s6_addr32[1], 2159 src->s6_addr32[2], src->s6_addr32[3], 2160 inet_rsk(req)->ir_num, 2161 dest->s6_addr32[0], dest->s6_addr32[1], 2162 dest->s6_addr32[2], dest->s6_addr32[3], 2163 ntohs(inet_rsk(req)->ir_rmt_port), 2164 TCP_SYN_RECV, 2165 0, 0, /* could print option size, but that is af dependent. */ 2166 1, /* timers active (only the expire timer) */ 2167 jiffies_to_clock_t(ttd), 2168 req->num_timeout, 2169 from_kuid_munged(seq_user_ns(seq), 2170 sk_uid(req->rsk_listener)), 2171 0, /* non standard timer */ 2172 0, /* open_requests have no inode */ 2173 0, req); 2174 } 2175 2176 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i) 2177 { 2178 const struct in6_addr *dest, *src; 2179 __u16 destp, srcp; 2180 int timer_active; 2181 unsigned long timer_expires; 2182 const struct inet_sock *inet = inet_sk(sp); 2183 const struct tcp_sock *tp = tcp_sk(sp); 2184 const struct inet_connection_sock *icsk = inet_csk(sp); 2185 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq; 2186 u8 icsk_pending; 2187 int rx_queue; 2188 int state; 2189 2190 dest = &sp->sk_v6_daddr; 2191 src = &sp->sk_v6_rcv_saddr; 2192 destp = ntohs(inet->inet_dport); 2193 srcp = ntohs(inet->inet_sport); 2194 2195 icsk_pending = smp_load_acquire(&icsk->icsk_pending); 2196 if (icsk_pending == ICSK_TIME_RETRANS || 2197 icsk_pending == ICSK_TIME_REO_TIMEOUT || 2198 icsk_pending == ICSK_TIME_LOSS_PROBE) { 2199 timer_active = 1; 2200 timer_expires = icsk_timeout(icsk); 2201 } else if (icsk_pending == ICSK_TIME_PROBE0) { 2202 timer_active = 4; 2203 timer_expires = icsk_timeout(icsk); 2204 } else if (timer_pending(&sp->sk_timer)) { 2205 timer_active = 2; 2206 timer_expires = sp->sk_timer.expires; 2207 } else { 2208 timer_active = 0; 2209 timer_expires = jiffies; 2210 } 2211 2212 state = inet_sk_state_load(sp); 2213 if (state == TCP_LISTEN) 2214 rx_queue = READ_ONCE(sp->sk_ack_backlog); 2215 else 2216 /* Because we don't lock the socket, 2217 * we might find a transient negative value. 2218 */ 2219 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) - 2220 READ_ONCE(tp->copied_seq), 0); 2221 2222 seq_printf(seq, 2223 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2224 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n", 2225 i, 2226 src->s6_addr32[0], src->s6_addr32[1], 2227 src->s6_addr32[2], src->s6_addr32[3], srcp, 2228 dest->s6_addr32[0], dest->s6_addr32[1], 2229 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2230 state, 2231 READ_ONCE(tp->write_seq) - tp->snd_una, 2232 rx_queue, 2233 timer_active, 2234 jiffies_delta_to_clock_t(timer_expires - jiffies), 2235 READ_ONCE(icsk->icsk_retransmits), 2236 from_kuid_munged(seq_user_ns(seq), sk_uid(sp)), 2237 READ_ONCE(icsk->icsk_probes_out), 2238 sock_i_ino(sp), 2239 refcount_read(&sp->sk_refcnt), sp, 2240 jiffies_to_clock_t(icsk->icsk_rto), 2241 jiffies_to_clock_t(icsk->icsk_ack.ato), 2242 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp), 2243 tcp_snd_cwnd(tp), 2244 state == TCP_LISTEN ? 2245 fastopenq->max_qlen : 2246 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh) 2247 ); 2248 } 2249 2250 static void get_timewait6_sock(struct seq_file *seq, 2251 struct inet_timewait_sock *tw, int i) 2252 { 2253 long delta = tw->tw_timer.expires - jiffies; 2254 const struct in6_addr *dest, *src; 2255 __u16 destp, srcp; 2256 2257 dest = &tw->tw_v6_daddr; 2258 src = &tw->tw_v6_rcv_saddr; 2259 destp = ntohs(tw->tw_dport); 2260 srcp = ntohs(tw->tw_sport); 2261 2262 seq_printf(seq, 2263 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2264 "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", 2265 i, 2266 src->s6_addr32[0], src->s6_addr32[1], 2267 src->s6_addr32[2], src->s6_addr32[3], srcp, 2268 dest->s6_addr32[0], dest->s6_addr32[1], 2269 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2270 READ_ONCE(tw->tw_substate), 0, 0, 2271 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, 2272 refcount_read(&tw->tw_refcnt), tw); 2273 } 2274 2275 static int tcp6_seq_show(struct seq_file *seq, void *v) 2276 { 2277 struct tcp_iter_state *st; 2278 struct sock *sk = v; 2279 2280 if (v == SEQ_START_TOKEN) { 2281 seq_puts(seq, 2282 " sl " 2283 "local_address " 2284 "remote_address " 2285 "st tx_queue rx_queue tr tm->when retrnsmt" 2286 " uid timeout inode\n"); 2287 goto out; 2288 } 2289 st = seq->private; 2290 2291 if (sk->sk_state == TCP_TIME_WAIT) 2292 get_timewait6_sock(seq, v, st->num); 2293 else if (sk->sk_state == TCP_NEW_SYN_RECV) 2294 get_openreq6(seq, v, st->num); 2295 else 2296 get_tcp6_sock(seq, v, st->num); 2297 out: 2298 return 0; 2299 } 2300 2301 static const struct seq_operations tcp6_seq_ops = { 2302 .show = tcp6_seq_show, 2303 .start = tcp_seq_start, 2304 .next = tcp_seq_next, 2305 .stop = tcp_seq_stop, 2306 }; 2307 2308 static struct tcp_seq_afinfo tcp6_seq_afinfo = { 2309 .family = AF_INET6, 2310 }; 2311 2312 int __net_init tcp6_proc_init(struct net *net) 2313 { 2314 if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops, 2315 sizeof(struct tcp_iter_state), &tcp6_seq_afinfo)) 2316 return -ENOMEM; 2317 return 0; 2318 } 2319 2320 void tcp6_proc_exit(struct net *net) 2321 { 2322 remove_proc_entry("tcp6", net->proc_net); 2323 } 2324 #endif 2325 2326 struct proto tcpv6_prot = { 2327 .name = "TCPv6", 2328 .owner = THIS_MODULE, 2329 .close = tcp_close, 2330 .pre_connect = tcp_v6_pre_connect, 2331 .connect = tcp_v6_connect, 2332 .disconnect = tcp_disconnect, 2333 .accept = inet_csk_accept, 2334 .ioctl = tcp_ioctl, 2335 .init = tcp_v6_init_sock, 2336 .destroy = tcp_v4_destroy_sock, 2337 .shutdown = tcp_shutdown, 2338 .setsockopt = tcp_setsockopt, 2339 .getsockopt = tcp_getsockopt, 2340 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, 2341 .keepalive = tcp_set_keepalive, 2342 .recvmsg = tcp_recvmsg, 2343 .sendmsg = tcp_sendmsg, 2344 .splice_eof = tcp_splice_eof, 2345 .backlog_rcv = tcp_v6_do_rcv, 2346 .release_cb = tcp_release_cb, 2347 .hash = inet6_hash, 2348 .unhash = inet_unhash, 2349 .get_port = inet_csk_get_port, 2350 .put_port = inet_put_port, 2351 #ifdef CONFIG_BPF_SYSCALL 2352 .psock_update_sk_prot = tcp_bpf_update_proto, 2353 #endif 2354 .enter_memory_pressure = tcp_enter_memory_pressure, 2355 .leave_memory_pressure = tcp_leave_memory_pressure, 2356 .stream_memory_free = tcp_stream_memory_free, 2357 .sockets_allocated = &tcp_sockets_allocated, 2358 2359 .memory_allocated = &net_aligned_data.tcp_memory_allocated, 2360 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 2361 2362 .memory_pressure = &tcp_memory_pressure, 2363 .sysctl_mem = sysctl_tcp_mem, 2364 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 2365 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 2366 .max_header = MAX_TCP_HEADER, 2367 .obj_size = sizeof(struct tcp6_sock), 2368 .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6), 2369 .slab_flags = SLAB_TYPESAFE_BY_RCU, 2370 .twsk_prot = &tcp6_timewait_sock_ops, 2371 .rsk_prot = &tcp6_request_sock_ops, 2372 .h.hashinfo = NULL, 2373 .no_autobind = true, 2374 .diag_destroy = tcp_abort, 2375 }; 2376 EXPORT_SYMBOL_GPL(tcpv6_prot); 2377 2378 2379 static struct inet_protosw tcpv6_protosw = { 2380 .type = SOCK_STREAM, 2381 .protocol = IPPROTO_TCP, 2382 .prot = &tcpv6_prot, 2383 .ops = &inet6_stream_ops, 2384 .flags = INET_PROTOSW_PERMANENT | 2385 INET_PROTOSW_ICSK, 2386 }; 2387 2388 static int __net_init tcpv6_net_init(struct net *net) 2389 { 2390 int res; 2391 2392 res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6, 2393 SOCK_RAW, IPPROTO_TCP, net); 2394 if (!res) 2395 net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC; 2396 2397 return res; 2398 } 2399 2400 static void __net_exit tcpv6_net_exit(struct net *net) 2401 { 2402 inet_ctl_sock_destroy(net->ipv6.tcp_sk); 2403 } 2404 2405 static struct pernet_operations tcpv6_net_ops = { 2406 .init = tcpv6_net_init, 2407 .exit = tcpv6_net_exit, 2408 }; 2409 2410 int __init tcpv6_init(void) 2411 { 2412 int ret; 2413 2414 net_hotdata.tcpv6_protocol = (struct inet6_protocol) { 2415 .handler = tcp_v6_rcv, 2416 .err_handler = tcp_v6_err, 2417 .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL, 2418 }; 2419 ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2420 if (ret) 2421 goto out; 2422 2423 /* register inet6 protocol */ 2424 ret = inet6_register_protosw(&tcpv6_protosw); 2425 if (ret) 2426 goto out_tcpv6_protocol; 2427 2428 ret = register_pernet_subsys(&tcpv6_net_ops); 2429 if (ret) 2430 goto out_tcpv6_protosw; 2431 2432 ret = mptcpv6_init(); 2433 if (ret) 2434 goto out_tcpv6_pernet_subsys; 2435 2436 out: 2437 return ret; 2438 2439 out_tcpv6_pernet_subsys: 2440 unregister_pernet_subsys(&tcpv6_net_ops); 2441 out_tcpv6_protosw: 2442 inet6_unregister_protosw(&tcpv6_protosw); 2443 out_tcpv6_protocol: 2444 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2445 goto out; 2446 } 2447 2448 void tcpv6_exit(void) 2449 { 2450 unregister_pernet_subsys(&tcpv6_net_ops); 2451 inet6_unregister_protosw(&tcpv6_protosw); 2452 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2453 } 2454