1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TCP over IPv6 4 * Linux INET6 implementation 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 * 9 * Based on: 10 * linux/net/ipv4/tcp.c 11 * linux/net/ipv4/tcp_input.c 12 * linux/net/ipv4/tcp_output.c 13 * 14 * Fixes: 15 * Hideaki YOSHIFUJI : sin6_scope_id support 16 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which 17 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind 18 * a single port at the same time. 19 * YOSHIFUJI Hideaki @USAGI: convert /proc/net/tcp6 to seq_file. 20 */ 21 22 #include <linux/bottom_half.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/types.h> 26 #include <linux/socket.h> 27 #include <linux/sockios.h> 28 #include <linux/net.h> 29 #include <linux/jiffies.h> 30 #include <linux/in.h> 31 #include <linux/in6.h> 32 #include <linux/netdevice.h> 33 #include <linux/init.h> 34 #include <linux/jhash.h> 35 #include <linux/ipsec.h> 36 #include <linux/times.h> 37 #include <linux/slab.h> 38 #include <linux/uaccess.h> 39 #include <linux/ipv6.h> 40 #include <linux/icmpv6.h> 41 #include <linux/random.h> 42 #include <linux/indirect_call_wrapper.h> 43 44 #include <net/tcp.h> 45 #include <net/ndisc.h> 46 #include <net/inet6_hashtables.h> 47 #include <net/inet6_connection_sock.h> 48 #include <net/ipv6.h> 49 #include <net/transp_v6.h> 50 #include <net/addrconf.h> 51 #include <net/ip6_route.h> 52 #include <net/ip6_checksum.h> 53 #include <net/inet_ecn.h> 54 #include <net/protocol.h> 55 #include <net/xfrm.h> 56 #include <net/snmp.h> 57 #include <net/dsfield.h> 58 #include <net/timewait_sock.h> 59 #include <net/inet_common.h> 60 #include <net/secure_seq.h> 61 #include <net/hotdata.h> 62 #include <net/busy_poll.h> 63 #include <net/rstreason.h> 64 65 #include <linux/proc_fs.h> 66 #include <linux/seq_file.h> 67 68 #include <crypto/hash.h> 69 #include <linux/scatterlist.h> 70 71 #include <trace/events/tcp.h> 72 73 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 74 enum sk_rst_reason reason); 75 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 76 struct request_sock *req); 77 78 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb); 79 80 static const struct inet_connection_sock_af_ops ipv6_mapped; 81 const struct inet_connection_sock_af_ops ipv6_specific; 82 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 83 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific; 84 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific; 85 #endif 86 87 /* Helper returning the inet6 address from a given tcp socket. 88 * It can be used in TCP stack instead of inet6_sk(sk). 89 * This avoids a dereference and allow compiler optimizations. 90 * It is a specialized version of inet6_sk_generic(). 91 */ 92 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \ 93 struct tcp6_sock, tcp)->inet6) 94 95 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 96 { 97 struct dst_entry *dst = skb_dst(skb); 98 99 if (dst && dst_hold_safe(dst)) { 100 rcu_assign_pointer(sk->sk_rx_dst, dst); 101 sk->sk_rx_dst_ifindex = skb->skb_iif; 102 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst)); 103 } 104 } 105 106 static u32 tcp_v6_init_seq(const struct sk_buff *skb) 107 { 108 return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32, 109 ipv6_hdr(skb)->saddr.s6_addr32, 110 tcp_hdr(skb)->dest, 111 tcp_hdr(skb)->source); 112 } 113 114 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb) 115 { 116 return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32, 117 ipv6_hdr(skb)->saddr.s6_addr32); 118 } 119 120 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr, 121 int addr_len) 122 { 123 /* This check is replicated from tcp_v6_connect() and intended to 124 * prevent BPF program called below from accessing bytes that are out 125 * of the bound specified by user in addr_len. 126 */ 127 if (addr_len < SIN6_LEN_RFC2133) 128 return -EINVAL; 129 130 sock_owned_by_me(sk); 131 132 return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len); 133 } 134 135 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, 136 int addr_len) 137 { 138 struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr; 139 struct inet_connection_sock *icsk = inet_csk(sk); 140 struct in6_addr *saddr = NULL, *final_p, final; 141 struct inet_timewait_death_row *tcp_death_row; 142 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 143 struct inet_sock *inet = inet_sk(sk); 144 struct tcp_sock *tp = tcp_sk(sk); 145 struct net *net = sock_net(sk); 146 struct ipv6_txoptions *opt; 147 struct dst_entry *dst; 148 struct flowi6 fl6; 149 int addr_type; 150 int err; 151 152 if (addr_len < SIN6_LEN_RFC2133) 153 return -EINVAL; 154 155 if (usin->sin6_family != AF_INET6) 156 return -EAFNOSUPPORT; 157 158 memset(&fl6, 0, sizeof(fl6)); 159 160 if (inet6_test_bit(SNDFLOW, sk)) { 161 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK; 162 IP6_ECN_flow_init(fl6.flowlabel); 163 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) { 164 struct ip6_flowlabel *flowlabel; 165 flowlabel = fl6_sock_lookup(sk, fl6.flowlabel); 166 if (IS_ERR(flowlabel)) 167 return -EINVAL; 168 fl6_sock_release(flowlabel); 169 } 170 } 171 172 /* 173 * connect() to INADDR_ANY means loopback (BSD'ism). 174 */ 175 176 if (ipv6_addr_any(&usin->sin6_addr)) { 177 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) 178 ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK), 179 &usin->sin6_addr); 180 else 181 usin->sin6_addr = in6addr_loopback; 182 } 183 184 addr_type = ipv6_addr_type(&usin->sin6_addr); 185 186 if (addr_type & IPV6_ADDR_MULTICAST) 187 return -ENETUNREACH; 188 189 if (addr_type&IPV6_ADDR_LINKLOCAL) { 190 if (addr_len >= sizeof(struct sockaddr_in6) && 191 usin->sin6_scope_id) { 192 /* If interface is set while binding, indices 193 * must coincide. 194 */ 195 if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id)) 196 return -EINVAL; 197 198 sk->sk_bound_dev_if = usin->sin6_scope_id; 199 } 200 201 /* Connect to link-local address requires an interface */ 202 if (!sk->sk_bound_dev_if) 203 return -EINVAL; 204 } 205 206 if (tp->rx_opt.ts_recent_stamp && 207 !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) { 208 tp->rx_opt.ts_recent = 0; 209 tp->rx_opt.ts_recent_stamp = 0; 210 WRITE_ONCE(tp->write_seq, 0); 211 } 212 213 sk->sk_v6_daddr = usin->sin6_addr; 214 np->flow_label = fl6.flowlabel; 215 216 /* 217 * TCP over IPv4 218 */ 219 220 if (addr_type & IPV6_ADDR_MAPPED) { 221 u32 exthdrlen = icsk->icsk_ext_hdr_len; 222 struct sockaddr_in sin; 223 224 if (ipv6_only_sock(sk)) 225 return -ENETUNREACH; 226 227 sin.sin_family = AF_INET; 228 sin.sin_port = usin->sin6_port; 229 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3]; 230 231 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 232 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped); 233 if (sk_is_mptcp(sk)) 234 mptcpv6_handle_mapped(sk, true); 235 sk->sk_backlog_rcv = tcp_v4_do_rcv; 236 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 237 tp->af_specific = &tcp_sock_ipv6_mapped_specific; 238 #endif 239 240 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin)); 241 242 if (err) { 243 icsk->icsk_ext_hdr_len = exthdrlen; 244 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 245 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific); 246 if (sk_is_mptcp(sk)) 247 mptcpv6_handle_mapped(sk, false); 248 sk->sk_backlog_rcv = tcp_v6_do_rcv; 249 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 250 tp->af_specific = &tcp_sock_ipv6_specific; 251 #endif 252 goto failure; 253 } 254 np->saddr = sk->sk_v6_rcv_saddr; 255 256 return err; 257 } 258 259 if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) 260 saddr = &sk->sk_v6_rcv_saddr; 261 262 fl6.flowi6_proto = IPPROTO_TCP; 263 fl6.daddr = sk->sk_v6_daddr; 264 fl6.saddr = saddr ? *saddr : np->saddr; 265 fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label); 266 fl6.flowi6_oif = sk->sk_bound_dev_if; 267 fl6.flowi6_mark = sk->sk_mark; 268 fl6.fl6_dport = usin->sin6_port; 269 fl6.fl6_sport = inet->inet_sport; 270 if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6.fl6_sport) 271 fl6.flowi6_flags = FLOWI_FLAG_ANY_SPORT; 272 fl6.flowi6_uid = sk->sk_uid; 273 274 opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk)); 275 final_p = fl6_update_dst(&fl6, opt, &final); 276 277 security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6)); 278 279 dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p); 280 if (IS_ERR(dst)) { 281 err = PTR_ERR(dst); 282 goto failure; 283 } 284 285 tp->tcp_usec_ts = dst_tcp_usec_ts(dst); 286 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row; 287 288 if (!saddr) { 289 saddr = &fl6.saddr; 290 291 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6); 292 if (err) 293 goto failure; 294 } 295 296 /* set the source address */ 297 np->saddr = *saddr; 298 inet->inet_rcv_saddr = LOOPBACK4_IPV6; 299 300 sk->sk_gso_type = SKB_GSO_TCPV6; 301 ip6_dst_store(sk, dst, NULL, NULL); 302 303 icsk->icsk_ext_hdr_len = 0; 304 if (opt) 305 icsk->icsk_ext_hdr_len = opt->opt_flen + 306 opt->opt_nflen; 307 308 tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr); 309 310 inet->inet_dport = usin->sin6_port; 311 312 tcp_set_state(sk, TCP_SYN_SENT); 313 err = inet6_hash_connect(tcp_death_row, sk); 314 if (err) 315 goto late_failure; 316 317 sk_set_txhash(sk); 318 319 if (likely(!tp->repair)) { 320 if (!tp->write_seq) 321 WRITE_ONCE(tp->write_seq, 322 secure_tcpv6_seq(np->saddr.s6_addr32, 323 sk->sk_v6_daddr.s6_addr32, 324 inet->inet_sport, 325 inet->inet_dport)); 326 tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, 327 sk->sk_v6_daddr.s6_addr32); 328 } 329 330 if (tcp_fastopen_defer_connect(sk, &err)) 331 return err; 332 if (err) 333 goto late_failure; 334 335 err = tcp_connect(sk); 336 if (err) 337 goto late_failure; 338 339 return 0; 340 341 late_failure: 342 tcp_set_state(sk, TCP_CLOSE); 343 inet_bhash2_reset_saddr(sk); 344 failure: 345 inet->inet_dport = 0; 346 sk->sk_route_caps = 0; 347 return err; 348 } 349 350 static void tcp_v6_mtu_reduced(struct sock *sk) 351 { 352 struct dst_entry *dst; 353 u32 mtu; 354 355 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 356 return; 357 358 mtu = READ_ONCE(tcp_sk(sk)->mtu_info); 359 360 /* Drop requests trying to increase our current mss. 361 * Check done in __ip6_rt_update_pmtu() is too late. 362 */ 363 if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache) 364 return; 365 366 dst = inet6_csk_update_pmtu(sk, mtu); 367 if (!dst) 368 return; 369 370 if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) { 371 tcp_sync_mss(sk, dst_mtu(dst)); 372 tcp_simple_retransmit(sk); 373 } 374 } 375 376 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 377 u8 type, u8 code, int offset, __be32 info) 378 { 379 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; 380 const struct tcphdr *th = (struct tcphdr *)(skb->data+offset); 381 struct net *net = dev_net_rcu(skb->dev); 382 struct request_sock *fastopen; 383 struct ipv6_pinfo *np; 384 struct tcp_sock *tp; 385 __u32 seq, snd_una; 386 struct sock *sk; 387 bool fatal; 388 int err; 389 390 sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 391 &hdr->daddr, th->dest, 392 &hdr->saddr, ntohs(th->source), 393 skb->dev->ifindex, inet6_sdif(skb)); 394 395 if (!sk) { 396 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), 397 ICMP6_MIB_INERRORS); 398 return -ENOENT; 399 } 400 401 if (sk->sk_state == TCP_TIME_WAIT) { 402 /* To increase the counter of ignored icmps for TCP-AO */ 403 tcp_ao_ignore_icmp(sk, AF_INET6, type, code); 404 inet_twsk_put(inet_twsk(sk)); 405 return 0; 406 } 407 seq = ntohl(th->seq); 408 fatal = icmpv6_err_convert(type, code, &err); 409 if (sk->sk_state == TCP_NEW_SYN_RECV) { 410 tcp_req_err(sk, seq, fatal); 411 return 0; 412 } 413 414 if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) { 415 sock_put(sk); 416 return 0; 417 } 418 419 bh_lock_sock(sk); 420 if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG) 421 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS); 422 423 if (sk->sk_state == TCP_CLOSE) 424 goto out; 425 426 if (static_branch_unlikely(&ip6_min_hopcount)) { 427 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 428 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) { 429 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 430 goto out; 431 } 432 } 433 434 tp = tcp_sk(sk); 435 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */ 436 fastopen = rcu_dereference(tp->fastopen_rsk); 437 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una; 438 if (sk->sk_state != TCP_LISTEN && 439 !between(seq, snd_una, tp->snd_nxt)) { 440 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS); 441 goto out; 442 } 443 444 np = tcp_inet6_sk(sk); 445 446 if (type == NDISC_REDIRECT) { 447 if (!sock_owned_by_user(sk)) { 448 struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie); 449 450 if (dst) 451 dst->ops->redirect(dst, sk, skb); 452 } 453 goto out; 454 } 455 456 if (type == ICMPV6_PKT_TOOBIG) { 457 u32 mtu = ntohl(info); 458 459 /* We are not interested in TCP_LISTEN and open_requests 460 * (SYN-ACKs send out by Linux are always <576bytes so 461 * they should go through unfragmented). 462 */ 463 if (sk->sk_state == TCP_LISTEN) 464 goto out; 465 466 if (!ip6_sk_accept_pmtu(sk)) 467 goto out; 468 469 if (mtu < IPV6_MIN_MTU) 470 goto out; 471 472 WRITE_ONCE(tp->mtu_info, mtu); 473 474 if (!sock_owned_by_user(sk)) 475 tcp_v6_mtu_reduced(sk); 476 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, 477 &sk->sk_tsq_flags)) 478 sock_hold(sk); 479 goto out; 480 } 481 482 483 /* Might be for an request_sock */ 484 switch (sk->sk_state) { 485 case TCP_SYN_SENT: 486 case TCP_SYN_RECV: 487 /* Only in fast or simultaneous open. If a fast open socket is 488 * already accepted it is treated as a connected one below. 489 */ 490 if (fastopen && !fastopen->sk) 491 break; 492 493 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th); 494 495 if (!sock_owned_by_user(sk)) 496 tcp_done_with_error(sk, err); 497 else 498 WRITE_ONCE(sk->sk_err_soft, err); 499 goto out; 500 case TCP_LISTEN: 501 break; 502 default: 503 /* check if this ICMP message allows revert of backoff. 504 * (see RFC 6069) 505 */ 506 if (!fastopen && type == ICMPV6_DEST_UNREACH && 507 code == ICMPV6_NOROUTE) 508 tcp_ld_RTO_revert(sk, seq); 509 } 510 511 if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) { 512 WRITE_ONCE(sk->sk_err, err); 513 sk_error_report(sk); 514 } else { 515 WRITE_ONCE(sk->sk_err_soft, err); 516 } 517 out: 518 bh_unlock_sock(sk); 519 sock_put(sk); 520 return 0; 521 } 522 523 524 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 525 struct flowi *fl, 526 struct request_sock *req, 527 struct tcp_fastopen_cookie *foc, 528 enum tcp_synack_type synack_type, 529 struct sk_buff *syn_skb) 530 { 531 struct inet_request_sock *ireq = inet_rsk(req); 532 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 533 struct ipv6_txoptions *opt; 534 struct flowi6 *fl6 = &fl->u.ip6; 535 struct sk_buff *skb; 536 int err = -ENOMEM; 537 u8 tclass; 538 539 /* First, grab a route. */ 540 if (!dst && (dst = inet6_csk_route_req(sk, fl6, req, 541 IPPROTO_TCP)) == NULL) 542 goto done; 543 544 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb); 545 546 if (skb) { 547 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr, 548 &ireq->ir_v6_rmt_addr); 549 550 fl6->daddr = ireq->ir_v6_rmt_addr; 551 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts) 552 fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts)); 553 554 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ? 555 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) | 556 (np->tclass & INET_ECN_MASK) : 557 np->tclass; 558 559 if (!INET_ECN_is_capable(tclass) && 560 tcp_bpf_ca_needs_ecn((struct sock *)req)) 561 tclass |= INET_ECN_ECT_0; 562 563 rcu_read_lock(); 564 opt = ireq->ipv6_opt; 565 if (!opt) 566 opt = rcu_dereference(np->opt); 567 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark), 568 opt, tclass, READ_ONCE(sk->sk_priority)); 569 rcu_read_unlock(); 570 err = net_xmit_eval(err); 571 } 572 573 done: 574 return err; 575 } 576 577 578 static void tcp_v6_reqsk_destructor(struct request_sock *req) 579 { 580 kfree(inet_rsk(req)->ipv6_opt); 581 consume_skb(inet_rsk(req)->pktopts); 582 } 583 584 #ifdef CONFIG_TCP_MD5SIG 585 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk, 586 const struct in6_addr *addr, 587 int l3index) 588 { 589 return tcp_md5_do_lookup(sk, l3index, 590 (union tcp_md5_addr *)addr, AF_INET6); 591 } 592 593 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk, 594 const struct sock *addr_sk) 595 { 596 int l3index; 597 598 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), 599 addr_sk->sk_bound_dev_if); 600 return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr, 601 l3index); 602 } 603 604 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname, 605 sockptr_t optval, int optlen) 606 { 607 struct tcp_md5sig cmd; 608 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr; 609 union tcp_ao_addr *addr; 610 int l3index = 0; 611 u8 prefixlen; 612 bool l3flag; 613 u8 flags; 614 615 if (optlen < sizeof(cmd)) 616 return -EINVAL; 617 618 if (copy_from_sockptr(&cmd, optval, sizeof(cmd))) 619 return -EFAULT; 620 621 if (sin6->sin6_family != AF_INET6) 622 return -EINVAL; 623 624 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 625 l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 626 627 if (optname == TCP_MD5SIG_EXT && 628 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) { 629 prefixlen = cmd.tcpm_prefixlen; 630 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) && 631 prefixlen > 32)) 632 return -EINVAL; 633 } else { 634 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128; 635 } 636 637 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex && 638 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) { 639 struct net_device *dev; 640 641 rcu_read_lock(); 642 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex); 643 if (dev && netif_is_l3_master(dev)) 644 l3index = dev->ifindex; 645 rcu_read_unlock(); 646 647 /* ok to reference set/not set outside of rcu; 648 * right now device MUST be an L3 master 649 */ 650 if (!dev || !l3index) 651 return -EINVAL; 652 } 653 654 if (!cmd.tcpm_keylen) { 655 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) 656 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3], 657 AF_INET, prefixlen, 658 l3index, flags); 659 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr, 660 AF_INET6, prefixlen, l3index, flags); 661 } 662 663 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN) 664 return -EINVAL; 665 666 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) { 667 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3]; 668 669 /* Don't allow keys for peers that have a matching TCP-AO key. 670 * See the comment in tcp_ao_add_cmd() 671 */ 672 if (tcp_ao_required(sk, addr, AF_INET, 673 l3flag ? l3index : -1, false)) 674 return -EKEYREJECTED; 675 return tcp_md5_do_add(sk, addr, 676 AF_INET, prefixlen, l3index, flags, 677 cmd.tcpm_key, cmd.tcpm_keylen); 678 } 679 680 addr = (union tcp_md5_addr *)&sin6->sin6_addr; 681 682 /* Don't allow keys for peers that have a matching TCP-AO key. 683 * See the comment in tcp_ao_add_cmd() 684 */ 685 if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false)) 686 return -EKEYREJECTED; 687 688 return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags, 689 cmd.tcpm_key, cmd.tcpm_keylen); 690 } 691 692 static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp, 693 const struct in6_addr *daddr, 694 const struct in6_addr *saddr, 695 const struct tcphdr *th, int nbytes) 696 { 697 struct tcp6_pseudohdr *bp; 698 struct scatterlist sg; 699 struct tcphdr *_th; 700 701 bp = hp->scratch; 702 /* 1. TCP pseudo-header (RFC2460) */ 703 bp->saddr = *saddr; 704 bp->daddr = *daddr; 705 bp->protocol = cpu_to_be32(IPPROTO_TCP); 706 bp->len = cpu_to_be32(nbytes); 707 708 _th = (struct tcphdr *)(bp + 1); 709 memcpy(_th, th, sizeof(*th)); 710 _th->check = 0; 711 712 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); 713 ahash_request_set_crypt(hp->req, &sg, NULL, 714 sizeof(*bp) + sizeof(*th)); 715 return crypto_ahash_update(hp->req); 716 } 717 718 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key, 719 const struct in6_addr *daddr, struct in6_addr *saddr, 720 const struct tcphdr *th) 721 { 722 struct tcp_sigpool hp; 723 724 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 725 goto clear_hash_nostart; 726 727 if (crypto_ahash_init(hp.req)) 728 goto clear_hash; 729 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) 730 goto clear_hash; 731 if (tcp_md5_hash_key(&hp, key)) 732 goto clear_hash; 733 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 734 if (crypto_ahash_final(hp.req)) 735 goto clear_hash; 736 737 tcp_sigpool_end(&hp); 738 return 0; 739 740 clear_hash: 741 tcp_sigpool_end(&hp); 742 clear_hash_nostart: 743 memset(md5_hash, 0, 16); 744 return 1; 745 } 746 747 static int tcp_v6_md5_hash_skb(char *md5_hash, 748 const struct tcp_md5sig_key *key, 749 const struct sock *sk, 750 const struct sk_buff *skb) 751 { 752 const struct tcphdr *th = tcp_hdr(skb); 753 const struct in6_addr *saddr, *daddr; 754 struct tcp_sigpool hp; 755 756 if (sk) { /* valid for establish/request sockets */ 757 saddr = &sk->sk_v6_rcv_saddr; 758 daddr = &sk->sk_v6_daddr; 759 } else { 760 const struct ipv6hdr *ip6h = ipv6_hdr(skb); 761 saddr = &ip6h->saddr; 762 daddr = &ip6h->daddr; 763 } 764 765 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 766 goto clear_hash_nostart; 767 768 if (crypto_ahash_init(hp.req)) 769 goto clear_hash; 770 771 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) 772 goto clear_hash; 773 if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2)) 774 goto clear_hash; 775 if (tcp_md5_hash_key(&hp, key)) 776 goto clear_hash; 777 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 778 if (crypto_ahash_final(hp.req)) 779 goto clear_hash; 780 781 tcp_sigpool_end(&hp); 782 return 0; 783 784 clear_hash: 785 tcp_sigpool_end(&hp); 786 clear_hash_nostart: 787 memset(md5_hash, 0, 16); 788 return 1; 789 } 790 #endif 791 792 static void tcp_v6_init_req(struct request_sock *req, 793 const struct sock *sk_listener, 794 struct sk_buff *skb, 795 u32 tw_isn) 796 { 797 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 798 struct inet_request_sock *ireq = inet_rsk(req); 799 const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener); 800 801 ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; 802 ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; 803 ireq->ir_rmt_addr = LOOPBACK4_IPV6; 804 ireq->ir_loc_addr = LOOPBACK4_IPV6; 805 806 /* So that link locals have meaning */ 807 if ((!sk_listener->sk_bound_dev_if || l3_slave) && 808 ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL) 809 ireq->ir_iif = tcp_v6_iif(skb); 810 811 if (!tw_isn && 812 (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) || 813 np->rxopt.bits.rxinfo || 814 np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim || 815 np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) { 816 refcount_inc(&skb->users); 817 ireq->pktopts = skb; 818 } 819 } 820 821 static struct dst_entry *tcp_v6_route_req(const struct sock *sk, 822 struct sk_buff *skb, 823 struct flowi *fl, 824 struct request_sock *req, 825 u32 tw_isn) 826 { 827 tcp_v6_init_req(req, sk, skb, tw_isn); 828 829 if (security_inet_conn_request(sk, skb, req)) 830 return NULL; 831 832 return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP); 833 } 834 835 struct request_sock_ops tcp6_request_sock_ops __read_mostly = { 836 .family = AF_INET6, 837 .obj_size = sizeof(struct tcp6_request_sock), 838 .rtx_syn_ack = tcp_rtx_synack, 839 .send_ack = tcp_v6_reqsk_send_ack, 840 .destructor = tcp_v6_reqsk_destructor, 841 .send_reset = tcp_v6_send_reset, 842 .syn_ack_timeout = tcp_syn_ack_timeout, 843 }; 844 845 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { 846 .mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - 847 sizeof(struct ipv6hdr), 848 #ifdef CONFIG_TCP_MD5SIG 849 .req_md5_lookup = tcp_v6_md5_lookup, 850 .calc_md5_hash = tcp_v6_md5_hash_skb, 851 #endif 852 #ifdef CONFIG_TCP_AO 853 .ao_lookup = tcp_v6_ao_lookup_rsk, 854 .ao_calc_key = tcp_v6_ao_calc_key_rsk, 855 .ao_synack_hash = tcp_v6_ao_synack_hash, 856 #endif 857 #ifdef CONFIG_SYN_COOKIES 858 .cookie_init_seq = cookie_v6_init_sequence, 859 #endif 860 .route_req = tcp_v6_route_req, 861 .init_seq = tcp_v6_init_seq, 862 .init_ts_off = tcp_v6_init_ts_off, 863 .send_synack = tcp_v6_send_synack, 864 }; 865 866 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq, 867 u32 ack, u32 win, u32 tsval, u32 tsecr, 868 int oif, int rst, u8 tclass, __be32 label, 869 u32 priority, u32 txhash, struct tcp_key *key) 870 { 871 struct net *net = sk ? sock_net(sk) : dev_net_rcu(skb_dst(skb)->dev); 872 unsigned int tot_len = sizeof(struct tcphdr); 873 struct sock *ctl_sk = net->ipv6.tcp_sk; 874 const struct tcphdr *th = tcp_hdr(skb); 875 __be32 mrst = 0, *topt; 876 struct dst_entry *dst; 877 struct sk_buff *buff; 878 struct tcphdr *t1; 879 struct flowi6 fl6; 880 u32 mark = 0; 881 882 if (tsecr) 883 tot_len += TCPOLEN_TSTAMP_ALIGNED; 884 if (tcp_key_is_md5(key)) 885 tot_len += TCPOLEN_MD5SIG_ALIGNED; 886 if (tcp_key_is_ao(key)) 887 tot_len += tcp_ao_len_aligned(key->ao_key); 888 889 #ifdef CONFIG_MPTCP 890 if (rst && !tcp_key_is_md5(key)) { 891 mrst = mptcp_reset_option(skb); 892 893 if (mrst) 894 tot_len += sizeof(__be32); 895 } 896 #endif 897 898 buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 899 if (!buff) 900 return; 901 902 skb_reserve(buff, MAX_TCP_HEADER); 903 904 t1 = skb_push(buff, tot_len); 905 skb_reset_transport_header(buff); 906 907 /* Swap the send and the receive. */ 908 memset(t1, 0, sizeof(*t1)); 909 t1->dest = th->source; 910 t1->source = th->dest; 911 t1->doff = tot_len / 4; 912 t1->seq = htonl(seq); 913 t1->ack_seq = htonl(ack); 914 t1->ack = !rst || !th->ack; 915 t1->rst = rst; 916 t1->window = htons(win); 917 918 topt = (__be32 *)(t1 + 1); 919 920 if (tsecr) { 921 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 922 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP); 923 *topt++ = htonl(tsval); 924 *topt++ = htonl(tsecr); 925 } 926 927 if (mrst) 928 *topt++ = mrst; 929 930 #ifdef CONFIG_TCP_MD5SIG 931 if (tcp_key_is_md5(key)) { 932 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 933 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG); 934 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key, 935 &ipv6_hdr(skb)->saddr, 936 &ipv6_hdr(skb)->daddr, t1); 937 } 938 #endif 939 #ifdef CONFIG_TCP_AO 940 if (tcp_key_is_ao(key)) { 941 *topt++ = htonl((TCPOPT_AO << 24) | 942 (tcp_ao_len(key->ao_key) << 16) | 943 (key->ao_key->sndid << 8) | 944 (key->rcv_next)); 945 946 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key, 947 key->traffic_key, 948 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr, 949 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr, 950 t1, key->sne); 951 } 952 #endif 953 954 memset(&fl6, 0, sizeof(fl6)); 955 fl6.daddr = ipv6_hdr(skb)->saddr; 956 fl6.saddr = ipv6_hdr(skb)->daddr; 957 fl6.flowlabel = label; 958 959 buff->ip_summed = CHECKSUM_PARTIAL; 960 961 __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr); 962 963 fl6.flowi6_proto = IPPROTO_TCP; 964 if (rt6_need_strict(&fl6.daddr) && !oif) 965 fl6.flowi6_oif = tcp_v6_iif(skb); 966 else { 967 if (!oif && netif_index_is_l3_master(net, skb->skb_iif)) 968 oif = skb->skb_iif; 969 970 fl6.flowi6_oif = oif; 971 } 972 973 if (sk) { 974 /* unconstify the socket only to attach it to buff with care. */ 975 skb_set_owner_edemux(buff, (struct sock *)sk); 976 977 if (sk->sk_state == TCP_TIME_WAIT) 978 mark = inet_twsk(sk)->tw_mark; 979 else 980 mark = READ_ONCE(sk->sk_mark); 981 skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC); 982 } 983 if (txhash) { 984 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */ 985 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4); 986 } 987 fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark; 988 fl6.fl6_dport = t1->dest; 989 fl6.fl6_sport = t1->source; 990 fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL); 991 security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6)); 992 993 /* Pass a socket to ip6_dst_lookup either it is for RST 994 * Underlying function will use this to retrieve the network 995 * namespace 996 */ 997 if (sk && sk->sk_state != TCP_TIME_WAIT) 998 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/ 999 else 1000 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL); 1001 if (!IS_ERR(dst)) { 1002 skb_dst_set(buff, dst); 1003 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL, 1004 tclass, priority); 1005 TCP_INC_STATS(net, TCP_MIB_OUTSEGS); 1006 if (rst) 1007 TCP_INC_STATS(net, TCP_MIB_OUTRSTS); 1008 return; 1009 } 1010 1011 kfree_skb(buff); 1012 } 1013 1014 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 1015 enum sk_rst_reason reason) 1016 { 1017 const struct tcphdr *th = tcp_hdr(skb); 1018 struct ipv6hdr *ipv6h = ipv6_hdr(skb); 1019 const __u8 *md5_hash_location = NULL; 1020 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1021 bool allocated_traffic_key = false; 1022 #endif 1023 const struct tcp_ao_hdr *aoh; 1024 struct tcp_key key = {}; 1025 u32 seq = 0, ack_seq = 0; 1026 __be32 label = 0; 1027 u32 priority = 0; 1028 struct net *net; 1029 u32 txhash = 0; 1030 int oif = 0; 1031 #ifdef CONFIG_TCP_MD5SIG 1032 unsigned char newhash[16]; 1033 int genhash; 1034 struct sock *sk1 = NULL; 1035 #endif 1036 1037 if (th->rst) 1038 return; 1039 1040 /* If sk not NULL, it means we did a successful lookup and incoming 1041 * route had to be correct. prequeue might have dropped our dst. 1042 */ 1043 if (!sk && !ipv6_unicast_destination(skb)) 1044 return; 1045 1046 net = sk ? sock_net(sk) : dev_net_rcu(skb_dst(skb)->dev); 1047 /* Invalid TCP option size or twice included auth */ 1048 if (tcp_parse_auth_options(th, &md5_hash_location, &aoh)) 1049 return; 1050 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1051 rcu_read_lock(); 1052 #endif 1053 #ifdef CONFIG_TCP_MD5SIG 1054 if (sk && sk_fullsock(sk)) { 1055 int l3index; 1056 1057 /* sdif set, means packet ingressed via a device 1058 * in an L3 domain and inet_iif is set to it. 1059 */ 1060 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1061 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index); 1062 if (key.md5_key) 1063 key.type = TCP_KEY_MD5; 1064 } else if (md5_hash_location) { 1065 int dif = tcp_v6_iif_l3_slave(skb); 1066 int sdif = tcp_v6_sdif(skb); 1067 int l3index; 1068 1069 /* 1070 * active side is lost. Try to find listening socket through 1071 * source port, and then find md5 key through listening socket. 1072 * we are not loose security here: 1073 * Incoming packet is checked with md5 hash with finding key, 1074 * no RST generated if md5 hash doesn't match. 1075 */ 1076 sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, 1077 NULL, 0, &ipv6h->saddr, th->source, 1078 &ipv6h->daddr, ntohs(th->source), 1079 dif, sdif); 1080 if (!sk1) 1081 goto out; 1082 1083 /* sdif set, means packet ingressed via a device 1084 * in an L3 domain and dif is set to it. 1085 */ 1086 l3index = tcp_v6_sdif(skb) ? dif : 0; 1087 1088 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index); 1089 if (!key.md5_key) 1090 goto out; 1091 key.type = TCP_KEY_MD5; 1092 1093 genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb); 1094 if (genhash || memcmp(md5_hash_location, newhash, 16) != 0) 1095 goto out; 1096 } 1097 #endif 1098 1099 if (th->ack) 1100 seq = ntohl(th->ack_seq); 1101 else 1102 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len - 1103 (th->doff << 2); 1104 1105 #ifdef CONFIG_TCP_AO 1106 if (aoh) { 1107 int l3index; 1108 1109 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1110 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq, 1111 &key.ao_key, &key.traffic_key, 1112 &allocated_traffic_key, 1113 &key.rcv_next, &key.sne)) 1114 goto out; 1115 key.type = TCP_KEY_AO; 1116 } 1117 #endif 1118 1119 if (sk) { 1120 oif = sk->sk_bound_dev_if; 1121 if (sk_fullsock(sk)) { 1122 if (inet6_test_bit(REPFLOW, sk)) 1123 label = ip6_flowlabel(ipv6h); 1124 priority = READ_ONCE(sk->sk_priority); 1125 txhash = sk->sk_txhash; 1126 } 1127 if (sk->sk_state == TCP_TIME_WAIT) { 1128 label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel); 1129 priority = inet_twsk(sk)->tw_priority; 1130 txhash = inet_twsk(sk)->tw_txhash; 1131 } 1132 } else { 1133 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) 1134 label = ip6_flowlabel(ipv6h); 1135 } 1136 1137 trace_tcp_send_reset(sk, skb, reason); 1138 1139 tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1, 1140 ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK, 1141 label, priority, txhash, 1142 &key); 1143 1144 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1145 out: 1146 if (allocated_traffic_key) 1147 kfree(key.traffic_key); 1148 rcu_read_unlock(); 1149 #endif 1150 } 1151 1152 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq, 1153 u32 ack, u32 win, u32 tsval, u32 tsecr, int oif, 1154 struct tcp_key *key, u8 tclass, 1155 __be32 label, u32 priority, u32 txhash) 1156 { 1157 tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0, 1158 tclass, label, priority, txhash, key); 1159 } 1160 1161 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb, 1162 enum tcp_tw_status tw_status) 1163 { 1164 struct inet_timewait_sock *tw = inet_twsk(sk); 1165 struct tcp_timewait_sock *tcptw = tcp_twsk(sk); 1166 u8 tclass = tw->tw_tclass; 1167 struct tcp_key key = {}; 1168 1169 if (tw_status == TCP_TW_ACK_OOW) 1170 tclass &= ~INET_ECN_MASK; 1171 #ifdef CONFIG_TCP_AO 1172 struct tcp_ao_info *ao_info; 1173 1174 if (static_branch_unlikely(&tcp_ao_needed.key)) { 1175 1176 /* FIXME: the segment to-be-acked is not verified yet */ 1177 ao_info = rcu_dereference(tcptw->ao_info); 1178 if (ao_info) { 1179 const struct tcp_ao_hdr *aoh; 1180 1181 /* Invalid TCP option size or twice included auth */ 1182 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1183 goto out; 1184 if (aoh) 1185 key.ao_key = tcp_ao_established_key(sk, ao_info, 1186 aoh->rnext_keyid, -1); 1187 } 1188 } 1189 if (key.ao_key) { 1190 struct tcp_ao_key *rnext_key; 1191 1192 key.traffic_key = snd_other_key(key.ao_key); 1193 /* rcv_next switches to our rcv_next */ 1194 rnext_key = READ_ONCE(ao_info->rnext_key); 1195 key.rcv_next = rnext_key->rcvid; 1196 key.sne = READ_ONCE(ao_info->snd_sne); 1197 key.type = TCP_KEY_AO; 1198 #else 1199 if (0) { 1200 #endif 1201 #ifdef CONFIG_TCP_MD5SIG 1202 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1203 key.md5_key = tcp_twsk_md5_key(tcptw); 1204 if (key.md5_key) 1205 key.type = TCP_KEY_MD5; 1206 #endif 1207 } 1208 1209 tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, 1210 READ_ONCE(tcptw->tw_rcv_nxt), 1211 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale, 1212 tcp_tw_tsval(tcptw), 1213 READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if, 1214 &key, tclass, cpu_to_be32(tw->tw_flowlabel), 1215 tw->tw_priority, tw->tw_txhash); 1216 1217 #ifdef CONFIG_TCP_AO 1218 out: 1219 #endif 1220 inet_twsk_put(tw); 1221 } 1222 1223 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 1224 struct request_sock *req) 1225 { 1226 struct tcp_key key = {}; 1227 1228 #ifdef CONFIG_TCP_AO 1229 if (static_branch_unlikely(&tcp_ao_needed.key) && 1230 tcp_rsk_used_ao(req)) { 1231 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr; 1232 const struct tcp_ao_hdr *aoh; 1233 int l3index; 1234 1235 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1236 /* Invalid TCP option size or twice included auth */ 1237 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1238 return; 1239 if (!aoh) 1240 return; 1241 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1242 (union tcp_ao_addr *)addr, 1243 AF_INET6, aoh->rnext_keyid, -1); 1244 if (unlikely(!key.ao_key)) { 1245 /* Send ACK with any matching MKT for the peer */ 1246 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1247 (union tcp_ao_addr *)addr, 1248 AF_INET6, -1, -1); 1249 /* Matching key disappeared (user removed the key?) 1250 * let the handshake timeout. 1251 */ 1252 if (!key.ao_key) { 1253 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n", 1254 addr, 1255 ntohs(tcp_hdr(skb)->source), 1256 &ipv6_hdr(skb)->daddr, 1257 ntohs(tcp_hdr(skb)->dest)); 1258 return; 1259 } 1260 } 1261 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC); 1262 if (!key.traffic_key) 1263 return; 1264 1265 key.type = TCP_KEY_AO; 1266 key.rcv_next = aoh->keyid; 1267 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req); 1268 #else 1269 if (0) { 1270 #endif 1271 #ifdef CONFIG_TCP_MD5SIG 1272 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1273 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1274 1275 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, 1276 l3index); 1277 if (key.md5_key) 1278 key.type = TCP_KEY_MD5; 1279 #endif 1280 } 1281 1282 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV 1283 * sk->sk_state == TCP_SYN_RECV -> for Fast Open. 1284 */ 1285 tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ? 1286 tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt, 1287 tcp_rsk(req)->rcv_nxt, 1288 tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale, 1289 tcp_rsk_tsval(tcp_rsk(req)), 1290 req->ts_recent, sk->sk_bound_dev_if, 1291 &key, ipv6_get_dsfield(ipv6_hdr(skb)) & ~INET_ECN_MASK, 1292 0, 1293 READ_ONCE(sk->sk_priority), 1294 READ_ONCE(tcp_rsk(req)->txhash)); 1295 if (tcp_key_is_ao(&key)) 1296 kfree(key.traffic_key); 1297 } 1298 1299 1300 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb) 1301 { 1302 #ifdef CONFIG_SYN_COOKIES 1303 const struct tcphdr *th = tcp_hdr(skb); 1304 1305 if (!th->syn) 1306 sk = cookie_v6_check(sk, skb); 1307 #endif 1308 return sk; 1309 } 1310 1311 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 1312 struct tcphdr *th, u32 *cookie) 1313 { 1314 u16 mss = 0; 1315 #ifdef CONFIG_SYN_COOKIES 1316 mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops, 1317 &tcp_request_sock_ipv6_ops, sk, th); 1318 if (mss) { 1319 *cookie = __cookie_v6_init_sequence(iph, th, &mss); 1320 tcp_synq_overflow(sk); 1321 } 1322 #endif 1323 return mss; 1324 } 1325 1326 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb) 1327 { 1328 if (skb->protocol == htons(ETH_P_IP)) 1329 return tcp_v4_conn_request(sk, skb); 1330 1331 if (!ipv6_unicast_destination(skb)) 1332 goto drop; 1333 1334 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 1335 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 1336 return 0; 1337 } 1338 1339 return tcp_conn_request(&tcp6_request_sock_ops, 1340 &tcp_request_sock_ipv6_ops, sk, skb); 1341 1342 drop: 1343 tcp_listendrop(sk); 1344 return 0; /* don't send reset */ 1345 } 1346 1347 static void tcp_v6_restore_cb(struct sk_buff *skb) 1348 { 1349 /* We need to move header back to the beginning if xfrm6_policy_check() 1350 * and tcp_v6_fill_cb() are going to be called again. 1351 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there. 1352 */ 1353 memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6, 1354 sizeof(struct inet6_skb_parm)); 1355 } 1356 1357 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 1358 struct request_sock *req, 1359 struct dst_entry *dst, 1360 struct request_sock *req_unhash, 1361 bool *own_req) 1362 { 1363 struct inet_request_sock *ireq; 1364 struct ipv6_pinfo *newnp; 1365 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1366 struct ipv6_txoptions *opt; 1367 struct inet_sock *newinet; 1368 bool found_dup_sk = false; 1369 struct tcp_sock *newtp; 1370 struct sock *newsk; 1371 #ifdef CONFIG_TCP_MD5SIG 1372 struct tcp_md5sig_key *key; 1373 int l3index; 1374 #endif 1375 struct flowi6 fl6; 1376 1377 if (skb->protocol == htons(ETH_P_IP)) { 1378 /* 1379 * v6 mapped 1380 */ 1381 1382 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst, 1383 req_unhash, own_req); 1384 1385 if (!newsk) 1386 return NULL; 1387 1388 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1389 1390 newnp = tcp_inet6_sk(newsk); 1391 newtp = tcp_sk(newsk); 1392 1393 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1394 1395 newnp->saddr = newsk->sk_v6_rcv_saddr; 1396 1397 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; 1398 if (sk_is_mptcp(newsk)) 1399 mptcpv6_handle_mapped(newsk, true); 1400 newsk->sk_backlog_rcv = tcp_v4_do_rcv; 1401 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1402 newtp->af_specific = &tcp_sock_ipv6_mapped_specific; 1403 #endif 1404 1405 newnp->ipv6_mc_list = NULL; 1406 newnp->ipv6_ac_list = NULL; 1407 newnp->ipv6_fl_list = NULL; 1408 newnp->pktoptions = NULL; 1409 newnp->opt = NULL; 1410 newnp->mcast_oif = inet_iif(skb); 1411 newnp->mcast_hops = ip_hdr(skb)->ttl; 1412 newnp->rcv_flowinfo = 0; 1413 if (inet6_test_bit(REPFLOW, sk)) 1414 newnp->flow_label = 0; 1415 1416 /* 1417 * No need to charge this sock to the relevant IPv6 refcnt debug socks count 1418 * here, tcp_create_openreq_child now does this for us, see the comment in 1419 * that function for the gory details. -acme 1420 */ 1421 1422 /* It is tricky place. Until this moment IPv4 tcp 1423 worked with IPv6 icsk.icsk_af_ops. 1424 Sync it now. 1425 */ 1426 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie); 1427 1428 return newsk; 1429 } 1430 1431 ireq = inet_rsk(req); 1432 1433 if (sk_acceptq_is_full(sk)) 1434 goto out_overflow; 1435 1436 if (!dst) { 1437 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP); 1438 if (!dst) 1439 goto out; 1440 } 1441 1442 newsk = tcp_create_openreq_child(sk, req, skb); 1443 if (!newsk) 1444 goto out_nonewsk; 1445 1446 /* 1447 * No need to charge this sock to the relevant IPv6 refcnt debug socks 1448 * count here, tcp_create_openreq_child now does this for us, see the 1449 * comment in that function for the gory details. -acme 1450 */ 1451 1452 newsk->sk_gso_type = SKB_GSO_TCPV6; 1453 inet6_sk_rx_dst_set(newsk, skb); 1454 1455 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1456 1457 newtp = tcp_sk(newsk); 1458 newinet = inet_sk(newsk); 1459 newnp = tcp_inet6_sk(newsk); 1460 1461 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1462 1463 ip6_dst_store(newsk, dst, NULL, NULL); 1464 1465 newnp->saddr = ireq->ir_v6_loc_addr; 1466 1467 /* Now IPv6 options... 1468 1469 First: no IPv4 options. 1470 */ 1471 newinet->inet_opt = NULL; 1472 newnp->ipv6_mc_list = NULL; 1473 newnp->ipv6_ac_list = NULL; 1474 newnp->ipv6_fl_list = NULL; 1475 1476 /* Clone RX bits */ 1477 newnp->rxopt.all = np->rxopt.all; 1478 1479 newnp->pktoptions = NULL; 1480 newnp->opt = NULL; 1481 newnp->mcast_oif = tcp_v6_iif(skb); 1482 newnp->mcast_hops = ipv6_hdr(skb)->hop_limit; 1483 newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb)); 1484 if (inet6_test_bit(REPFLOW, sk)) 1485 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb)); 1486 1487 /* Set ToS of the new socket based upon the value of incoming SYN. 1488 * ECT bits are set later in tcp_init_transfer(). 1489 */ 1490 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)) 1491 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK; 1492 1493 /* Clone native IPv6 options from listening socket (if any) 1494 1495 Yes, keeping reference count would be much more clever, 1496 but we make one more one thing there: reattach optmem 1497 to newsk. 1498 */ 1499 opt = ireq->ipv6_opt; 1500 if (!opt) 1501 opt = rcu_dereference(np->opt); 1502 if (opt) { 1503 opt = ipv6_dup_options(newsk, opt); 1504 RCU_INIT_POINTER(newnp->opt, opt); 1505 } 1506 inet_csk(newsk)->icsk_ext_hdr_len = 0; 1507 if (opt) 1508 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen + 1509 opt->opt_flen; 1510 1511 tcp_ca_openreq_child(newsk, dst); 1512 1513 tcp_sync_mss(newsk, dst_mtu(dst)); 1514 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); 1515 1516 tcp_initialize_rcv_mss(newsk); 1517 1518 #ifdef CONFIG_TCP_MD5SIG 1519 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif); 1520 1521 if (!tcp_rsk_used_ao(req)) { 1522 /* Copy over the MD5 key from the original socket */ 1523 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index); 1524 if (key) { 1525 const union tcp_md5_addr *addr; 1526 1527 addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr; 1528 if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) { 1529 inet_csk_prepare_forced_close(newsk); 1530 tcp_done(newsk); 1531 goto out; 1532 } 1533 } 1534 } 1535 #endif 1536 #ifdef CONFIG_TCP_AO 1537 /* Copy over tcp_ao_info if any */ 1538 if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6)) 1539 goto out; /* OOM */ 1540 #endif 1541 1542 if (__inet_inherit_port(sk, newsk) < 0) { 1543 inet_csk_prepare_forced_close(newsk); 1544 tcp_done(newsk); 1545 goto out; 1546 } 1547 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash), 1548 &found_dup_sk); 1549 if (*own_req) { 1550 tcp_move_syn(newtp, req); 1551 1552 /* Clone pktoptions received with SYN, if we own the req */ 1553 if (ireq->pktopts) { 1554 newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk); 1555 consume_skb(ireq->pktopts); 1556 ireq->pktopts = NULL; 1557 if (newnp->pktoptions) 1558 tcp_v6_restore_cb(newnp->pktoptions); 1559 } 1560 } else { 1561 if (!req_unhash && found_dup_sk) { 1562 /* This code path should only be executed in the 1563 * syncookie case only 1564 */ 1565 bh_unlock_sock(newsk); 1566 sock_put(newsk); 1567 newsk = NULL; 1568 } 1569 } 1570 1571 return newsk; 1572 1573 out_overflow: 1574 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS); 1575 out_nonewsk: 1576 dst_release(dst); 1577 out: 1578 tcp_listendrop(sk); 1579 return NULL; 1580 } 1581 1582 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 1583 u32)); 1584 /* The socket must have it's spinlock held when we get 1585 * here, unless it is a TCP_LISTEN socket. 1586 * 1587 * We have a potential double-lock case here, so even when 1588 * doing backlog processing we use the BH locking scheme. 1589 * This is because we cannot sleep with the original spinlock 1590 * held. 1591 */ 1592 INDIRECT_CALLABLE_SCOPE 1593 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) 1594 { 1595 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1596 struct sk_buff *opt_skb = NULL; 1597 enum skb_drop_reason reason; 1598 struct tcp_sock *tp; 1599 1600 /* Imagine: socket is IPv6. IPv4 packet arrives, 1601 goes to IPv4 receive handler and backlogged. 1602 From backlog it always goes here. Kerboom... 1603 Fortunately, tcp_rcv_established and rcv_established 1604 handle them correctly, but it is not case with 1605 tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK 1606 */ 1607 1608 if (skb->protocol == htons(ETH_P_IP)) 1609 return tcp_v4_do_rcv(sk, skb); 1610 1611 /* 1612 * socket locking is here for SMP purposes as backlog rcv 1613 * is currently called with bh processing disabled. 1614 */ 1615 1616 /* Do Stevens' IPV6_PKTOPTIONS. 1617 1618 Yes, guys, it is the only place in our code, where we 1619 may make it not affecting IPv4. 1620 The rest of code is protocol independent, 1621 and I do not like idea to uglify IPv4. 1622 1623 Actually, all the idea behind IPV6_PKTOPTIONS 1624 looks not very well thought. For now we latch 1625 options, received in the last packet, enqueued 1626 by tcp. Feel free to propose better solution. 1627 --ANK (980728) 1628 */ 1629 if (np->rxopt.all && sk->sk_state != TCP_LISTEN) 1630 opt_skb = skb_clone_and_charge_r(skb, sk); 1631 1632 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */ 1633 struct dst_entry *dst; 1634 1635 dst = rcu_dereference_protected(sk->sk_rx_dst, 1636 lockdep_sock_is_held(sk)); 1637 1638 sock_rps_save_rxhash(sk, skb); 1639 sk_mark_napi_id(sk, skb); 1640 if (dst) { 1641 if (sk->sk_rx_dst_ifindex != skb->skb_iif || 1642 INDIRECT_CALL_1(dst->ops->check, ip6_dst_check, 1643 dst, sk->sk_rx_dst_cookie) == NULL) { 1644 RCU_INIT_POINTER(sk->sk_rx_dst, NULL); 1645 dst_release(dst); 1646 } 1647 } 1648 1649 tcp_rcv_established(sk, skb); 1650 if (opt_skb) 1651 goto ipv6_pktoptions; 1652 return 0; 1653 } 1654 1655 if (tcp_checksum_complete(skb)) 1656 goto csum_err; 1657 1658 if (sk->sk_state == TCP_LISTEN) { 1659 struct sock *nsk = tcp_v6_cookie_check(sk, skb); 1660 1661 if (nsk != sk) { 1662 if (nsk) { 1663 reason = tcp_child_process(sk, nsk, skb); 1664 if (reason) 1665 goto reset; 1666 } 1667 return 0; 1668 } 1669 } else 1670 sock_rps_save_rxhash(sk, skb); 1671 1672 reason = tcp_rcv_state_process(sk, skb); 1673 if (reason) 1674 goto reset; 1675 if (opt_skb) 1676 goto ipv6_pktoptions; 1677 return 0; 1678 1679 reset: 1680 tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason)); 1681 discard: 1682 if (opt_skb) 1683 __kfree_skb(opt_skb); 1684 sk_skb_reason_drop(sk, skb, reason); 1685 return 0; 1686 csum_err: 1687 reason = SKB_DROP_REASON_TCP_CSUM; 1688 trace_tcp_bad_csum(skb); 1689 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS); 1690 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS); 1691 goto discard; 1692 1693 1694 ipv6_pktoptions: 1695 /* Do you ask, what is it? 1696 1697 1. skb was enqueued by tcp. 1698 2. skb is added to tail of read queue, rather than out of order. 1699 3. socket is not in passive state. 1700 4. Finally, it really contains options, which user wants to receive. 1701 */ 1702 tp = tcp_sk(sk); 1703 if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt && 1704 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 1705 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo) 1706 WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb)); 1707 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim) 1708 WRITE_ONCE(np->mcast_hops, 1709 ipv6_hdr(opt_skb)->hop_limit); 1710 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass) 1711 np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb)); 1712 if (inet6_test_bit(REPFLOW, sk)) 1713 np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb)); 1714 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) { 1715 tcp_v6_restore_cb(opt_skb); 1716 opt_skb = xchg(&np->pktoptions, opt_skb); 1717 } else { 1718 __kfree_skb(opt_skb); 1719 opt_skb = xchg(&np->pktoptions, NULL); 1720 } 1721 } 1722 1723 consume_skb(opt_skb); 1724 return 0; 1725 } 1726 1727 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr, 1728 const struct tcphdr *th) 1729 { 1730 /* This is tricky: we move IP6CB at its correct location into 1731 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because 1732 * _decode_session6() uses IP6CB(). 1733 * barrier() makes sure compiler won't play aliasing games. 1734 */ 1735 memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb), 1736 sizeof(struct inet6_skb_parm)); 1737 barrier(); 1738 1739 TCP_SKB_CB(skb)->seq = ntohl(th->seq); 1740 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin + 1741 skb->len - th->doff*4); 1742 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq); 1743 TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th); 1744 TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr); 1745 TCP_SKB_CB(skb)->sacked = 0; 1746 TCP_SKB_CB(skb)->has_rxtstamp = 1747 skb->tstamp || skb_hwtstamps(skb)->hwtstamp; 1748 } 1749 1750 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb) 1751 { 1752 struct net *net = dev_net_rcu(skb->dev); 1753 enum skb_drop_reason drop_reason; 1754 enum tcp_tw_status tw_status; 1755 int sdif = inet6_sdif(skb); 1756 int dif = inet6_iif(skb); 1757 const struct tcphdr *th; 1758 const struct ipv6hdr *hdr; 1759 struct sock *sk = NULL; 1760 bool refcounted; 1761 int ret; 1762 u32 isn; 1763 1764 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 1765 if (skb->pkt_type != PACKET_HOST) 1766 goto discard_it; 1767 1768 /* 1769 * Count it even if it's bad. 1770 */ 1771 __TCP_INC_STATS(net, TCP_MIB_INSEGS); 1772 1773 if (!pskb_may_pull(skb, sizeof(struct tcphdr))) 1774 goto discard_it; 1775 1776 th = (const struct tcphdr *)skb->data; 1777 1778 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) { 1779 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL; 1780 goto bad_packet; 1781 } 1782 if (!pskb_may_pull(skb, th->doff*4)) 1783 goto discard_it; 1784 1785 if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo)) 1786 goto csum_error; 1787 1788 th = (const struct tcphdr *)skb->data; 1789 hdr = ipv6_hdr(skb); 1790 1791 lookup: 1792 sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th), 1793 th->source, th->dest, inet6_iif(skb), sdif, 1794 &refcounted); 1795 if (!sk) 1796 goto no_tcp_socket; 1797 1798 if (sk->sk_state == TCP_TIME_WAIT) 1799 goto do_time_wait; 1800 1801 if (sk->sk_state == TCP_NEW_SYN_RECV) { 1802 struct request_sock *req = inet_reqsk(sk); 1803 bool req_stolen = false; 1804 struct sock *nsk; 1805 1806 sk = req->rsk_listener; 1807 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) 1808 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1809 else 1810 drop_reason = tcp_inbound_hash(sk, req, skb, 1811 &hdr->saddr, &hdr->daddr, 1812 AF_INET6, dif, sdif); 1813 if (drop_reason) { 1814 sk_drops_add(sk, skb); 1815 reqsk_put(req); 1816 goto discard_it; 1817 } 1818 if (tcp_checksum_complete(skb)) { 1819 reqsk_put(req); 1820 goto csum_error; 1821 } 1822 if (unlikely(sk->sk_state != TCP_LISTEN)) { 1823 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb); 1824 if (!nsk) { 1825 inet_csk_reqsk_queue_drop_and_put(sk, req); 1826 goto lookup; 1827 } 1828 sk = nsk; 1829 /* reuseport_migrate_sock() has already held one sk_refcnt 1830 * before returning. 1831 */ 1832 } else { 1833 sock_hold(sk); 1834 } 1835 refcounted = true; 1836 nsk = NULL; 1837 if (!tcp_filter(sk, skb)) { 1838 th = (const struct tcphdr *)skb->data; 1839 hdr = ipv6_hdr(skb); 1840 tcp_v6_fill_cb(skb, hdr, th); 1841 nsk = tcp_check_req(sk, skb, req, false, &req_stolen, 1842 &drop_reason); 1843 } else { 1844 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 1845 } 1846 if (!nsk) { 1847 reqsk_put(req); 1848 if (req_stolen) { 1849 /* Another cpu got exclusive access to req 1850 * and created a full blown socket. 1851 * Try to feed this packet to this socket 1852 * instead of discarding it. 1853 */ 1854 tcp_v6_restore_cb(skb); 1855 sock_put(sk); 1856 goto lookup; 1857 } 1858 goto discard_and_relse; 1859 } 1860 nf_reset_ct(skb); 1861 if (nsk == sk) { 1862 reqsk_put(req); 1863 tcp_v6_restore_cb(skb); 1864 } else { 1865 drop_reason = tcp_child_process(sk, nsk, skb); 1866 if (drop_reason) { 1867 enum sk_rst_reason rst_reason; 1868 1869 rst_reason = sk_rst_convert_drop_reason(drop_reason); 1870 tcp_v6_send_reset(nsk, skb, rst_reason); 1871 goto discard_and_relse; 1872 } 1873 sock_put(sk); 1874 return 0; 1875 } 1876 } 1877 1878 process: 1879 if (static_branch_unlikely(&ip6_min_hopcount)) { 1880 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 1881 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) { 1882 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 1883 drop_reason = SKB_DROP_REASON_TCP_MINTTL; 1884 goto discard_and_relse; 1885 } 1886 } 1887 1888 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) { 1889 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1890 goto discard_and_relse; 1891 } 1892 1893 drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr, 1894 AF_INET6, dif, sdif); 1895 if (drop_reason) 1896 goto discard_and_relse; 1897 1898 nf_reset_ct(skb); 1899 1900 if (tcp_filter(sk, skb)) { 1901 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 1902 goto discard_and_relse; 1903 } 1904 th = (const struct tcphdr *)skb->data; 1905 hdr = ipv6_hdr(skb); 1906 tcp_v6_fill_cb(skb, hdr, th); 1907 1908 skb->dev = NULL; 1909 1910 if (sk->sk_state == TCP_LISTEN) { 1911 ret = tcp_v6_do_rcv(sk, skb); 1912 goto put_and_return; 1913 } 1914 1915 sk_incoming_cpu_update(sk); 1916 1917 bh_lock_sock_nested(sk); 1918 tcp_segs_in(tcp_sk(sk), skb); 1919 ret = 0; 1920 if (!sock_owned_by_user(sk)) { 1921 ret = tcp_v6_do_rcv(sk, skb); 1922 } else { 1923 if (tcp_add_backlog(sk, skb, &drop_reason)) 1924 goto discard_and_relse; 1925 } 1926 bh_unlock_sock(sk); 1927 put_and_return: 1928 if (refcounted) 1929 sock_put(sk); 1930 return ret ? -1 : 0; 1931 1932 no_tcp_socket: 1933 drop_reason = SKB_DROP_REASON_NO_SOCKET; 1934 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) 1935 goto discard_it; 1936 1937 tcp_v6_fill_cb(skb, hdr, th); 1938 1939 if (tcp_checksum_complete(skb)) { 1940 csum_error: 1941 drop_reason = SKB_DROP_REASON_TCP_CSUM; 1942 trace_tcp_bad_csum(skb); 1943 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS); 1944 bad_packet: 1945 __TCP_INC_STATS(net, TCP_MIB_INERRS); 1946 } else { 1947 tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason)); 1948 } 1949 1950 discard_it: 1951 SKB_DR_OR(drop_reason, NOT_SPECIFIED); 1952 sk_skb_reason_drop(sk, skb, drop_reason); 1953 return 0; 1954 1955 discard_and_relse: 1956 sk_drops_add(sk, skb); 1957 if (refcounted) 1958 sock_put(sk); 1959 goto discard_it; 1960 1961 do_time_wait: 1962 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) { 1963 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1964 inet_twsk_put(inet_twsk(sk)); 1965 goto discard_it; 1966 } 1967 1968 tcp_v6_fill_cb(skb, hdr, th); 1969 1970 if (tcp_checksum_complete(skb)) { 1971 inet_twsk_put(inet_twsk(sk)); 1972 goto csum_error; 1973 } 1974 1975 tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn, 1976 &drop_reason); 1977 switch (tw_status) { 1978 case TCP_TW_SYN: 1979 { 1980 struct sock *sk2; 1981 1982 sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, 1983 skb, __tcp_hdrlen(th), 1984 &ipv6_hdr(skb)->saddr, th->source, 1985 &ipv6_hdr(skb)->daddr, 1986 ntohs(th->dest), 1987 tcp_v6_iif_l3_slave(skb), 1988 sdif); 1989 if (sk2) { 1990 struct inet_timewait_sock *tw = inet_twsk(sk); 1991 inet_twsk_deschedule_put(tw); 1992 sk = sk2; 1993 tcp_v6_restore_cb(skb); 1994 refcounted = false; 1995 __this_cpu_write(tcp_tw_isn, isn); 1996 goto process; 1997 } 1998 } 1999 /* to ACK */ 2000 fallthrough; 2001 case TCP_TW_ACK: 2002 case TCP_TW_ACK_OOW: 2003 tcp_v6_timewait_ack(sk, skb, tw_status); 2004 break; 2005 case TCP_TW_RST: 2006 tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET); 2007 inet_twsk_deschedule_put(inet_twsk(sk)); 2008 goto discard_it; 2009 case TCP_TW_SUCCESS: 2010 ; 2011 } 2012 goto discard_it; 2013 } 2014 2015 void tcp_v6_early_demux(struct sk_buff *skb) 2016 { 2017 struct net *net = dev_net_rcu(skb->dev); 2018 const struct ipv6hdr *hdr; 2019 const struct tcphdr *th; 2020 struct sock *sk; 2021 2022 if (skb->pkt_type != PACKET_HOST) 2023 return; 2024 2025 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr))) 2026 return; 2027 2028 hdr = ipv6_hdr(skb); 2029 th = tcp_hdr(skb); 2030 2031 if (th->doff < sizeof(struct tcphdr) / 4) 2032 return; 2033 2034 /* Note : We use inet6_iif() here, not tcp_v6_iif() */ 2035 sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 2036 &hdr->saddr, th->source, 2037 &hdr->daddr, ntohs(th->dest), 2038 inet6_iif(skb), inet6_sdif(skb)); 2039 if (sk) { 2040 skb->sk = sk; 2041 skb->destructor = sock_edemux; 2042 if (sk_fullsock(sk)) { 2043 struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst); 2044 2045 if (dst) 2046 dst = dst_check(dst, sk->sk_rx_dst_cookie); 2047 if (dst && 2048 sk->sk_rx_dst_ifindex == skb->skb_iif) 2049 skb_dst_set_noref(skb, dst); 2050 } 2051 } 2052 } 2053 2054 static struct timewait_sock_ops tcp6_timewait_sock_ops = { 2055 .twsk_obj_size = sizeof(struct tcp6_timewait_sock), 2056 .twsk_destructor = tcp_twsk_destructor, 2057 }; 2058 2059 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) 2060 { 2061 __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr); 2062 } 2063 2064 const struct inet_connection_sock_af_ops ipv6_specific = { 2065 .queue_xmit = inet6_csk_xmit, 2066 .send_check = tcp_v6_send_check, 2067 .rebuild_header = inet6_sk_rebuild_header, 2068 .sk_rx_dst_set = inet6_sk_rx_dst_set, 2069 .conn_request = tcp_v6_conn_request, 2070 .syn_recv_sock = tcp_v6_syn_recv_sock, 2071 .net_header_len = sizeof(struct ipv6hdr), 2072 .setsockopt = ipv6_setsockopt, 2073 .getsockopt = ipv6_getsockopt, 2074 .mtu_reduced = tcp_v6_mtu_reduced, 2075 }; 2076 2077 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2078 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = { 2079 #ifdef CONFIG_TCP_MD5SIG 2080 .md5_lookup = tcp_v6_md5_lookup, 2081 .calc_md5_hash = tcp_v6_md5_hash_skb, 2082 .md5_parse = tcp_v6_parse_md5_keys, 2083 #endif 2084 #ifdef CONFIG_TCP_AO 2085 .ao_lookup = tcp_v6_ao_lookup, 2086 .calc_ao_hash = tcp_v6_ao_hash_skb, 2087 .ao_parse = tcp_v6_parse_ao, 2088 .ao_calc_key_sk = tcp_v6_ao_calc_key_sk, 2089 #endif 2090 }; 2091 #endif 2092 2093 /* 2094 * TCP over IPv4 via INET6 API 2095 */ 2096 static const struct inet_connection_sock_af_ops ipv6_mapped = { 2097 .queue_xmit = ip_queue_xmit, 2098 .send_check = tcp_v4_send_check, 2099 .rebuild_header = inet_sk_rebuild_header, 2100 .sk_rx_dst_set = inet_sk_rx_dst_set, 2101 .conn_request = tcp_v6_conn_request, 2102 .syn_recv_sock = tcp_v6_syn_recv_sock, 2103 .net_header_len = sizeof(struct iphdr), 2104 .setsockopt = ipv6_setsockopt, 2105 .getsockopt = ipv6_getsockopt, 2106 .mtu_reduced = tcp_v4_mtu_reduced, 2107 }; 2108 2109 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2110 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = { 2111 #ifdef CONFIG_TCP_MD5SIG 2112 .md5_lookup = tcp_v4_md5_lookup, 2113 .calc_md5_hash = tcp_v4_md5_hash_skb, 2114 .md5_parse = tcp_v6_parse_md5_keys, 2115 #endif 2116 #ifdef CONFIG_TCP_AO 2117 .ao_lookup = tcp_v6_ao_lookup, 2118 .calc_ao_hash = tcp_v4_ao_hash_skb, 2119 .ao_parse = tcp_v6_parse_ao, 2120 .ao_calc_key_sk = tcp_v4_ao_calc_key_sk, 2121 #endif 2122 }; 2123 #endif 2124 2125 /* NOTE: A lot of things set to zero explicitly by call to 2126 * sk_alloc() so need not be done here. 2127 */ 2128 static int tcp_v6_init_sock(struct sock *sk) 2129 { 2130 struct inet_connection_sock *icsk = inet_csk(sk); 2131 2132 tcp_init_sock(sk); 2133 2134 icsk->icsk_af_ops = &ipv6_specific; 2135 2136 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2137 tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific; 2138 #endif 2139 2140 return 0; 2141 } 2142 2143 #ifdef CONFIG_PROC_FS 2144 /* Proc filesystem TCPv6 sock list dumping. */ 2145 static void get_openreq6(struct seq_file *seq, 2146 const struct request_sock *req, int i) 2147 { 2148 long ttd = req->rsk_timer.expires - jiffies; 2149 const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr; 2150 const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr; 2151 2152 if (ttd < 0) 2153 ttd = 0; 2154 2155 seq_printf(seq, 2156 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2157 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n", 2158 i, 2159 src->s6_addr32[0], src->s6_addr32[1], 2160 src->s6_addr32[2], src->s6_addr32[3], 2161 inet_rsk(req)->ir_num, 2162 dest->s6_addr32[0], dest->s6_addr32[1], 2163 dest->s6_addr32[2], dest->s6_addr32[3], 2164 ntohs(inet_rsk(req)->ir_rmt_port), 2165 TCP_SYN_RECV, 2166 0, 0, /* could print option size, but that is af dependent. */ 2167 1, /* timers active (only the expire timer) */ 2168 jiffies_to_clock_t(ttd), 2169 req->num_timeout, 2170 from_kuid_munged(seq_user_ns(seq), 2171 sock_i_uid(req->rsk_listener)), 2172 0, /* non standard timer */ 2173 0, /* open_requests have no inode */ 2174 0, req); 2175 } 2176 2177 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i) 2178 { 2179 const struct in6_addr *dest, *src; 2180 __u16 destp, srcp; 2181 int timer_active; 2182 unsigned long timer_expires; 2183 const struct inet_sock *inet = inet_sk(sp); 2184 const struct tcp_sock *tp = tcp_sk(sp); 2185 const struct inet_connection_sock *icsk = inet_csk(sp); 2186 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq; 2187 u8 icsk_pending; 2188 int rx_queue; 2189 int state; 2190 2191 dest = &sp->sk_v6_daddr; 2192 src = &sp->sk_v6_rcv_saddr; 2193 destp = ntohs(inet->inet_dport); 2194 srcp = ntohs(inet->inet_sport); 2195 2196 icsk_pending = smp_load_acquire(&icsk->icsk_pending); 2197 if (icsk_pending == ICSK_TIME_RETRANS || 2198 icsk_pending == ICSK_TIME_REO_TIMEOUT || 2199 icsk_pending == ICSK_TIME_LOSS_PROBE) { 2200 timer_active = 1; 2201 timer_expires = icsk_timeout(icsk); 2202 } else if (icsk_pending == ICSK_TIME_PROBE0) { 2203 timer_active = 4; 2204 timer_expires = icsk_timeout(icsk); 2205 } else if (timer_pending(&sp->sk_timer)) { 2206 timer_active = 2; 2207 timer_expires = sp->sk_timer.expires; 2208 } else { 2209 timer_active = 0; 2210 timer_expires = jiffies; 2211 } 2212 2213 state = inet_sk_state_load(sp); 2214 if (state == TCP_LISTEN) 2215 rx_queue = READ_ONCE(sp->sk_ack_backlog); 2216 else 2217 /* Because we don't lock the socket, 2218 * we might find a transient negative value. 2219 */ 2220 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) - 2221 READ_ONCE(tp->copied_seq), 0); 2222 2223 seq_printf(seq, 2224 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2225 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n", 2226 i, 2227 src->s6_addr32[0], src->s6_addr32[1], 2228 src->s6_addr32[2], src->s6_addr32[3], srcp, 2229 dest->s6_addr32[0], dest->s6_addr32[1], 2230 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2231 state, 2232 READ_ONCE(tp->write_seq) - tp->snd_una, 2233 rx_queue, 2234 timer_active, 2235 jiffies_delta_to_clock_t(timer_expires - jiffies), 2236 icsk->icsk_retransmits, 2237 from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)), 2238 icsk->icsk_probes_out, 2239 sock_i_ino(sp), 2240 refcount_read(&sp->sk_refcnt), sp, 2241 jiffies_to_clock_t(icsk->icsk_rto), 2242 jiffies_to_clock_t(icsk->icsk_ack.ato), 2243 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp), 2244 tcp_snd_cwnd(tp), 2245 state == TCP_LISTEN ? 2246 fastopenq->max_qlen : 2247 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh) 2248 ); 2249 } 2250 2251 static void get_timewait6_sock(struct seq_file *seq, 2252 struct inet_timewait_sock *tw, int i) 2253 { 2254 long delta = tw->tw_timer.expires - jiffies; 2255 const struct in6_addr *dest, *src; 2256 __u16 destp, srcp; 2257 2258 dest = &tw->tw_v6_daddr; 2259 src = &tw->tw_v6_rcv_saddr; 2260 destp = ntohs(tw->tw_dport); 2261 srcp = ntohs(tw->tw_sport); 2262 2263 seq_printf(seq, 2264 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2265 "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", 2266 i, 2267 src->s6_addr32[0], src->s6_addr32[1], 2268 src->s6_addr32[2], src->s6_addr32[3], srcp, 2269 dest->s6_addr32[0], dest->s6_addr32[1], 2270 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2271 READ_ONCE(tw->tw_substate), 0, 0, 2272 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, 2273 refcount_read(&tw->tw_refcnt), tw); 2274 } 2275 2276 static int tcp6_seq_show(struct seq_file *seq, void *v) 2277 { 2278 struct tcp_iter_state *st; 2279 struct sock *sk = v; 2280 2281 if (v == SEQ_START_TOKEN) { 2282 seq_puts(seq, 2283 " sl " 2284 "local_address " 2285 "remote_address " 2286 "st tx_queue rx_queue tr tm->when retrnsmt" 2287 " uid timeout inode\n"); 2288 goto out; 2289 } 2290 st = seq->private; 2291 2292 if (sk->sk_state == TCP_TIME_WAIT) 2293 get_timewait6_sock(seq, v, st->num); 2294 else if (sk->sk_state == TCP_NEW_SYN_RECV) 2295 get_openreq6(seq, v, st->num); 2296 else 2297 get_tcp6_sock(seq, v, st->num); 2298 out: 2299 return 0; 2300 } 2301 2302 static const struct seq_operations tcp6_seq_ops = { 2303 .show = tcp6_seq_show, 2304 .start = tcp_seq_start, 2305 .next = tcp_seq_next, 2306 .stop = tcp_seq_stop, 2307 }; 2308 2309 static struct tcp_seq_afinfo tcp6_seq_afinfo = { 2310 .family = AF_INET6, 2311 }; 2312 2313 int __net_init tcp6_proc_init(struct net *net) 2314 { 2315 if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops, 2316 sizeof(struct tcp_iter_state), &tcp6_seq_afinfo)) 2317 return -ENOMEM; 2318 return 0; 2319 } 2320 2321 void tcp6_proc_exit(struct net *net) 2322 { 2323 remove_proc_entry("tcp6", net->proc_net); 2324 } 2325 #endif 2326 2327 struct proto tcpv6_prot = { 2328 .name = "TCPv6", 2329 .owner = THIS_MODULE, 2330 .close = tcp_close, 2331 .pre_connect = tcp_v6_pre_connect, 2332 .connect = tcp_v6_connect, 2333 .disconnect = tcp_disconnect, 2334 .accept = inet_csk_accept, 2335 .ioctl = tcp_ioctl, 2336 .init = tcp_v6_init_sock, 2337 .destroy = tcp_v4_destroy_sock, 2338 .shutdown = tcp_shutdown, 2339 .setsockopt = tcp_setsockopt, 2340 .getsockopt = tcp_getsockopt, 2341 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, 2342 .keepalive = tcp_set_keepalive, 2343 .recvmsg = tcp_recvmsg, 2344 .sendmsg = tcp_sendmsg, 2345 .splice_eof = tcp_splice_eof, 2346 .backlog_rcv = tcp_v6_do_rcv, 2347 .release_cb = tcp_release_cb, 2348 .hash = inet6_hash, 2349 .unhash = inet_unhash, 2350 .get_port = inet_csk_get_port, 2351 .put_port = inet_put_port, 2352 #ifdef CONFIG_BPF_SYSCALL 2353 .psock_update_sk_prot = tcp_bpf_update_proto, 2354 #endif 2355 .enter_memory_pressure = tcp_enter_memory_pressure, 2356 .leave_memory_pressure = tcp_leave_memory_pressure, 2357 .stream_memory_free = tcp_stream_memory_free, 2358 .sockets_allocated = &tcp_sockets_allocated, 2359 2360 .memory_allocated = &tcp_memory_allocated, 2361 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 2362 2363 .memory_pressure = &tcp_memory_pressure, 2364 .orphan_count = &tcp_orphan_count, 2365 .sysctl_mem = sysctl_tcp_mem, 2366 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 2367 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 2368 .max_header = MAX_TCP_HEADER, 2369 .obj_size = sizeof(struct tcp6_sock), 2370 .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6), 2371 .slab_flags = SLAB_TYPESAFE_BY_RCU, 2372 .twsk_prot = &tcp6_timewait_sock_ops, 2373 .rsk_prot = &tcp6_request_sock_ops, 2374 .h.hashinfo = NULL, 2375 .no_autobind = true, 2376 .diag_destroy = tcp_abort, 2377 }; 2378 EXPORT_SYMBOL_GPL(tcpv6_prot); 2379 2380 2381 static struct inet_protosw tcpv6_protosw = { 2382 .type = SOCK_STREAM, 2383 .protocol = IPPROTO_TCP, 2384 .prot = &tcpv6_prot, 2385 .ops = &inet6_stream_ops, 2386 .flags = INET_PROTOSW_PERMANENT | 2387 INET_PROTOSW_ICSK, 2388 }; 2389 2390 static int __net_init tcpv6_net_init(struct net *net) 2391 { 2392 int res; 2393 2394 res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6, 2395 SOCK_RAW, IPPROTO_TCP, net); 2396 if (!res) 2397 net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC; 2398 2399 return res; 2400 } 2401 2402 static void __net_exit tcpv6_net_exit(struct net *net) 2403 { 2404 inet_ctl_sock_destroy(net->ipv6.tcp_sk); 2405 } 2406 2407 static struct pernet_operations tcpv6_net_ops = { 2408 .init = tcpv6_net_init, 2409 .exit = tcpv6_net_exit, 2410 }; 2411 2412 int __init tcpv6_init(void) 2413 { 2414 int ret; 2415 2416 net_hotdata.tcpv6_protocol = (struct inet6_protocol) { 2417 .handler = tcp_v6_rcv, 2418 .err_handler = tcp_v6_err, 2419 .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL, 2420 }; 2421 ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2422 if (ret) 2423 goto out; 2424 2425 /* register inet6 protocol */ 2426 ret = inet6_register_protosw(&tcpv6_protosw); 2427 if (ret) 2428 goto out_tcpv6_protocol; 2429 2430 ret = register_pernet_subsys(&tcpv6_net_ops); 2431 if (ret) 2432 goto out_tcpv6_protosw; 2433 2434 ret = mptcpv6_init(); 2435 if (ret) 2436 goto out_tcpv6_pernet_subsys; 2437 2438 out: 2439 return ret; 2440 2441 out_tcpv6_pernet_subsys: 2442 unregister_pernet_subsys(&tcpv6_net_ops); 2443 out_tcpv6_protosw: 2444 inet6_unregister_protosw(&tcpv6_protosw); 2445 out_tcpv6_protocol: 2446 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2447 goto out; 2448 } 2449 2450 void tcpv6_exit(void) 2451 { 2452 unregister_pernet_subsys(&tcpv6_net_ops); 2453 inet6_unregister_protosw(&tcpv6_protosw); 2454 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2455 } 2456