1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TCP over IPv6 4 * Linux INET6 implementation 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 * 9 * Based on: 10 * linux/net/ipv4/tcp.c 11 * linux/net/ipv4/tcp_input.c 12 * linux/net/ipv4/tcp_output.c 13 * 14 * Fixes: 15 * Hideaki YOSHIFUJI : sin6_scope_id support 16 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which 17 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind 18 * a single port at the same time. 19 * YOSHIFUJI Hideaki @USAGI: convert /proc/net/tcp6 to seq_file. 20 */ 21 22 #include <linux/bottom_half.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/types.h> 26 #include <linux/socket.h> 27 #include <linux/sockios.h> 28 #include <linux/net.h> 29 #include <linux/jiffies.h> 30 #include <linux/in.h> 31 #include <linux/in6.h> 32 #include <linux/netdevice.h> 33 #include <linux/init.h> 34 #include <linux/jhash.h> 35 #include <linux/ipsec.h> 36 #include <linux/times.h> 37 #include <linux/slab.h> 38 #include <linux/uaccess.h> 39 #include <linux/ipv6.h> 40 #include <linux/icmpv6.h> 41 #include <linux/random.h> 42 #include <linux/indirect_call_wrapper.h> 43 44 #include <net/tcp.h> 45 #include <net/ndisc.h> 46 #include <net/inet6_hashtables.h> 47 #include <net/inet6_connection_sock.h> 48 #include <net/ipv6.h> 49 #include <net/transp_v6.h> 50 #include <net/addrconf.h> 51 #include <net/ip6_route.h> 52 #include <net/ip6_checksum.h> 53 #include <net/inet_ecn.h> 54 #include <net/protocol.h> 55 #include <net/xfrm.h> 56 #include <net/snmp.h> 57 #include <net/dsfield.h> 58 #include <net/timewait_sock.h> 59 #include <net/inet_common.h> 60 #include <net/secure_seq.h> 61 #include <net/hotdata.h> 62 #include <net/busy_poll.h> 63 #include <net/rstreason.h> 64 65 #include <linux/proc_fs.h> 66 #include <linux/seq_file.h> 67 68 #include <crypto/hash.h> 69 #include <linux/scatterlist.h> 70 71 #include <trace/events/tcp.h> 72 73 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 74 enum sk_rst_reason reason); 75 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 76 struct request_sock *req); 77 78 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb); 79 80 static const struct inet_connection_sock_af_ops ipv6_mapped; 81 const struct inet_connection_sock_af_ops ipv6_specific; 82 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 83 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific; 84 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific; 85 #endif 86 87 /* Helper returning the inet6 address from a given tcp socket. 88 * It can be used in TCP stack instead of inet6_sk(sk). 89 * This avoids a dereference and allow compiler optimizations. 90 * It is a specialized version of inet6_sk_generic(). 91 */ 92 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \ 93 struct tcp6_sock, tcp)->inet6) 94 95 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 96 { 97 struct dst_entry *dst = skb_dst(skb); 98 99 if (dst && dst_hold_safe(dst)) { 100 rcu_assign_pointer(sk->sk_rx_dst, dst); 101 sk->sk_rx_dst_ifindex = skb->skb_iif; 102 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst)); 103 } 104 } 105 106 static u32 tcp_v6_init_seq(const struct sk_buff *skb) 107 { 108 return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32, 109 ipv6_hdr(skb)->saddr.s6_addr32, 110 tcp_hdr(skb)->dest, 111 tcp_hdr(skb)->source); 112 } 113 114 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb) 115 { 116 return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32, 117 ipv6_hdr(skb)->saddr.s6_addr32); 118 } 119 120 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr, 121 int addr_len) 122 { 123 /* This check is replicated from tcp_v6_connect() and intended to 124 * prevent BPF program called below from accessing bytes that are out 125 * of the bound specified by user in addr_len. 126 */ 127 if (addr_len < SIN6_LEN_RFC2133) 128 return -EINVAL; 129 130 sock_owned_by_me(sk); 131 132 return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len); 133 } 134 135 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, 136 int addr_len) 137 { 138 struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr; 139 struct inet_connection_sock *icsk = inet_csk(sk); 140 struct in6_addr *saddr = NULL, *final_p, final; 141 struct inet_timewait_death_row *tcp_death_row; 142 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 143 struct inet_sock *inet = inet_sk(sk); 144 struct tcp_sock *tp = tcp_sk(sk); 145 struct net *net = sock_net(sk); 146 struct ipv6_txoptions *opt; 147 struct dst_entry *dst; 148 struct flowi6 fl6; 149 int addr_type; 150 int err; 151 152 if (addr_len < SIN6_LEN_RFC2133) 153 return -EINVAL; 154 155 if (usin->sin6_family != AF_INET6) 156 return -EAFNOSUPPORT; 157 158 memset(&fl6, 0, sizeof(fl6)); 159 160 if (inet6_test_bit(SNDFLOW, sk)) { 161 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK; 162 IP6_ECN_flow_init(fl6.flowlabel); 163 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) { 164 struct ip6_flowlabel *flowlabel; 165 flowlabel = fl6_sock_lookup(sk, fl6.flowlabel); 166 if (IS_ERR(flowlabel)) 167 return -EINVAL; 168 fl6_sock_release(flowlabel); 169 } 170 } 171 172 /* 173 * connect() to INADDR_ANY means loopback (BSD'ism). 174 */ 175 176 if (ipv6_addr_any(&usin->sin6_addr)) { 177 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) 178 ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK), 179 &usin->sin6_addr); 180 else 181 usin->sin6_addr = in6addr_loopback; 182 } 183 184 addr_type = ipv6_addr_type(&usin->sin6_addr); 185 186 if (addr_type & IPV6_ADDR_MULTICAST) 187 return -ENETUNREACH; 188 189 if (addr_type&IPV6_ADDR_LINKLOCAL) { 190 if (addr_len >= sizeof(struct sockaddr_in6) && 191 usin->sin6_scope_id) { 192 /* If interface is set while binding, indices 193 * must coincide. 194 */ 195 if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id)) 196 return -EINVAL; 197 198 sk->sk_bound_dev_if = usin->sin6_scope_id; 199 } 200 201 /* Connect to link-local address requires an interface */ 202 if (!sk->sk_bound_dev_if) 203 return -EINVAL; 204 } 205 206 if (tp->rx_opt.ts_recent_stamp && 207 !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) { 208 tp->rx_opt.ts_recent = 0; 209 tp->rx_opt.ts_recent_stamp = 0; 210 WRITE_ONCE(tp->write_seq, 0); 211 } 212 213 sk->sk_v6_daddr = usin->sin6_addr; 214 np->flow_label = fl6.flowlabel; 215 216 /* 217 * TCP over IPv4 218 */ 219 220 if (addr_type & IPV6_ADDR_MAPPED) { 221 u32 exthdrlen = icsk->icsk_ext_hdr_len; 222 struct sockaddr_in sin; 223 224 if (ipv6_only_sock(sk)) 225 return -ENETUNREACH; 226 227 sin.sin_family = AF_INET; 228 sin.sin_port = usin->sin6_port; 229 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3]; 230 231 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 232 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped); 233 if (sk_is_mptcp(sk)) 234 mptcpv6_handle_mapped(sk, true); 235 sk->sk_backlog_rcv = tcp_v4_do_rcv; 236 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 237 tp->af_specific = &tcp_sock_ipv6_mapped_specific; 238 #endif 239 240 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin)); 241 242 if (err) { 243 icsk->icsk_ext_hdr_len = exthdrlen; 244 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 245 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific); 246 if (sk_is_mptcp(sk)) 247 mptcpv6_handle_mapped(sk, false); 248 sk->sk_backlog_rcv = tcp_v6_do_rcv; 249 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 250 tp->af_specific = &tcp_sock_ipv6_specific; 251 #endif 252 goto failure; 253 } 254 np->saddr = sk->sk_v6_rcv_saddr; 255 256 return err; 257 } 258 259 if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) 260 saddr = &sk->sk_v6_rcv_saddr; 261 262 fl6.flowi6_proto = IPPROTO_TCP; 263 fl6.daddr = sk->sk_v6_daddr; 264 fl6.saddr = saddr ? *saddr : np->saddr; 265 fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label); 266 fl6.flowi6_oif = sk->sk_bound_dev_if; 267 fl6.flowi6_mark = sk->sk_mark; 268 fl6.fl6_dport = usin->sin6_port; 269 fl6.fl6_sport = inet->inet_sport; 270 if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6.fl6_sport) 271 fl6.flowi6_flags = FLOWI_FLAG_ANY_SPORT; 272 fl6.flowi6_uid = sk_uid(sk); 273 274 opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk)); 275 final_p = fl6_update_dst(&fl6, opt, &final); 276 277 security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6)); 278 279 dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p); 280 if (IS_ERR(dst)) { 281 err = PTR_ERR(dst); 282 goto failure; 283 } 284 285 tp->tcp_usec_ts = dst_tcp_usec_ts(dst); 286 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row; 287 288 if (!saddr) { 289 saddr = &fl6.saddr; 290 291 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6); 292 if (err) 293 goto failure; 294 } 295 296 /* set the source address */ 297 np->saddr = *saddr; 298 inet->inet_rcv_saddr = LOOPBACK4_IPV6; 299 300 sk->sk_gso_type = SKB_GSO_TCPV6; 301 ip6_dst_store(sk, dst, NULL, NULL); 302 303 icsk->icsk_ext_hdr_len = 0; 304 if (opt) 305 icsk->icsk_ext_hdr_len = opt->opt_flen + 306 opt->opt_nflen; 307 308 tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr); 309 310 inet->inet_dport = usin->sin6_port; 311 312 tcp_set_state(sk, TCP_SYN_SENT); 313 err = inet6_hash_connect(tcp_death_row, sk); 314 if (err) 315 goto late_failure; 316 317 sk_set_txhash(sk); 318 319 if (likely(!tp->repair)) { 320 if (!tp->write_seq) 321 WRITE_ONCE(tp->write_seq, 322 secure_tcpv6_seq(np->saddr.s6_addr32, 323 sk->sk_v6_daddr.s6_addr32, 324 inet->inet_sport, 325 inet->inet_dport)); 326 tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, 327 sk->sk_v6_daddr.s6_addr32); 328 } 329 330 if (tcp_fastopen_defer_connect(sk, &err)) 331 return err; 332 if (err) 333 goto late_failure; 334 335 err = tcp_connect(sk); 336 if (err) 337 goto late_failure; 338 339 return 0; 340 341 late_failure: 342 tcp_set_state(sk, TCP_CLOSE); 343 inet_bhash2_reset_saddr(sk); 344 failure: 345 inet->inet_dport = 0; 346 sk->sk_route_caps = 0; 347 return err; 348 } 349 350 static void tcp_v6_mtu_reduced(struct sock *sk) 351 { 352 struct dst_entry *dst; 353 u32 mtu; 354 355 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 356 return; 357 358 mtu = READ_ONCE(tcp_sk(sk)->mtu_info); 359 360 /* Drop requests trying to increase our current mss. 361 * Check done in __ip6_rt_update_pmtu() is too late. 362 */ 363 if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache) 364 return; 365 366 dst = inet6_csk_update_pmtu(sk, mtu); 367 if (!dst) 368 return; 369 370 if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) { 371 tcp_sync_mss(sk, dst_mtu(dst)); 372 tcp_simple_retransmit(sk); 373 } 374 } 375 376 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 377 u8 type, u8 code, int offset, __be32 info) 378 { 379 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; 380 const struct tcphdr *th = (struct tcphdr *)(skb->data+offset); 381 struct net *net = dev_net_rcu(skb->dev); 382 struct request_sock *fastopen; 383 struct ipv6_pinfo *np; 384 struct tcp_sock *tp; 385 __u32 seq, snd_una; 386 struct sock *sk; 387 bool fatal; 388 int err; 389 390 sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 391 &hdr->daddr, th->dest, 392 &hdr->saddr, ntohs(th->source), 393 skb->dev->ifindex, inet6_sdif(skb)); 394 395 if (!sk) { 396 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), 397 ICMP6_MIB_INERRORS); 398 return -ENOENT; 399 } 400 401 if (sk->sk_state == TCP_TIME_WAIT) { 402 /* To increase the counter of ignored icmps for TCP-AO */ 403 tcp_ao_ignore_icmp(sk, AF_INET6, type, code); 404 inet_twsk_put(inet_twsk(sk)); 405 return 0; 406 } 407 seq = ntohl(th->seq); 408 fatal = icmpv6_err_convert(type, code, &err); 409 if (sk->sk_state == TCP_NEW_SYN_RECV) { 410 tcp_req_err(sk, seq, fatal); 411 return 0; 412 } 413 414 if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) { 415 sock_put(sk); 416 return 0; 417 } 418 419 bh_lock_sock(sk); 420 if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG) 421 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS); 422 423 if (sk->sk_state == TCP_CLOSE) 424 goto out; 425 426 if (static_branch_unlikely(&ip6_min_hopcount)) { 427 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 428 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) { 429 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 430 goto out; 431 } 432 } 433 434 tp = tcp_sk(sk); 435 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */ 436 fastopen = rcu_dereference(tp->fastopen_rsk); 437 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una; 438 if (sk->sk_state != TCP_LISTEN && 439 !between(seq, snd_una, tp->snd_nxt)) { 440 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS); 441 goto out; 442 } 443 444 np = tcp_inet6_sk(sk); 445 446 if (type == NDISC_REDIRECT) { 447 if (!sock_owned_by_user(sk)) { 448 struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie); 449 450 if (dst) 451 dst->ops->redirect(dst, sk, skb); 452 } 453 goto out; 454 } 455 456 if (type == ICMPV6_PKT_TOOBIG) { 457 u32 mtu = ntohl(info); 458 459 /* We are not interested in TCP_LISTEN and open_requests 460 * (SYN-ACKs send out by Linux are always <576bytes so 461 * they should go through unfragmented). 462 */ 463 if (sk->sk_state == TCP_LISTEN) 464 goto out; 465 466 if (!ip6_sk_accept_pmtu(sk)) 467 goto out; 468 469 if (mtu < IPV6_MIN_MTU) 470 goto out; 471 472 WRITE_ONCE(tp->mtu_info, mtu); 473 474 if (!sock_owned_by_user(sk)) 475 tcp_v6_mtu_reduced(sk); 476 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, 477 &sk->sk_tsq_flags)) 478 sock_hold(sk); 479 goto out; 480 } 481 482 483 /* Might be for an request_sock */ 484 switch (sk->sk_state) { 485 case TCP_SYN_SENT: 486 case TCP_SYN_RECV: 487 /* Only in fast or simultaneous open. If a fast open socket is 488 * already accepted it is treated as a connected one below. 489 */ 490 if (fastopen && !fastopen->sk) 491 break; 492 493 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th); 494 495 if (!sock_owned_by_user(sk)) 496 tcp_done_with_error(sk, err); 497 else 498 WRITE_ONCE(sk->sk_err_soft, err); 499 goto out; 500 case TCP_LISTEN: 501 break; 502 default: 503 /* check if this ICMP message allows revert of backoff. 504 * (see RFC 6069) 505 */ 506 if (!fastopen && type == ICMPV6_DEST_UNREACH && 507 code == ICMPV6_NOROUTE) 508 tcp_ld_RTO_revert(sk, seq); 509 } 510 511 if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) { 512 WRITE_ONCE(sk->sk_err, err); 513 sk_error_report(sk); 514 } else { 515 WRITE_ONCE(sk->sk_err_soft, err); 516 } 517 out: 518 bh_unlock_sock(sk); 519 sock_put(sk); 520 return 0; 521 } 522 523 524 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 525 struct flowi *fl, 526 struct request_sock *req, 527 struct tcp_fastopen_cookie *foc, 528 enum tcp_synack_type synack_type, 529 struct sk_buff *syn_skb) 530 { 531 struct inet_request_sock *ireq = inet_rsk(req); 532 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 533 struct ipv6_txoptions *opt; 534 struct flowi6 *fl6 = &fl->u.ip6; 535 struct sk_buff *skb; 536 int err = -ENOMEM; 537 u8 tclass; 538 539 /* First, grab a route. */ 540 if (!dst && (dst = inet6_csk_route_req(sk, fl6, req, 541 IPPROTO_TCP)) == NULL) 542 goto done; 543 544 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb); 545 546 if (skb) { 547 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr, 548 &ireq->ir_v6_rmt_addr); 549 550 fl6->daddr = ireq->ir_v6_rmt_addr; 551 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts) 552 fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts)); 553 554 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ? 555 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) | 556 (np->tclass & INET_ECN_MASK) : 557 np->tclass; 558 559 if (!INET_ECN_is_capable(tclass) && 560 tcp_bpf_ca_needs_ecn((struct sock *)req)) 561 tclass |= INET_ECN_ECT_0; 562 563 rcu_read_lock(); 564 opt = ireq->ipv6_opt; 565 if (!opt) 566 opt = rcu_dereference(np->opt); 567 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark), 568 opt, tclass, READ_ONCE(sk->sk_priority)); 569 rcu_read_unlock(); 570 err = net_xmit_eval(err); 571 } 572 573 done: 574 return err; 575 } 576 577 578 static void tcp_v6_reqsk_destructor(struct request_sock *req) 579 { 580 kfree(inet_rsk(req)->ipv6_opt); 581 consume_skb(inet_rsk(req)->pktopts); 582 } 583 584 #ifdef CONFIG_TCP_MD5SIG 585 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk, 586 const struct in6_addr *addr, 587 int l3index) 588 { 589 return tcp_md5_do_lookup(sk, l3index, 590 (union tcp_md5_addr *)addr, AF_INET6); 591 } 592 593 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk, 594 const struct sock *addr_sk) 595 { 596 int l3index; 597 598 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), 599 addr_sk->sk_bound_dev_if); 600 return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr, 601 l3index); 602 } 603 604 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname, 605 sockptr_t optval, int optlen) 606 { 607 struct tcp_md5sig cmd; 608 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr; 609 union tcp_ao_addr *addr; 610 int l3index = 0; 611 u8 prefixlen; 612 bool l3flag; 613 u8 flags; 614 615 if (optlen < sizeof(cmd)) 616 return -EINVAL; 617 618 if (copy_from_sockptr(&cmd, optval, sizeof(cmd))) 619 return -EFAULT; 620 621 if (sin6->sin6_family != AF_INET6) 622 return -EINVAL; 623 624 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 625 l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 626 627 if (optname == TCP_MD5SIG_EXT && 628 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) { 629 prefixlen = cmd.tcpm_prefixlen; 630 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) && 631 prefixlen > 32)) 632 return -EINVAL; 633 } else { 634 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128; 635 } 636 637 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex && 638 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) { 639 struct net_device *dev; 640 641 rcu_read_lock(); 642 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex); 643 if (dev && netif_is_l3_master(dev)) 644 l3index = dev->ifindex; 645 rcu_read_unlock(); 646 647 /* ok to reference set/not set outside of rcu; 648 * right now device MUST be an L3 master 649 */ 650 if (!dev || !l3index) 651 return -EINVAL; 652 } 653 654 if (!cmd.tcpm_keylen) { 655 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) 656 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3], 657 AF_INET, prefixlen, 658 l3index, flags); 659 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr, 660 AF_INET6, prefixlen, l3index, flags); 661 } 662 663 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN) 664 return -EINVAL; 665 666 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) { 667 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3]; 668 669 /* Don't allow keys for peers that have a matching TCP-AO key. 670 * See the comment in tcp_ao_add_cmd() 671 */ 672 if (tcp_ao_required(sk, addr, AF_INET, 673 l3flag ? l3index : -1, false)) 674 return -EKEYREJECTED; 675 return tcp_md5_do_add(sk, addr, 676 AF_INET, prefixlen, l3index, flags, 677 cmd.tcpm_key, cmd.tcpm_keylen); 678 } 679 680 addr = (union tcp_md5_addr *)&sin6->sin6_addr; 681 682 /* Don't allow keys for peers that have a matching TCP-AO key. 683 * See the comment in tcp_ao_add_cmd() 684 */ 685 if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false)) 686 return -EKEYREJECTED; 687 688 return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags, 689 cmd.tcpm_key, cmd.tcpm_keylen); 690 } 691 692 static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp, 693 const struct in6_addr *daddr, 694 const struct in6_addr *saddr, 695 const struct tcphdr *th, int nbytes) 696 { 697 struct tcp6_pseudohdr *bp; 698 struct scatterlist sg; 699 struct tcphdr *_th; 700 701 bp = hp->scratch; 702 /* 1. TCP pseudo-header (RFC2460) */ 703 bp->saddr = *saddr; 704 bp->daddr = *daddr; 705 bp->protocol = cpu_to_be32(IPPROTO_TCP); 706 bp->len = cpu_to_be32(nbytes); 707 708 _th = (struct tcphdr *)(bp + 1); 709 memcpy(_th, th, sizeof(*th)); 710 _th->check = 0; 711 712 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); 713 ahash_request_set_crypt(hp->req, &sg, NULL, 714 sizeof(*bp) + sizeof(*th)); 715 return crypto_ahash_update(hp->req); 716 } 717 718 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key, 719 const struct in6_addr *daddr, struct in6_addr *saddr, 720 const struct tcphdr *th) 721 { 722 struct tcp_sigpool hp; 723 724 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 725 goto clear_hash_nostart; 726 727 if (crypto_ahash_init(hp.req)) 728 goto clear_hash; 729 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) 730 goto clear_hash; 731 if (tcp_md5_hash_key(&hp, key)) 732 goto clear_hash; 733 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 734 if (crypto_ahash_final(hp.req)) 735 goto clear_hash; 736 737 tcp_sigpool_end(&hp); 738 return 0; 739 740 clear_hash: 741 tcp_sigpool_end(&hp); 742 clear_hash_nostart: 743 memset(md5_hash, 0, 16); 744 return 1; 745 } 746 747 static int tcp_v6_md5_hash_skb(char *md5_hash, 748 const struct tcp_md5sig_key *key, 749 const struct sock *sk, 750 const struct sk_buff *skb) 751 { 752 const struct tcphdr *th = tcp_hdr(skb); 753 const struct in6_addr *saddr, *daddr; 754 struct tcp_sigpool hp; 755 756 if (sk) { /* valid for establish/request sockets */ 757 saddr = &sk->sk_v6_rcv_saddr; 758 daddr = &sk->sk_v6_daddr; 759 } else { 760 const struct ipv6hdr *ip6h = ipv6_hdr(skb); 761 saddr = &ip6h->saddr; 762 daddr = &ip6h->daddr; 763 } 764 765 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 766 goto clear_hash_nostart; 767 768 if (crypto_ahash_init(hp.req)) 769 goto clear_hash; 770 771 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) 772 goto clear_hash; 773 if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2)) 774 goto clear_hash; 775 if (tcp_md5_hash_key(&hp, key)) 776 goto clear_hash; 777 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 778 if (crypto_ahash_final(hp.req)) 779 goto clear_hash; 780 781 tcp_sigpool_end(&hp); 782 return 0; 783 784 clear_hash: 785 tcp_sigpool_end(&hp); 786 clear_hash_nostart: 787 memset(md5_hash, 0, 16); 788 return 1; 789 } 790 #endif 791 792 static void tcp_v6_init_req(struct request_sock *req, 793 const struct sock *sk_listener, 794 struct sk_buff *skb, 795 u32 tw_isn) 796 { 797 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 798 struct inet_request_sock *ireq = inet_rsk(req); 799 const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener); 800 801 ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; 802 ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; 803 ireq->ir_rmt_addr = LOOPBACK4_IPV6; 804 ireq->ir_loc_addr = LOOPBACK4_IPV6; 805 806 /* So that link locals have meaning */ 807 if ((!sk_listener->sk_bound_dev_if || l3_slave) && 808 ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL) 809 ireq->ir_iif = tcp_v6_iif(skb); 810 811 if (!tw_isn && 812 (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) || 813 np->rxopt.bits.rxinfo || 814 np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim || 815 np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) { 816 refcount_inc(&skb->users); 817 ireq->pktopts = skb; 818 } 819 } 820 821 static struct dst_entry *tcp_v6_route_req(const struct sock *sk, 822 struct sk_buff *skb, 823 struct flowi *fl, 824 struct request_sock *req, 825 u32 tw_isn) 826 { 827 tcp_v6_init_req(req, sk, skb, tw_isn); 828 829 if (security_inet_conn_request(sk, skb, req)) 830 return NULL; 831 832 return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP); 833 } 834 835 struct request_sock_ops tcp6_request_sock_ops __read_mostly = { 836 .family = AF_INET6, 837 .obj_size = sizeof(struct tcp6_request_sock), 838 .send_ack = tcp_v6_reqsk_send_ack, 839 .destructor = tcp_v6_reqsk_destructor, 840 .send_reset = tcp_v6_send_reset, 841 .syn_ack_timeout = tcp_syn_ack_timeout, 842 }; 843 844 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { 845 .mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - 846 sizeof(struct ipv6hdr), 847 #ifdef CONFIG_TCP_MD5SIG 848 .req_md5_lookup = tcp_v6_md5_lookup, 849 .calc_md5_hash = tcp_v6_md5_hash_skb, 850 #endif 851 #ifdef CONFIG_TCP_AO 852 .ao_lookup = tcp_v6_ao_lookup_rsk, 853 .ao_calc_key = tcp_v6_ao_calc_key_rsk, 854 .ao_synack_hash = tcp_v6_ao_synack_hash, 855 #endif 856 #ifdef CONFIG_SYN_COOKIES 857 .cookie_init_seq = cookie_v6_init_sequence, 858 #endif 859 .route_req = tcp_v6_route_req, 860 .init_seq = tcp_v6_init_seq, 861 .init_ts_off = tcp_v6_init_ts_off, 862 .send_synack = tcp_v6_send_synack, 863 }; 864 865 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq, 866 u32 ack, u32 win, u32 tsval, u32 tsecr, 867 int oif, int rst, u8 tclass, __be32 label, 868 u32 priority, u32 txhash, struct tcp_key *key) 869 { 870 struct net *net = sk ? sock_net(sk) : dev_net_rcu(skb_dst(skb)->dev); 871 unsigned int tot_len = sizeof(struct tcphdr); 872 struct sock *ctl_sk = net->ipv6.tcp_sk; 873 const struct tcphdr *th = tcp_hdr(skb); 874 __be32 mrst = 0, *topt; 875 struct dst_entry *dst; 876 struct sk_buff *buff; 877 struct tcphdr *t1; 878 struct flowi6 fl6; 879 u32 mark = 0; 880 881 if (tsecr) 882 tot_len += TCPOLEN_TSTAMP_ALIGNED; 883 if (tcp_key_is_md5(key)) 884 tot_len += TCPOLEN_MD5SIG_ALIGNED; 885 if (tcp_key_is_ao(key)) 886 tot_len += tcp_ao_len_aligned(key->ao_key); 887 888 #ifdef CONFIG_MPTCP 889 if (rst && !tcp_key_is_md5(key)) { 890 mrst = mptcp_reset_option(skb); 891 892 if (mrst) 893 tot_len += sizeof(__be32); 894 } 895 #endif 896 897 buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 898 if (!buff) 899 return; 900 901 skb_reserve(buff, MAX_TCP_HEADER); 902 903 t1 = skb_push(buff, tot_len); 904 skb_reset_transport_header(buff); 905 906 /* Swap the send and the receive. */ 907 memset(t1, 0, sizeof(*t1)); 908 t1->dest = th->source; 909 t1->source = th->dest; 910 t1->doff = tot_len / 4; 911 t1->seq = htonl(seq); 912 t1->ack_seq = htonl(ack); 913 t1->ack = !rst || !th->ack; 914 t1->rst = rst; 915 t1->window = htons(win); 916 917 topt = (__be32 *)(t1 + 1); 918 919 if (tsecr) { 920 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 921 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP); 922 *topt++ = htonl(tsval); 923 *topt++ = htonl(tsecr); 924 } 925 926 if (mrst) 927 *topt++ = mrst; 928 929 #ifdef CONFIG_TCP_MD5SIG 930 if (tcp_key_is_md5(key)) { 931 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 932 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG); 933 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key, 934 &ipv6_hdr(skb)->saddr, 935 &ipv6_hdr(skb)->daddr, t1); 936 } 937 #endif 938 #ifdef CONFIG_TCP_AO 939 if (tcp_key_is_ao(key)) { 940 *topt++ = htonl((TCPOPT_AO << 24) | 941 (tcp_ao_len(key->ao_key) << 16) | 942 (key->ao_key->sndid << 8) | 943 (key->rcv_next)); 944 945 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key, 946 key->traffic_key, 947 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr, 948 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr, 949 t1, key->sne); 950 } 951 #endif 952 953 memset(&fl6, 0, sizeof(fl6)); 954 fl6.daddr = ipv6_hdr(skb)->saddr; 955 fl6.saddr = ipv6_hdr(skb)->daddr; 956 fl6.flowlabel = label; 957 958 buff->ip_summed = CHECKSUM_PARTIAL; 959 960 __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr); 961 962 fl6.flowi6_proto = IPPROTO_TCP; 963 if (rt6_need_strict(&fl6.daddr) && !oif) 964 fl6.flowi6_oif = tcp_v6_iif(skb); 965 else { 966 if (!oif && netif_index_is_l3_master(net, skb->skb_iif)) 967 oif = skb->skb_iif; 968 969 fl6.flowi6_oif = oif; 970 } 971 972 if (sk) { 973 /* unconstify the socket only to attach it to buff with care. */ 974 skb_set_owner_edemux(buff, (struct sock *)sk); 975 976 if (sk->sk_state == TCP_TIME_WAIT) 977 mark = inet_twsk(sk)->tw_mark; 978 else 979 mark = READ_ONCE(sk->sk_mark); 980 skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC); 981 } 982 if (txhash) { 983 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */ 984 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4); 985 } 986 fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark; 987 fl6.fl6_dport = t1->dest; 988 fl6.fl6_sport = t1->source; 989 fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL); 990 security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6)); 991 992 /* Pass a socket to ip6_dst_lookup either it is for RST 993 * Underlying function will use this to retrieve the network 994 * namespace 995 */ 996 if (sk && sk->sk_state != TCP_TIME_WAIT) 997 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/ 998 else 999 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL); 1000 if (!IS_ERR(dst)) { 1001 skb_dst_set(buff, dst); 1002 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL, 1003 tclass, priority); 1004 TCP_INC_STATS(net, TCP_MIB_OUTSEGS); 1005 if (rst) 1006 TCP_INC_STATS(net, TCP_MIB_OUTRSTS); 1007 return; 1008 } 1009 1010 kfree_skb(buff); 1011 } 1012 1013 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 1014 enum sk_rst_reason reason) 1015 { 1016 const struct tcphdr *th = tcp_hdr(skb); 1017 struct ipv6hdr *ipv6h = ipv6_hdr(skb); 1018 const __u8 *md5_hash_location = NULL; 1019 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1020 bool allocated_traffic_key = false; 1021 #endif 1022 const struct tcp_ao_hdr *aoh; 1023 struct tcp_key key = {}; 1024 u32 seq = 0, ack_seq = 0; 1025 __be32 label = 0; 1026 u32 priority = 0; 1027 struct net *net; 1028 u32 txhash = 0; 1029 int oif = 0; 1030 #ifdef CONFIG_TCP_MD5SIG 1031 unsigned char newhash[16]; 1032 int genhash; 1033 struct sock *sk1 = NULL; 1034 #endif 1035 1036 if (th->rst) 1037 return; 1038 1039 /* If sk not NULL, it means we did a successful lookup and incoming 1040 * route had to be correct. prequeue might have dropped our dst. 1041 */ 1042 if (!sk && !ipv6_unicast_destination(skb)) 1043 return; 1044 1045 net = sk ? sock_net(sk) : dev_net_rcu(skb_dst(skb)->dev); 1046 /* Invalid TCP option size or twice included auth */ 1047 if (tcp_parse_auth_options(th, &md5_hash_location, &aoh)) 1048 return; 1049 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1050 rcu_read_lock(); 1051 #endif 1052 #ifdef CONFIG_TCP_MD5SIG 1053 if (sk && sk_fullsock(sk)) { 1054 int l3index; 1055 1056 /* sdif set, means packet ingressed via a device 1057 * in an L3 domain and inet_iif is set to it. 1058 */ 1059 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1060 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index); 1061 if (key.md5_key) 1062 key.type = TCP_KEY_MD5; 1063 } else if (md5_hash_location) { 1064 int dif = tcp_v6_iif_l3_slave(skb); 1065 int sdif = tcp_v6_sdif(skb); 1066 int l3index; 1067 1068 /* 1069 * active side is lost. Try to find listening socket through 1070 * source port, and then find md5 key through listening socket. 1071 * we are not loose security here: 1072 * Incoming packet is checked with md5 hash with finding key, 1073 * no RST generated if md5 hash doesn't match. 1074 */ 1075 sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, 1076 NULL, 0, &ipv6h->saddr, th->source, 1077 &ipv6h->daddr, ntohs(th->source), 1078 dif, sdif); 1079 if (!sk1) 1080 goto out; 1081 1082 /* sdif set, means packet ingressed via a device 1083 * in an L3 domain and dif is set to it. 1084 */ 1085 l3index = tcp_v6_sdif(skb) ? dif : 0; 1086 1087 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index); 1088 if (!key.md5_key) 1089 goto out; 1090 key.type = TCP_KEY_MD5; 1091 1092 genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb); 1093 if (genhash || memcmp(md5_hash_location, newhash, 16) != 0) 1094 goto out; 1095 } 1096 #endif 1097 1098 if (th->ack) 1099 seq = ntohl(th->ack_seq); 1100 else 1101 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len - 1102 (th->doff << 2); 1103 1104 #ifdef CONFIG_TCP_AO 1105 if (aoh) { 1106 int l3index; 1107 1108 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1109 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq, 1110 &key.ao_key, &key.traffic_key, 1111 &allocated_traffic_key, 1112 &key.rcv_next, &key.sne)) 1113 goto out; 1114 key.type = TCP_KEY_AO; 1115 } 1116 #endif 1117 1118 if (sk) { 1119 oif = sk->sk_bound_dev_if; 1120 if (sk_fullsock(sk)) { 1121 if (inet6_test_bit(REPFLOW, sk)) 1122 label = ip6_flowlabel(ipv6h); 1123 priority = READ_ONCE(sk->sk_priority); 1124 txhash = sk->sk_txhash; 1125 } 1126 if (sk->sk_state == TCP_TIME_WAIT) { 1127 label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel); 1128 priority = inet_twsk(sk)->tw_priority; 1129 txhash = inet_twsk(sk)->tw_txhash; 1130 } 1131 } else { 1132 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) 1133 label = ip6_flowlabel(ipv6h); 1134 } 1135 1136 trace_tcp_send_reset(sk, skb, reason); 1137 1138 tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1, 1139 ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK, 1140 label, priority, txhash, 1141 &key); 1142 1143 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1144 out: 1145 if (allocated_traffic_key) 1146 kfree(key.traffic_key); 1147 rcu_read_unlock(); 1148 #endif 1149 } 1150 1151 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq, 1152 u32 ack, u32 win, u32 tsval, u32 tsecr, int oif, 1153 struct tcp_key *key, u8 tclass, 1154 __be32 label, u32 priority, u32 txhash) 1155 { 1156 tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0, 1157 tclass, label, priority, txhash, key); 1158 } 1159 1160 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb, 1161 enum tcp_tw_status tw_status) 1162 { 1163 struct inet_timewait_sock *tw = inet_twsk(sk); 1164 struct tcp_timewait_sock *tcptw = tcp_twsk(sk); 1165 u8 tclass = tw->tw_tclass; 1166 struct tcp_key key = {}; 1167 1168 if (tw_status == TCP_TW_ACK_OOW) 1169 tclass &= ~INET_ECN_MASK; 1170 #ifdef CONFIG_TCP_AO 1171 struct tcp_ao_info *ao_info; 1172 1173 if (static_branch_unlikely(&tcp_ao_needed.key)) { 1174 1175 /* FIXME: the segment to-be-acked is not verified yet */ 1176 ao_info = rcu_dereference(tcptw->ao_info); 1177 if (ao_info) { 1178 const struct tcp_ao_hdr *aoh; 1179 1180 /* Invalid TCP option size or twice included auth */ 1181 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1182 goto out; 1183 if (aoh) 1184 key.ao_key = tcp_ao_established_key(sk, ao_info, 1185 aoh->rnext_keyid, -1); 1186 } 1187 } 1188 if (key.ao_key) { 1189 struct tcp_ao_key *rnext_key; 1190 1191 key.traffic_key = snd_other_key(key.ao_key); 1192 /* rcv_next switches to our rcv_next */ 1193 rnext_key = READ_ONCE(ao_info->rnext_key); 1194 key.rcv_next = rnext_key->rcvid; 1195 key.sne = READ_ONCE(ao_info->snd_sne); 1196 key.type = TCP_KEY_AO; 1197 #else 1198 if (0) { 1199 #endif 1200 #ifdef CONFIG_TCP_MD5SIG 1201 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1202 key.md5_key = tcp_twsk_md5_key(tcptw); 1203 if (key.md5_key) 1204 key.type = TCP_KEY_MD5; 1205 #endif 1206 } 1207 1208 tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, 1209 READ_ONCE(tcptw->tw_rcv_nxt), 1210 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale, 1211 tcp_tw_tsval(tcptw), 1212 READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if, 1213 &key, tclass, cpu_to_be32(tw->tw_flowlabel), 1214 tw->tw_priority, tw->tw_txhash); 1215 1216 #ifdef CONFIG_TCP_AO 1217 out: 1218 #endif 1219 inet_twsk_put(tw); 1220 } 1221 1222 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 1223 struct request_sock *req) 1224 { 1225 struct tcp_key key = {}; 1226 1227 #ifdef CONFIG_TCP_AO 1228 if (static_branch_unlikely(&tcp_ao_needed.key) && 1229 tcp_rsk_used_ao(req)) { 1230 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr; 1231 const struct tcp_ao_hdr *aoh; 1232 int l3index; 1233 1234 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1235 /* Invalid TCP option size or twice included auth */ 1236 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1237 return; 1238 if (!aoh) 1239 return; 1240 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1241 (union tcp_ao_addr *)addr, 1242 AF_INET6, aoh->rnext_keyid, -1); 1243 if (unlikely(!key.ao_key)) { 1244 /* Send ACK with any matching MKT for the peer */ 1245 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1246 (union tcp_ao_addr *)addr, 1247 AF_INET6, -1, -1); 1248 /* Matching key disappeared (user removed the key?) 1249 * let the handshake timeout. 1250 */ 1251 if (!key.ao_key) { 1252 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n", 1253 addr, 1254 ntohs(tcp_hdr(skb)->source), 1255 &ipv6_hdr(skb)->daddr, 1256 ntohs(tcp_hdr(skb)->dest)); 1257 return; 1258 } 1259 } 1260 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC); 1261 if (!key.traffic_key) 1262 return; 1263 1264 key.type = TCP_KEY_AO; 1265 key.rcv_next = aoh->keyid; 1266 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req); 1267 #else 1268 if (0) { 1269 #endif 1270 #ifdef CONFIG_TCP_MD5SIG 1271 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1272 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1273 1274 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, 1275 l3index); 1276 if (key.md5_key) 1277 key.type = TCP_KEY_MD5; 1278 #endif 1279 } 1280 1281 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV 1282 * sk->sk_state == TCP_SYN_RECV -> for Fast Open. 1283 */ 1284 tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ? 1285 tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt, 1286 tcp_rsk(req)->rcv_nxt, 1287 tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale, 1288 tcp_rsk_tsval(tcp_rsk(req)), 1289 req->ts_recent, sk->sk_bound_dev_if, 1290 &key, ipv6_get_dsfield(ipv6_hdr(skb)) & ~INET_ECN_MASK, 1291 0, 1292 READ_ONCE(sk->sk_priority), 1293 READ_ONCE(tcp_rsk(req)->txhash)); 1294 if (tcp_key_is_ao(&key)) 1295 kfree(key.traffic_key); 1296 } 1297 1298 1299 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb) 1300 { 1301 #ifdef CONFIG_SYN_COOKIES 1302 const struct tcphdr *th = tcp_hdr(skb); 1303 1304 if (!th->syn) 1305 sk = cookie_v6_check(sk, skb); 1306 #endif 1307 return sk; 1308 } 1309 1310 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 1311 struct tcphdr *th, u32 *cookie) 1312 { 1313 u16 mss = 0; 1314 #ifdef CONFIG_SYN_COOKIES 1315 mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops, 1316 &tcp_request_sock_ipv6_ops, sk, th); 1317 if (mss) { 1318 *cookie = __cookie_v6_init_sequence(iph, th, &mss); 1319 tcp_synq_overflow(sk); 1320 } 1321 #endif 1322 return mss; 1323 } 1324 1325 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb) 1326 { 1327 if (skb->protocol == htons(ETH_P_IP)) 1328 return tcp_v4_conn_request(sk, skb); 1329 1330 if (!ipv6_unicast_destination(skb)) 1331 goto drop; 1332 1333 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 1334 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 1335 return 0; 1336 } 1337 1338 return tcp_conn_request(&tcp6_request_sock_ops, 1339 &tcp_request_sock_ipv6_ops, sk, skb); 1340 1341 drop: 1342 tcp_listendrop(sk); 1343 return 0; /* don't send reset */ 1344 } 1345 1346 static void tcp_v6_restore_cb(struct sk_buff *skb) 1347 { 1348 /* We need to move header back to the beginning if xfrm6_policy_check() 1349 * and tcp_v6_fill_cb() are going to be called again. 1350 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there. 1351 */ 1352 memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6, 1353 sizeof(struct inet6_skb_parm)); 1354 } 1355 1356 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 1357 struct request_sock *req, 1358 struct dst_entry *dst, 1359 struct request_sock *req_unhash, 1360 bool *own_req) 1361 { 1362 struct inet_request_sock *ireq; 1363 struct ipv6_pinfo *newnp; 1364 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1365 struct ipv6_txoptions *opt; 1366 struct inet_sock *newinet; 1367 bool found_dup_sk = false; 1368 struct tcp_sock *newtp; 1369 struct sock *newsk; 1370 #ifdef CONFIG_TCP_MD5SIG 1371 struct tcp_md5sig_key *key; 1372 int l3index; 1373 #endif 1374 struct flowi6 fl6; 1375 1376 if (skb->protocol == htons(ETH_P_IP)) { 1377 /* 1378 * v6 mapped 1379 */ 1380 1381 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst, 1382 req_unhash, own_req); 1383 1384 if (!newsk) 1385 return NULL; 1386 1387 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1388 1389 newnp = tcp_inet6_sk(newsk); 1390 newtp = tcp_sk(newsk); 1391 1392 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1393 1394 newnp->saddr = newsk->sk_v6_rcv_saddr; 1395 1396 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; 1397 if (sk_is_mptcp(newsk)) 1398 mptcpv6_handle_mapped(newsk, true); 1399 newsk->sk_backlog_rcv = tcp_v4_do_rcv; 1400 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1401 newtp->af_specific = &tcp_sock_ipv6_mapped_specific; 1402 #endif 1403 1404 newnp->ipv6_mc_list = NULL; 1405 newnp->ipv6_ac_list = NULL; 1406 newnp->ipv6_fl_list = NULL; 1407 newnp->pktoptions = NULL; 1408 newnp->opt = NULL; 1409 newnp->mcast_oif = inet_iif(skb); 1410 newnp->mcast_hops = ip_hdr(skb)->ttl; 1411 newnp->rcv_flowinfo = 0; 1412 if (inet6_test_bit(REPFLOW, sk)) 1413 newnp->flow_label = 0; 1414 1415 /* 1416 * No need to charge this sock to the relevant IPv6 refcnt debug socks count 1417 * here, tcp_create_openreq_child now does this for us, see the comment in 1418 * that function for the gory details. -acme 1419 */ 1420 1421 /* It is tricky place. Until this moment IPv4 tcp 1422 worked with IPv6 icsk.icsk_af_ops. 1423 Sync it now. 1424 */ 1425 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie); 1426 1427 return newsk; 1428 } 1429 1430 ireq = inet_rsk(req); 1431 1432 if (sk_acceptq_is_full(sk)) 1433 goto out_overflow; 1434 1435 if (!dst) { 1436 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP); 1437 if (!dst) 1438 goto out; 1439 } 1440 1441 newsk = tcp_create_openreq_child(sk, req, skb); 1442 if (!newsk) 1443 goto out_nonewsk; 1444 1445 /* 1446 * No need to charge this sock to the relevant IPv6 refcnt debug socks 1447 * count here, tcp_create_openreq_child now does this for us, see the 1448 * comment in that function for the gory details. -acme 1449 */ 1450 1451 newsk->sk_gso_type = SKB_GSO_TCPV6; 1452 inet6_sk_rx_dst_set(newsk, skb); 1453 1454 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1455 1456 newtp = tcp_sk(newsk); 1457 newinet = inet_sk(newsk); 1458 newnp = tcp_inet6_sk(newsk); 1459 1460 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1461 1462 ip6_dst_store(newsk, dst, NULL, NULL); 1463 1464 newnp->saddr = ireq->ir_v6_loc_addr; 1465 1466 /* Now IPv6 options... 1467 1468 First: no IPv4 options. 1469 */ 1470 newinet->inet_opt = NULL; 1471 newnp->ipv6_mc_list = NULL; 1472 newnp->ipv6_ac_list = NULL; 1473 newnp->ipv6_fl_list = NULL; 1474 1475 /* Clone RX bits */ 1476 newnp->rxopt.all = np->rxopt.all; 1477 1478 newnp->pktoptions = NULL; 1479 newnp->opt = NULL; 1480 newnp->mcast_oif = tcp_v6_iif(skb); 1481 newnp->mcast_hops = ipv6_hdr(skb)->hop_limit; 1482 newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb)); 1483 if (inet6_test_bit(REPFLOW, sk)) 1484 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb)); 1485 1486 /* Set ToS of the new socket based upon the value of incoming SYN. 1487 * ECT bits are set later in tcp_init_transfer(). 1488 */ 1489 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)) 1490 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK; 1491 1492 /* Clone native IPv6 options from listening socket (if any) 1493 1494 Yes, keeping reference count would be much more clever, 1495 but we make one more one thing there: reattach optmem 1496 to newsk. 1497 */ 1498 opt = ireq->ipv6_opt; 1499 if (!opt) 1500 opt = rcu_dereference(np->opt); 1501 if (opt) { 1502 opt = ipv6_dup_options(newsk, opt); 1503 RCU_INIT_POINTER(newnp->opt, opt); 1504 } 1505 inet_csk(newsk)->icsk_ext_hdr_len = 0; 1506 if (opt) 1507 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen + 1508 opt->opt_flen; 1509 1510 tcp_ca_openreq_child(newsk, dst); 1511 1512 tcp_sync_mss(newsk, dst_mtu(dst)); 1513 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); 1514 1515 tcp_initialize_rcv_mss(newsk); 1516 1517 #ifdef CONFIG_TCP_MD5SIG 1518 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif); 1519 1520 if (!tcp_rsk_used_ao(req)) { 1521 /* Copy over the MD5 key from the original socket */ 1522 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index); 1523 if (key) { 1524 const union tcp_md5_addr *addr; 1525 1526 addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr; 1527 if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) { 1528 inet_csk_prepare_forced_close(newsk); 1529 tcp_done(newsk); 1530 goto out; 1531 } 1532 } 1533 } 1534 #endif 1535 #ifdef CONFIG_TCP_AO 1536 /* Copy over tcp_ao_info if any */ 1537 if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6)) 1538 goto out; /* OOM */ 1539 #endif 1540 1541 if (__inet_inherit_port(sk, newsk) < 0) { 1542 inet_csk_prepare_forced_close(newsk); 1543 tcp_done(newsk); 1544 goto out; 1545 } 1546 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash), 1547 &found_dup_sk); 1548 if (*own_req) { 1549 tcp_move_syn(newtp, req); 1550 1551 /* Clone pktoptions received with SYN, if we own the req */ 1552 if (ireq->pktopts) { 1553 newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk); 1554 consume_skb(ireq->pktopts); 1555 ireq->pktopts = NULL; 1556 if (newnp->pktoptions) 1557 tcp_v6_restore_cb(newnp->pktoptions); 1558 } 1559 } else { 1560 if (!req_unhash && found_dup_sk) { 1561 /* This code path should only be executed in the 1562 * syncookie case only 1563 */ 1564 bh_unlock_sock(newsk); 1565 sock_put(newsk); 1566 newsk = NULL; 1567 } 1568 } 1569 1570 return newsk; 1571 1572 out_overflow: 1573 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS); 1574 out_nonewsk: 1575 dst_release(dst); 1576 out: 1577 tcp_listendrop(sk); 1578 return NULL; 1579 } 1580 1581 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 1582 u32)); 1583 /* The socket must have it's spinlock held when we get 1584 * here, unless it is a TCP_LISTEN socket. 1585 * 1586 * We have a potential double-lock case here, so even when 1587 * doing backlog processing we use the BH locking scheme. 1588 * This is because we cannot sleep with the original spinlock 1589 * held. 1590 */ 1591 INDIRECT_CALLABLE_SCOPE 1592 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) 1593 { 1594 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1595 struct sk_buff *opt_skb = NULL; 1596 enum skb_drop_reason reason; 1597 struct tcp_sock *tp; 1598 1599 /* Imagine: socket is IPv6. IPv4 packet arrives, 1600 goes to IPv4 receive handler and backlogged. 1601 From backlog it always goes here. Kerboom... 1602 Fortunately, tcp_rcv_established and rcv_established 1603 handle them correctly, but it is not case with 1604 tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK 1605 */ 1606 1607 if (skb->protocol == htons(ETH_P_IP)) 1608 return tcp_v4_do_rcv(sk, skb); 1609 1610 /* 1611 * socket locking is here for SMP purposes as backlog rcv 1612 * is currently called with bh processing disabled. 1613 */ 1614 1615 /* Do Stevens' IPV6_PKTOPTIONS. 1616 1617 Yes, guys, it is the only place in our code, where we 1618 may make it not affecting IPv4. 1619 The rest of code is protocol independent, 1620 and I do not like idea to uglify IPv4. 1621 1622 Actually, all the idea behind IPV6_PKTOPTIONS 1623 looks not very well thought. For now we latch 1624 options, received in the last packet, enqueued 1625 by tcp. Feel free to propose better solution. 1626 --ANK (980728) 1627 */ 1628 if (np->rxopt.all && sk->sk_state != TCP_LISTEN) 1629 opt_skb = skb_clone_and_charge_r(skb, sk); 1630 1631 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */ 1632 struct dst_entry *dst; 1633 1634 dst = rcu_dereference_protected(sk->sk_rx_dst, 1635 lockdep_sock_is_held(sk)); 1636 1637 sock_rps_save_rxhash(sk, skb); 1638 sk_mark_napi_id(sk, skb); 1639 if (dst) { 1640 if (sk->sk_rx_dst_ifindex != skb->skb_iif || 1641 INDIRECT_CALL_1(dst->ops->check, ip6_dst_check, 1642 dst, sk->sk_rx_dst_cookie) == NULL) { 1643 RCU_INIT_POINTER(sk->sk_rx_dst, NULL); 1644 dst_release(dst); 1645 } 1646 } 1647 1648 tcp_rcv_established(sk, skb); 1649 if (opt_skb) 1650 goto ipv6_pktoptions; 1651 return 0; 1652 } 1653 1654 if (tcp_checksum_complete(skb)) 1655 goto csum_err; 1656 1657 if (sk->sk_state == TCP_LISTEN) { 1658 struct sock *nsk = tcp_v6_cookie_check(sk, skb); 1659 1660 if (nsk != sk) { 1661 if (nsk) { 1662 reason = tcp_child_process(sk, nsk, skb); 1663 if (reason) 1664 goto reset; 1665 } 1666 return 0; 1667 } 1668 } else 1669 sock_rps_save_rxhash(sk, skb); 1670 1671 reason = tcp_rcv_state_process(sk, skb); 1672 if (reason) 1673 goto reset; 1674 if (opt_skb) 1675 goto ipv6_pktoptions; 1676 return 0; 1677 1678 reset: 1679 tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason)); 1680 discard: 1681 if (opt_skb) 1682 __kfree_skb(opt_skb); 1683 sk_skb_reason_drop(sk, skb, reason); 1684 return 0; 1685 csum_err: 1686 reason = SKB_DROP_REASON_TCP_CSUM; 1687 trace_tcp_bad_csum(skb); 1688 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS); 1689 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS); 1690 goto discard; 1691 1692 1693 ipv6_pktoptions: 1694 /* Do you ask, what is it? 1695 1696 1. skb was enqueued by tcp. 1697 2. skb is added to tail of read queue, rather than out of order. 1698 3. socket is not in passive state. 1699 4. Finally, it really contains options, which user wants to receive. 1700 */ 1701 tp = tcp_sk(sk); 1702 if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt && 1703 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 1704 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo) 1705 WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb)); 1706 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim) 1707 WRITE_ONCE(np->mcast_hops, 1708 ipv6_hdr(opt_skb)->hop_limit); 1709 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass) 1710 np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb)); 1711 if (inet6_test_bit(REPFLOW, sk)) 1712 np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb)); 1713 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) { 1714 tcp_v6_restore_cb(opt_skb); 1715 opt_skb = xchg(&np->pktoptions, opt_skb); 1716 } else { 1717 __kfree_skb(opt_skb); 1718 opt_skb = xchg(&np->pktoptions, NULL); 1719 } 1720 } 1721 1722 consume_skb(opt_skb); 1723 return 0; 1724 } 1725 1726 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr, 1727 const struct tcphdr *th) 1728 { 1729 /* This is tricky: we move IP6CB at its correct location into 1730 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because 1731 * _decode_session6() uses IP6CB(). 1732 * barrier() makes sure compiler won't play aliasing games. 1733 */ 1734 memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb), 1735 sizeof(struct inet6_skb_parm)); 1736 barrier(); 1737 1738 TCP_SKB_CB(skb)->seq = ntohl(th->seq); 1739 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin + 1740 skb->len - th->doff*4); 1741 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq); 1742 TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th); 1743 TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr); 1744 TCP_SKB_CB(skb)->sacked = 0; 1745 TCP_SKB_CB(skb)->has_rxtstamp = 1746 skb->tstamp || skb_hwtstamps(skb)->hwtstamp; 1747 } 1748 1749 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb) 1750 { 1751 struct net *net = dev_net_rcu(skb->dev); 1752 enum skb_drop_reason drop_reason; 1753 enum tcp_tw_status tw_status; 1754 int sdif = inet6_sdif(skb); 1755 int dif = inet6_iif(skb); 1756 const struct tcphdr *th; 1757 const struct ipv6hdr *hdr; 1758 struct sock *sk = NULL; 1759 bool refcounted; 1760 int ret; 1761 u32 isn; 1762 1763 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 1764 if (skb->pkt_type != PACKET_HOST) 1765 goto discard_it; 1766 1767 /* 1768 * Count it even if it's bad. 1769 */ 1770 __TCP_INC_STATS(net, TCP_MIB_INSEGS); 1771 1772 if (!pskb_may_pull(skb, sizeof(struct tcphdr))) 1773 goto discard_it; 1774 1775 th = (const struct tcphdr *)skb->data; 1776 1777 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) { 1778 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL; 1779 goto bad_packet; 1780 } 1781 if (!pskb_may_pull(skb, th->doff*4)) 1782 goto discard_it; 1783 1784 if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo)) 1785 goto csum_error; 1786 1787 th = (const struct tcphdr *)skb->data; 1788 hdr = ipv6_hdr(skb); 1789 1790 lookup: 1791 sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th), 1792 th->source, th->dest, inet6_iif(skb), sdif, 1793 &refcounted); 1794 if (!sk) 1795 goto no_tcp_socket; 1796 1797 if (sk->sk_state == TCP_TIME_WAIT) 1798 goto do_time_wait; 1799 1800 if (sk->sk_state == TCP_NEW_SYN_RECV) { 1801 struct request_sock *req = inet_reqsk(sk); 1802 bool req_stolen = false; 1803 struct sock *nsk; 1804 1805 sk = req->rsk_listener; 1806 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) 1807 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1808 else 1809 drop_reason = tcp_inbound_hash(sk, req, skb, 1810 &hdr->saddr, &hdr->daddr, 1811 AF_INET6, dif, sdif); 1812 if (drop_reason) { 1813 sk_drops_add(sk, skb); 1814 reqsk_put(req); 1815 goto discard_it; 1816 } 1817 if (tcp_checksum_complete(skb)) { 1818 reqsk_put(req); 1819 goto csum_error; 1820 } 1821 if (unlikely(sk->sk_state != TCP_LISTEN)) { 1822 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb); 1823 if (!nsk) { 1824 inet_csk_reqsk_queue_drop_and_put(sk, req); 1825 goto lookup; 1826 } 1827 sk = nsk; 1828 /* reuseport_migrate_sock() has already held one sk_refcnt 1829 * before returning. 1830 */ 1831 } else { 1832 sock_hold(sk); 1833 } 1834 refcounted = true; 1835 nsk = NULL; 1836 if (!tcp_filter(sk, skb)) { 1837 th = (const struct tcphdr *)skb->data; 1838 hdr = ipv6_hdr(skb); 1839 tcp_v6_fill_cb(skb, hdr, th); 1840 nsk = tcp_check_req(sk, skb, req, false, &req_stolen, 1841 &drop_reason); 1842 } else { 1843 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 1844 } 1845 if (!nsk) { 1846 reqsk_put(req); 1847 if (req_stolen) { 1848 /* Another cpu got exclusive access to req 1849 * and created a full blown socket. 1850 * Try to feed this packet to this socket 1851 * instead of discarding it. 1852 */ 1853 tcp_v6_restore_cb(skb); 1854 sock_put(sk); 1855 goto lookup; 1856 } 1857 goto discard_and_relse; 1858 } 1859 nf_reset_ct(skb); 1860 if (nsk == sk) { 1861 reqsk_put(req); 1862 tcp_v6_restore_cb(skb); 1863 } else { 1864 drop_reason = tcp_child_process(sk, nsk, skb); 1865 if (drop_reason) { 1866 enum sk_rst_reason rst_reason; 1867 1868 rst_reason = sk_rst_convert_drop_reason(drop_reason); 1869 tcp_v6_send_reset(nsk, skb, rst_reason); 1870 goto discard_and_relse; 1871 } 1872 sock_put(sk); 1873 return 0; 1874 } 1875 } 1876 1877 process: 1878 if (static_branch_unlikely(&ip6_min_hopcount)) { 1879 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 1880 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) { 1881 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 1882 drop_reason = SKB_DROP_REASON_TCP_MINTTL; 1883 goto discard_and_relse; 1884 } 1885 } 1886 1887 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) { 1888 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1889 goto discard_and_relse; 1890 } 1891 1892 drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr, 1893 AF_INET6, dif, sdif); 1894 if (drop_reason) 1895 goto discard_and_relse; 1896 1897 nf_reset_ct(skb); 1898 1899 if (tcp_filter(sk, skb)) { 1900 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 1901 goto discard_and_relse; 1902 } 1903 th = (const struct tcphdr *)skb->data; 1904 hdr = ipv6_hdr(skb); 1905 tcp_v6_fill_cb(skb, hdr, th); 1906 1907 skb->dev = NULL; 1908 1909 if (sk->sk_state == TCP_LISTEN) { 1910 ret = tcp_v6_do_rcv(sk, skb); 1911 goto put_and_return; 1912 } 1913 1914 sk_incoming_cpu_update(sk); 1915 1916 bh_lock_sock_nested(sk); 1917 tcp_segs_in(tcp_sk(sk), skb); 1918 ret = 0; 1919 if (!sock_owned_by_user(sk)) { 1920 ret = tcp_v6_do_rcv(sk, skb); 1921 } else { 1922 if (tcp_add_backlog(sk, skb, &drop_reason)) 1923 goto discard_and_relse; 1924 } 1925 bh_unlock_sock(sk); 1926 put_and_return: 1927 if (refcounted) 1928 sock_put(sk); 1929 return ret ? -1 : 0; 1930 1931 no_tcp_socket: 1932 drop_reason = SKB_DROP_REASON_NO_SOCKET; 1933 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) 1934 goto discard_it; 1935 1936 tcp_v6_fill_cb(skb, hdr, th); 1937 1938 if (tcp_checksum_complete(skb)) { 1939 csum_error: 1940 drop_reason = SKB_DROP_REASON_TCP_CSUM; 1941 trace_tcp_bad_csum(skb); 1942 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS); 1943 bad_packet: 1944 __TCP_INC_STATS(net, TCP_MIB_INERRS); 1945 } else { 1946 tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason)); 1947 } 1948 1949 discard_it: 1950 SKB_DR_OR(drop_reason, NOT_SPECIFIED); 1951 sk_skb_reason_drop(sk, skb, drop_reason); 1952 return 0; 1953 1954 discard_and_relse: 1955 sk_drops_add(sk, skb); 1956 if (refcounted) 1957 sock_put(sk); 1958 goto discard_it; 1959 1960 do_time_wait: 1961 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) { 1962 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1963 inet_twsk_put(inet_twsk(sk)); 1964 goto discard_it; 1965 } 1966 1967 tcp_v6_fill_cb(skb, hdr, th); 1968 1969 if (tcp_checksum_complete(skb)) { 1970 inet_twsk_put(inet_twsk(sk)); 1971 goto csum_error; 1972 } 1973 1974 tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn, 1975 &drop_reason); 1976 switch (tw_status) { 1977 case TCP_TW_SYN: 1978 { 1979 struct sock *sk2; 1980 1981 sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, 1982 skb, __tcp_hdrlen(th), 1983 &ipv6_hdr(skb)->saddr, th->source, 1984 &ipv6_hdr(skb)->daddr, 1985 ntohs(th->dest), 1986 tcp_v6_iif_l3_slave(skb), 1987 sdif); 1988 if (sk2) { 1989 struct inet_timewait_sock *tw = inet_twsk(sk); 1990 inet_twsk_deschedule_put(tw); 1991 sk = sk2; 1992 tcp_v6_restore_cb(skb); 1993 refcounted = false; 1994 __this_cpu_write(tcp_tw_isn, isn); 1995 goto process; 1996 } 1997 } 1998 /* to ACK */ 1999 fallthrough; 2000 case TCP_TW_ACK: 2001 case TCP_TW_ACK_OOW: 2002 tcp_v6_timewait_ack(sk, skb, tw_status); 2003 break; 2004 case TCP_TW_RST: 2005 tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET); 2006 inet_twsk_deschedule_put(inet_twsk(sk)); 2007 goto discard_it; 2008 case TCP_TW_SUCCESS: 2009 ; 2010 } 2011 goto discard_it; 2012 } 2013 2014 void tcp_v6_early_demux(struct sk_buff *skb) 2015 { 2016 struct net *net = dev_net_rcu(skb->dev); 2017 const struct ipv6hdr *hdr; 2018 const struct tcphdr *th; 2019 struct sock *sk; 2020 2021 if (skb->pkt_type != PACKET_HOST) 2022 return; 2023 2024 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr))) 2025 return; 2026 2027 hdr = ipv6_hdr(skb); 2028 th = tcp_hdr(skb); 2029 2030 if (th->doff < sizeof(struct tcphdr) / 4) 2031 return; 2032 2033 /* Note : We use inet6_iif() here, not tcp_v6_iif() */ 2034 sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 2035 &hdr->saddr, th->source, 2036 &hdr->daddr, ntohs(th->dest), 2037 inet6_iif(skb), inet6_sdif(skb)); 2038 if (sk) { 2039 skb->sk = sk; 2040 skb->destructor = sock_edemux; 2041 if (sk_fullsock(sk)) { 2042 struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst); 2043 2044 if (dst) 2045 dst = dst_check(dst, sk->sk_rx_dst_cookie); 2046 if (dst && 2047 sk->sk_rx_dst_ifindex == skb->skb_iif) 2048 skb_dst_set_noref(skb, dst); 2049 } 2050 } 2051 } 2052 2053 static struct timewait_sock_ops tcp6_timewait_sock_ops = { 2054 .twsk_obj_size = sizeof(struct tcp6_timewait_sock), 2055 .twsk_destructor = tcp_twsk_destructor, 2056 }; 2057 2058 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) 2059 { 2060 __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr); 2061 } 2062 2063 const struct inet_connection_sock_af_ops ipv6_specific = { 2064 .queue_xmit = inet6_csk_xmit, 2065 .send_check = tcp_v6_send_check, 2066 .rebuild_header = inet6_sk_rebuild_header, 2067 .sk_rx_dst_set = inet6_sk_rx_dst_set, 2068 .conn_request = tcp_v6_conn_request, 2069 .syn_recv_sock = tcp_v6_syn_recv_sock, 2070 .net_header_len = sizeof(struct ipv6hdr), 2071 .setsockopt = ipv6_setsockopt, 2072 .getsockopt = ipv6_getsockopt, 2073 .mtu_reduced = tcp_v6_mtu_reduced, 2074 }; 2075 2076 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2077 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = { 2078 #ifdef CONFIG_TCP_MD5SIG 2079 .md5_lookup = tcp_v6_md5_lookup, 2080 .calc_md5_hash = tcp_v6_md5_hash_skb, 2081 .md5_parse = tcp_v6_parse_md5_keys, 2082 #endif 2083 #ifdef CONFIG_TCP_AO 2084 .ao_lookup = tcp_v6_ao_lookup, 2085 .calc_ao_hash = tcp_v6_ao_hash_skb, 2086 .ao_parse = tcp_v6_parse_ao, 2087 .ao_calc_key_sk = tcp_v6_ao_calc_key_sk, 2088 #endif 2089 }; 2090 #endif 2091 2092 /* 2093 * TCP over IPv4 via INET6 API 2094 */ 2095 static const struct inet_connection_sock_af_ops ipv6_mapped = { 2096 .queue_xmit = ip_queue_xmit, 2097 .send_check = tcp_v4_send_check, 2098 .rebuild_header = inet_sk_rebuild_header, 2099 .sk_rx_dst_set = inet_sk_rx_dst_set, 2100 .conn_request = tcp_v6_conn_request, 2101 .syn_recv_sock = tcp_v6_syn_recv_sock, 2102 .net_header_len = sizeof(struct iphdr), 2103 .setsockopt = ipv6_setsockopt, 2104 .getsockopt = ipv6_getsockopt, 2105 .mtu_reduced = tcp_v4_mtu_reduced, 2106 }; 2107 2108 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2109 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = { 2110 #ifdef CONFIG_TCP_MD5SIG 2111 .md5_lookup = tcp_v4_md5_lookup, 2112 .calc_md5_hash = tcp_v4_md5_hash_skb, 2113 .md5_parse = tcp_v6_parse_md5_keys, 2114 #endif 2115 #ifdef CONFIG_TCP_AO 2116 .ao_lookup = tcp_v6_ao_lookup, 2117 .calc_ao_hash = tcp_v4_ao_hash_skb, 2118 .ao_parse = tcp_v6_parse_ao, 2119 .ao_calc_key_sk = tcp_v4_ao_calc_key_sk, 2120 #endif 2121 }; 2122 #endif 2123 2124 /* NOTE: A lot of things set to zero explicitly by call to 2125 * sk_alloc() so need not be done here. 2126 */ 2127 static int tcp_v6_init_sock(struct sock *sk) 2128 { 2129 struct inet_connection_sock *icsk = inet_csk(sk); 2130 2131 tcp_init_sock(sk); 2132 2133 icsk->icsk_af_ops = &ipv6_specific; 2134 2135 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2136 tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific; 2137 #endif 2138 2139 return 0; 2140 } 2141 2142 #ifdef CONFIG_PROC_FS 2143 /* Proc filesystem TCPv6 sock list dumping. */ 2144 static void get_openreq6(struct seq_file *seq, 2145 const struct request_sock *req, int i) 2146 { 2147 long ttd = req->rsk_timer.expires - jiffies; 2148 const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr; 2149 const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr; 2150 2151 if (ttd < 0) 2152 ttd = 0; 2153 2154 seq_printf(seq, 2155 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2156 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n", 2157 i, 2158 src->s6_addr32[0], src->s6_addr32[1], 2159 src->s6_addr32[2], src->s6_addr32[3], 2160 inet_rsk(req)->ir_num, 2161 dest->s6_addr32[0], dest->s6_addr32[1], 2162 dest->s6_addr32[2], dest->s6_addr32[3], 2163 ntohs(inet_rsk(req)->ir_rmt_port), 2164 TCP_SYN_RECV, 2165 0, 0, /* could print option size, but that is af dependent. */ 2166 1, /* timers active (only the expire timer) */ 2167 jiffies_to_clock_t(ttd), 2168 req->num_timeout, 2169 from_kuid_munged(seq_user_ns(seq), 2170 sk_uid(req->rsk_listener)), 2171 0, /* non standard timer */ 2172 0, /* open_requests have no inode */ 2173 0, req); 2174 } 2175 2176 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i) 2177 { 2178 const struct in6_addr *dest, *src; 2179 __u16 destp, srcp; 2180 int timer_active; 2181 unsigned long timer_expires; 2182 const struct inet_sock *inet = inet_sk(sp); 2183 const struct tcp_sock *tp = tcp_sk(sp); 2184 const struct inet_connection_sock *icsk = inet_csk(sp); 2185 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq; 2186 u8 icsk_pending; 2187 int rx_queue; 2188 int state; 2189 2190 dest = &sp->sk_v6_daddr; 2191 src = &sp->sk_v6_rcv_saddr; 2192 destp = ntohs(inet->inet_dport); 2193 srcp = ntohs(inet->inet_sport); 2194 2195 icsk_pending = smp_load_acquire(&icsk->icsk_pending); 2196 if (icsk_pending == ICSK_TIME_RETRANS || 2197 icsk_pending == ICSK_TIME_REO_TIMEOUT || 2198 icsk_pending == ICSK_TIME_LOSS_PROBE) { 2199 timer_active = 1; 2200 timer_expires = icsk_timeout(icsk); 2201 } else if (icsk_pending == ICSK_TIME_PROBE0) { 2202 timer_active = 4; 2203 timer_expires = icsk_timeout(icsk); 2204 } else if (timer_pending(&sp->sk_timer)) { 2205 timer_active = 2; 2206 timer_expires = sp->sk_timer.expires; 2207 } else { 2208 timer_active = 0; 2209 timer_expires = jiffies; 2210 } 2211 2212 state = inet_sk_state_load(sp); 2213 if (state == TCP_LISTEN) 2214 rx_queue = READ_ONCE(sp->sk_ack_backlog); 2215 else 2216 /* Because we don't lock the socket, 2217 * we might find a transient negative value. 2218 */ 2219 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) - 2220 READ_ONCE(tp->copied_seq), 0); 2221 2222 seq_printf(seq, 2223 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2224 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n", 2225 i, 2226 src->s6_addr32[0], src->s6_addr32[1], 2227 src->s6_addr32[2], src->s6_addr32[3], srcp, 2228 dest->s6_addr32[0], dest->s6_addr32[1], 2229 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2230 state, 2231 READ_ONCE(tp->write_seq) - tp->snd_una, 2232 rx_queue, 2233 timer_active, 2234 jiffies_delta_to_clock_t(timer_expires - jiffies), 2235 icsk->icsk_retransmits, 2236 from_kuid_munged(seq_user_ns(seq), sk_uid(sp)), 2237 icsk->icsk_probes_out, 2238 sock_i_ino(sp), 2239 refcount_read(&sp->sk_refcnt), sp, 2240 jiffies_to_clock_t(icsk->icsk_rto), 2241 jiffies_to_clock_t(icsk->icsk_ack.ato), 2242 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp), 2243 tcp_snd_cwnd(tp), 2244 state == TCP_LISTEN ? 2245 fastopenq->max_qlen : 2246 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh) 2247 ); 2248 } 2249 2250 static void get_timewait6_sock(struct seq_file *seq, 2251 struct inet_timewait_sock *tw, int i) 2252 { 2253 long delta = tw->tw_timer.expires - jiffies; 2254 const struct in6_addr *dest, *src; 2255 __u16 destp, srcp; 2256 2257 dest = &tw->tw_v6_daddr; 2258 src = &tw->tw_v6_rcv_saddr; 2259 destp = ntohs(tw->tw_dport); 2260 srcp = ntohs(tw->tw_sport); 2261 2262 seq_printf(seq, 2263 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2264 "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", 2265 i, 2266 src->s6_addr32[0], src->s6_addr32[1], 2267 src->s6_addr32[2], src->s6_addr32[3], srcp, 2268 dest->s6_addr32[0], dest->s6_addr32[1], 2269 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2270 READ_ONCE(tw->tw_substate), 0, 0, 2271 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, 2272 refcount_read(&tw->tw_refcnt), tw); 2273 } 2274 2275 static int tcp6_seq_show(struct seq_file *seq, void *v) 2276 { 2277 struct tcp_iter_state *st; 2278 struct sock *sk = v; 2279 2280 if (v == SEQ_START_TOKEN) { 2281 seq_puts(seq, 2282 " sl " 2283 "local_address " 2284 "remote_address " 2285 "st tx_queue rx_queue tr tm->when retrnsmt" 2286 " uid timeout inode\n"); 2287 goto out; 2288 } 2289 st = seq->private; 2290 2291 if (sk->sk_state == TCP_TIME_WAIT) 2292 get_timewait6_sock(seq, v, st->num); 2293 else if (sk->sk_state == TCP_NEW_SYN_RECV) 2294 get_openreq6(seq, v, st->num); 2295 else 2296 get_tcp6_sock(seq, v, st->num); 2297 out: 2298 return 0; 2299 } 2300 2301 static const struct seq_operations tcp6_seq_ops = { 2302 .show = tcp6_seq_show, 2303 .start = tcp_seq_start, 2304 .next = tcp_seq_next, 2305 .stop = tcp_seq_stop, 2306 }; 2307 2308 static struct tcp_seq_afinfo tcp6_seq_afinfo = { 2309 .family = AF_INET6, 2310 }; 2311 2312 int __net_init tcp6_proc_init(struct net *net) 2313 { 2314 if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops, 2315 sizeof(struct tcp_iter_state), &tcp6_seq_afinfo)) 2316 return -ENOMEM; 2317 return 0; 2318 } 2319 2320 void tcp6_proc_exit(struct net *net) 2321 { 2322 remove_proc_entry("tcp6", net->proc_net); 2323 } 2324 #endif 2325 2326 struct proto tcpv6_prot = { 2327 .name = "TCPv6", 2328 .owner = THIS_MODULE, 2329 .close = tcp_close, 2330 .pre_connect = tcp_v6_pre_connect, 2331 .connect = tcp_v6_connect, 2332 .disconnect = tcp_disconnect, 2333 .accept = inet_csk_accept, 2334 .ioctl = tcp_ioctl, 2335 .init = tcp_v6_init_sock, 2336 .destroy = tcp_v4_destroy_sock, 2337 .shutdown = tcp_shutdown, 2338 .setsockopt = tcp_setsockopt, 2339 .getsockopt = tcp_getsockopt, 2340 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, 2341 .keepalive = tcp_set_keepalive, 2342 .recvmsg = tcp_recvmsg, 2343 .sendmsg = tcp_sendmsg, 2344 .splice_eof = tcp_splice_eof, 2345 .backlog_rcv = tcp_v6_do_rcv, 2346 .release_cb = tcp_release_cb, 2347 .hash = inet6_hash, 2348 .unhash = inet_unhash, 2349 .get_port = inet_csk_get_port, 2350 .put_port = inet_put_port, 2351 #ifdef CONFIG_BPF_SYSCALL 2352 .psock_update_sk_prot = tcp_bpf_update_proto, 2353 #endif 2354 .enter_memory_pressure = tcp_enter_memory_pressure, 2355 .leave_memory_pressure = tcp_leave_memory_pressure, 2356 .stream_memory_free = tcp_stream_memory_free, 2357 .sockets_allocated = &tcp_sockets_allocated, 2358 2359 .memory_allocated = &tcp_memory_allocated, 2360 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 2361 2362 .memory_pressure = &tcp_memory_pressure, 2363 .orphan_count = &tcp_orphan_count, 2364 .sysctl_mem = sysctl_tcp_mem, 2365 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 2366 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 2367 .max_header = MAX_TCP_HEADER, 2368 .obj_size = sizeof(struct tcp6_sock), 2369 .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6), 2370 .slab_flags = SLAB_TYPESAFE_BY_RCU, 2371 .twsk_prot = &tcp6_timewait_sock_ops, 2372 .rsk_prot = &tcp6_request_sock_ops, 2373 .h.hashinfo = NULL, 2374 .no_autobind = true, 2375 .diag_destroy = tcp_abort, 2376 }; 2377 EXPORT_SYMBOL_GPL(tcpv6_prot); 2378 2379 2380 static struct inet_protosw tcpv6_protosw = { 2381 .type = SOCK_STREAM, 2382 .protocol = IPPROTO_TCP, 2383 .prot = &tcpv6_prot, 2384 .ops = &inet6_stream_ops, 2385 .flags = INET_PROTOSW_PERMANENT | 2386 INET_PROTOSW_ICSK, 2387 }; 2388 2389 static int __net_init tcpv6_net_init(struct net *net) 2390 { 2391 int res; 2392 2393 res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6, 2394 SOCK_RAW, IPPROTO_TCP, net); 2395 if (!res) 2396 net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC; 2397 2398 return res; 2399 } 2400 2401 static void __net_exit tcpv6_net_exit(struct net *net) 2402 { 2403 inet_ctl_sock_destroy(net->ipv6.tcp_sk); 2404 } 2405 2406 static struct pernet_operations tcpv6_net_ops = { 2407 .init = tcpv6_net_init, 2408 .exit = tcpv6_net_exit, 2409 }; 2410 2411 int __init tcpv6_init(void) 2412 { 2413 int ret; 2414 2415 net_hotdata.tcpv6_protocol = (struct inet6_protocol) { 2416 .handler = tcp_v6_rcv, 2417 .err_handler = tcp_v6_err, 2418 .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL, 2419 }; 2420 ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2421 if (ret) 2422 goto out; 2423 2424 /* register inet6 protocol */ 2425 ret = inet6_register_protosw(&tcpv6_protosw); 2426 if (ret) 2427 goto out_tcpv6_protocol; 2428 2429 ret = register_pernet_subsys(&tcpv6_net_ops); 2430 if (ret) 2431 goto out_tcpv6_protosw; 2432 2433 ret = mptcpv6_init(); 2434 if (ret) 2435 goto out_tcpv6_pernet_subsys; 2436 2437 out: 2438 return ret; 2439 2440 out_tcpv6_pernet_subsys: 2441 unregister_pernet_subsys(&tcpv6_net_ops); 2442 out_tcpv6_protosw: 2443 inet6_unregister_protosw(&tcpv6_protosw); 2444 out_tcpv6_protocol: 2445 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2446 goto out; 2447 } 2448 2449 void tcpv6_exit(void) 2450 { 2451 unregister_pernet_subsys(&tcpv6_net_ops); 2452 inet6_unregister_protosw(&tcpv6_protosw); 2453 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2454 } 2455