1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TCP over IPv6 4 * Linux INET6 implementation 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 * 9 * Based on: 10 * linux/net/ipv4/tcp.c 11 * linux/net/ipv4/tcp_input.c 12 * linux/net/ipv4/tcp_output.c 13 * 14 * Fixes: 15 * Hideaki YOSHIFUJI : sin6_scope_id support 16 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which 17 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind 18 * a single port at the same time. 19 * YOSHIFUJI Hideaki @USAGI: convert /proc/net/tcp6 to seq_file. 20 */ 21 22 #include <linux/bottom_half.h> 23 #include <linux/module.h> 24 #include <linux/errno.h> 25 #include <linux/types.h> 26 #include <linux/socket.h> 27 #include <linux/sockios.h> 28 #include <linux/net.h> 29 #include <linux/jiffies.h> 30 #include <linux/in.h> 31 #include <linux/in6.h> 32 #include <linux/netdevice.h> 33 #include <linux/init.h> 34 #include <linux/jhash.h> 35 #include <linux/ipsec.h> 36 #include <linux/times.h> 37 #include <linux/slab.h> 38 #include <linux/uaccess.h> 39 #include <linux/ipv6.h> 40 #include <linux/icmpv6.h> 41 #include <linux/random.h> 42 #include <linux/indirect_call_wrapper.h> 43 44 #include <net/aligned_data.h> 45 #include <net/tcp.h> 46 #include <net/ndisc.h> 47 #include <net/inet6_hashtables.h> 48 #include <net/inet6_connection_sock.h> 49 #include <net/ipv6.h> 50 #include <net/transp_v6.h> 51 #include <net/addrconf.h> 52 #include <net/ip6_route.h> 53 #include <net/ip6_checksum.h> 54 #include <net/inet_ecn.h> 55 #include <net/protocol.h> 56 #include <net/xfrm.h> 57 #include <net/snmp.h> 58 #include <net/dsfield.h> 59 #include <net/timewait_sock.h> 60 #include <net/inet_common.h> 61 #include <net/secure_seq.h> 62 #include <net/hotdata.h> 63 #include <net/busy_poll.h> 64 #include <net/rstreason.h> 65 66 #include <linux/proc_fs.h> 67 #include <linux/seq_file.h> 68 69 #include <crypto/hash.h> 70 #include <linux/scatterlist.h> 71 72 #include <trace/events/tcp.h> 73 74 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 75 enum sk_rst_reason reason); 76 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 77 struct request_sock *req); 78 79 INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb); 80 81 static const struct inet_connection_sock_af_ops ipv6_mapped; 82 const struct inet_connection_sock_af_ops ipv6_specific; 83 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 84 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific; 85 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific; 86 #endif 87 88 /* Helper returning the inet6 address from a given tcp socket. 89 * It can be used in TCP stack instead of inet6_sk(sk). 90 * This avoids a dereference and allow compiler optimizations. 91 * It is a specialized version of inet6_sk_generic(). 92 */ 93 #define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \ 94 struct tcp6_sock, tcp)->inet6) 95 96 static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 97 { 98 struct dst_entry *dst = skb_dst(skb); 99 100 if (dst && dst_hold_safe(dst)) { 101 rcu_assign_pointer(sk->sk_rx_dst, dst); 102 sk->sk_rx_dst_ifindex = skb->skb_iif; 103 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst)); 104 } 105 } 106 107 static u32 tcp_v6_init_seq(const struct sk_buff *skb) 108 { 109 return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32, 110 ipv6_hdr(skb)->saddr.s6_addr32, 111 tcp_hdr(skb)->dest, 112 tcp_hdr(skb)->source); 113 } 114 115 static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb) 116 { 117 return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32, 118 ipv6_hdr(skb)->saddr.s6_addr32); 119 } 120 121 static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr, 122 int addr_len) 123 { 124 /* This check is replicated from tcp_v6_connect() and intended to 125 * prevent BPF program called below from accessing bytes that are out 126 * of the bound specified by user in addr_len. 127 */ 128 if (addr_len < SIN6_LEN_RFC2133) 129 return -EINVAL; 130 131 sock_owned_by_me(sk); 132 133 return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len); 134 } 135 136 static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, 137 int addr_len) 138 { 139 struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr; 140 struct inet_connection_sock *icsk = inet_csk(sk); 141 struct in6_addr *saddr = NULL, *final_p, final; 142 struct inet_timewait_death_row *tcp_death_row; 143 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 144 struct inet_sock *inet = inet_sk(sk); 145 struct tcp_sock *tp = tcp_sk(sk); 146 struct net *net = sock_net(sk); 147 struct ipv6_txoptions *opt; 148 struct dst_entry *dst; 149 struct flowi6 fl6; 150 int addr_type; 151 int err; 152 153 if (addr_len < SIN6_LEN_RFC2133) 154 return -EINVAL; 155 156 if (usin->sin6_family != AF_INET6) 157 return -EAFNOSUPPORT; 158 159 memset(&fl6, 0, sizeof(fl6)); 160 161 if (inet6_test_bit(SNDFLOW, sk)) { 162 fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK; 163 IP6_ECN_flow_init(fl6.flowlabel); 164 if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) { 165 struct ip6_flowlabel *flowlabel; 166 flowlabel = fl6_sock_lookup(sk, fl6.flowlabel); 167 if (IS_ERR(flowlabel)) 168 return -EINVAL; 169 fl6_sock_release(flowlabel); 170 } 171 } 172 173 /* 174 * connect() to INADDR_ANY means loopback (BSD'ism). 175 */ 176 177 if (ipv6_addr_any(&usin->sin6_addr)) { 178 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) 179 ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK), 180 &usin->sin6_addr); 181 else 182 usin->sin6_addr = in6addr_loopback; 183 } 184 185 addr_type = ipv6_addr_type(&usin->sin6_addr); 186 187 if (addr_type & IPV6_ADDR_MULTICAST) 188 return -ENETUNREACH; 189 190 if (addr_type&IPV6_ADDR_LINKLOCAL) { 191 if (addr_len >= sizeof(struct sockaddr_in6) && 192 usin->sin6_scope_id) { 193 /* If interface is set while binding, indices 194 * must coincide. 195 */ 196 if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id)) 197 return -EINVAL; 198 199 sk->sk_bound_dev_if = usin->sin6_scope_id; 200 } 201 202 /* Connect to link-local address requires an interface */ 203 if (!sk->sk_bound_dev_if) 204 return -EINVAL; 205 } 206 207 if (tp->rx_opt.ts_recent_stamp && 208 !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) { 209 tp->rx_opt.ts_recent = 0; 210 tp->rx_opt.ts_recent_stamp = 0; 211 WRITE_ONCE(tp->write_seq, 0); 212 } 213 214 sk->sk_v6_daddr = usin->sin6_addr; 215 np->flow_label = fl6.flowlabel; 216 217 /* 218 * TCP over IPv4 219 */ 220 221 if (addr_type & IPV6_ADDR_MAPPED) { 222 u32 exthdrlen = icsk->icsk_ext_hdr_len; 223 struct sockaddr_in sin; 224 225 if (ipv6_only_sock(sk)) 226 return -ENETUNREACH; 227 228 sin.sin_family = AF_INET; 229 sin.sin_port = usin->sin6_port; 230 sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3]; 231 232 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 233 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped); 234 if (sk_is_mptcp(sk)) 235 mptcpv6_handle_mapped(sk, true); 236 sk->sk_backlog_rcv = tcp_v4_do_rcv; 237 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 238 tp->af_specific = &tcp_sock_ipv6_mapped_specific; 239 #endif 240 241 err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin)); 242 243 if (err) { 244 icsk->icsk_ext_hdr_len = exthdrlen; 245 /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ 246 WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific); 247 if (sk_is_mptcp(sk)) 248 mptcpv6_handle_mapped(sk, false); 249 sk->sk_backlog_rcv = tcp_v6_do_rcv; 250 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 251 tp->af_specific = &tcp_sock_ipv6_specific; 252 #endif 253 goto failure; 254 } 255 np->saddr = sk->sk_v6_rcv_saddr; 256 257 return err; 258 } 259 260 if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) 261 saddr = &sk->sk_v6_rcv_saddr; 262 263 fl6.flowi6_proto = IPPROTO_TCP; 264 fl6.daddr = sk->sk_v6_daddr; 265 fl6.saddr = saddr ? *saddr : np->saddr; 266 fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label); 267 fl6.flowi6_oif = sk->sk_bound_dev_if; 268 fl6.flowi6_mark = sk->sk_mark; 269 fl6.fl6_dport = usin->sin6_port; 270 fl6.fl6_sport = inet->inet_sport; 271 if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6.fl6_sport) 272 fl6.flowi6_flags = FLOWI_FLAG_ANY_SPORT; 273 fl6.flowi6_uid = sk_uid(sk); 274 275 opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk)); 276 final_p = fl6_update_dst(&fl6, opt, &final); 277 278 security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6)); 279 280 dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p); 281 if (IS_ERR(dst)) { 282 err = PTR_ERR(dst); 283 goto failure; 284 } 285 286 tp->tcp_usec_ts = dst_tcp_usec_ts(dst); 287 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row; 288 289 if (!saddr) { 290 saddr = &fl6.saddr; 291 292 err = inet_bhash2_update_saddr(sk, saddr, AF_INET6); 293 if (err) 294 goto failure; 295 } 296 297 /* set the source address */ 298 np->saddr = *saddr; 299 inet->inet_rcv_saddr = LOOPBACK4_IPV6; 300 301 sk->sk_gso_type = SKB_GSO_TCPV6; 302 ip6_dst_store(sk, dst, NULL, NULL); 303 304 icsk->icsk_ext_hdr_len = 0; 305 if (opt) 306 icsk->icsk_ext_hdr_len = opt->opt_flen + 307 opt->opt_nflen; 308 309 tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr); 310 311 inet->inet_dport = usin->sin6_port; 312 313 tcp_set_state(sk, TCP_SYN_SENT); 314 err = inet6_hash_connect(tcp_death_row, sk); 315 if (err) 316 goto late_failure; 317 318 sk_set_txhash(sk); 319 320 if (likely(!tp->repair)) { 321 if (!tp->write_seq) 322 WRITE_ONCE(tp->write_seq, 323 secure_tcpv6_seq(np->saddr.s6_addr32, 324 sk->sk_v6_daddr.s6_addr32, 325 inet->inet_sport, 326 inet->inet_dport)); 327 tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, 328 sk->sk_v6_daddr.s6_addr32); 329 } 330 331 if (tcp_fastopen_defer_connect(sk, &err)) 332 return err; 333 if (err) 334 goto late_failure; 335 336 err = tcp_connect(sk); 337 if (err) 338 goto late_failure; 339 340 return 0; 341 342 late_failure: 343 tcp_set_state(sk, TCP_CLOSE); 344 inet_bhash2_reset_saddr(sk); 345 failure: 346 inet->inet_dport = 0; 347 sk->sk_route_caps = 0; 348 return err; 349 } 350 351 static void tcp_v6_mtu_reduced(struct sock *sk) 352 { 353 struct dst_entry *dst; 354 u32 mtu; 355 356 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 357 return; 358 359 mtu = READ_ONCE(tcp_sk(sk)->mtu_info); 360 361 /* Drop requests trying to increase our current mss. 362 * Check done in __ip6_rt_update_pmtu() is too late. 363 */ 364 if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache) 365 return; 366 367 dst = inet6_csk_update_pmtu(sk, mtu); 368 if (!dst) 369 return; 370 371 if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) { 372 tcp_sync_mss(sk, dst_mtu(dst)); 373 tcp_simple_retransmit(sk); 374 } 375 } 376 377 static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 378 u8 type, u8 code, int offset, __be32 info) 379 { 380 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; 381 const struct tcphdr *th = (struct tcphdr *)(skb->data+offset); 382 struct net *net = dev_net_rcu(skb->dev); 383 struct request_sock *fastopen; 384 struct ipv6_pinfo *np; 385 struct tcp_sock *tp; 386 __u32 seq, snd_una; 387 struct sock *sk; 388 bool fatal; 389 int err; 390 391 sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 392 &hdr->daddr, th->dest, 393 &hdr->saddr, ntohs(th->source), 394 skb->dev->ifindex, inet6_sdif(skb)); 395 396 if (!sk) { 397 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), 398 ICMP6_MIB_INERRORS); 399 return -ENOENT; 400 } 401 402 if (sk->sk_state == TCP_TIME_WAIT) { 403 /* To increase the counter of ignored icmps for TCP-AO */ 404 tcp_ao_ignore_icmp(sk, AF_INET6, type, code); 405 inet_twsk_put(inet_twsk(sk)); 406 return 0; 407 } 408 seq = ntohl(th->seq); 409 fatal = icmpv6_err_convert(type, code, &err); 410 if (sk->sk_state == TCP_NEW_SYN_RECV) { 411 tcp_req_err(sk, seq, fatal); 412 return 0; 413 } 414 415 if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) { 416 sock_put(sk); 417 return 0; 418 } 419 420 bh_lock_sock(sk); 421 if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG) 422 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS); 423 424 if (sk->sk_state == TCP_CLOSE) 425 goto out; 426 427 if (static_branch_unlikely(&ip6_min_hopcount)) { 428 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 429 if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) { 430 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 431 goto out; 432 } 433 } 434 435 tp = tcp_sk(sk); 436 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */ 437 fastopen = rcu_dereference(tp->fastopen_rsk); 438 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una; 439 if (sk->sk_state != TCP_LISTEN && 440 !between(seq, snd_una, tp->snd_nxt)) { 441 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS); 442 goto out; 443 } 444 445 np = tcp_inet6_sk(sk); 446 447 if (type == NDISC_REDIRECT) { 448 if (!sock_owned_by_user(sk)) { 449 struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie); 450 451 if (dst) 452 dst->ops->redirect(dst, sk, skb); 453 } 454 goto out; 455 } 456 457 if (type == ICMPV6_PKT_TOOBIG) { 458 u32 mtu = ntohl(info); 459 460 /* We are not interested in TCP_LISTEN and open_requests 461 * (SYN-ACKs send out by Linux are always <576bytes so 462 * they should go through unfragmented). 463 */ 464 if (sk->sk_state == TCP_LISTEN) 465 goto out; 466 467 if (!ip6_sk_accept_pmtu(sk)) 468 goto out; 469 470 if (mtu < IPV6_MIN_MTU) 471 goto out; 472 473 WRITE_ONCE(tp->mtu_info, mtu); 474 475 if (!sock_owned_by_user(sk)) 476 tcp_v6_mtu_reduced(sk); 477 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, 478 &sk->sk_tsq_flags)) 479 sock_hold(sk); 480 goto out; 481 } 482 483 484 /* Might be for an request_sock */ 485 switch (sk->sk_state) { 486 case TCP_SYN_SENT: 487 case TCP_SYN_RECV: 488 /* Only in fast or simultaneous open. If a fast open socket is 489 * already accepted it is treated as a connected one below. 490 */ 491 if (fastopen && !fastopen->sk) 492 break; 493 494 ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th); 495 496 if (!sock_owned_by_user(sk)) 497 tcp_done_with_error(sk, err); 498 else 499 WRITE_ONCE(sk->sk_err_soft, err); 500 goto out; 501 case TCP_LISTEN: 502 break; 503 default: 504 /* check if this ICMP message allows revert of backoff. 505 * (see RFC 6069) 506 */ 507 if (!fastopen && type == ICMPV6_DEST_UNREACH && 508 code == ICMPV6_NOROUTE) 509 tcp_ld_RTO_revert(sk, seq); 510 } 511 512 if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) { 513 WRITE_ONCE(sk->sk_err, err); 514 sk_error_report(sk); 515 } else { 516 WRITE_ONCE(sk->sk_err_soft, err); 517 } 518 out: 519 bh_unlock_sock(sk); 520 sock_put(sk); 521 return 0; 522 } 523 524 525 static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 526 struct flowi *fl, 527 struct request_sock *req, 528 struct tcp_fastopen_cookie *foc, 529 enum tcp_synack_type synack_type, 530 struct sk_buff *syn_skb) 531 { 532 struct inet_request_sock *ireq = inet_rsk(req); 533 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 534 struct ipv6_txoptions *opt; 535 struct flowi6 *fl6 = &fl->u.ip6; 536 struct sk_buff *skb; 537 int err = -ENOMEM; 538 u8 tclass; 539 540 /* First, grab a route. */ 541 if (!dst && (dst = inet6_csk_route_req(sk, fl6, req, 542 IPPROTO_TCP)) == NULL) 543 goto done; 544 545 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb); 546 547 if (skb) { 548 __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr, 549 &ireq->ir_v6_rmt_addr); 550 551 fl6->daddr = ireq->ir_v6_rmt_addr; 552 if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts) 553 fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts)); 554 555 tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ? 556 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) | 557 (np->tclass & INET_ECN_MASK) : 558 np->tclass; 559 560 if (!INET_ECN_is_capable(tclass) && 561 tcp_bpf_ca_needs_ecn((struct sock *)req)) 562 tclass |= INET_ECN_ECT_0; 563 564 rcu_read_lock(); 565 opt = ireq->ipv6_opt; 566 if (!opt) 567 opt = rcu_dereference(np->opt); 568 err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark), 569 opt, tclass, READ_ONCE(sk->sk_priority)); 570 rcu_read_unlock(); 571 err = net_xmit_eval(err); 572 } 573 574 done: 575 return err; 576 } 577 578 579 static void tcp_v6_reqsk_destructor(struct request_sock *req) 580 { 581 kfree(inet_rsk(req)->ipv6_opt); 582 consume_skb(inet_rsk(req)->pktopts); 583 } 584 585 #ifdef CONFIG_TCP_MD5SIG 586 static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk, 587 const struct in6_addr *addr, 588 int l3index) 589 { 590 return tcp_md5_do_lookup(sk, l3index, 591 (union tcp_md5_addr *)addr, AF_INET6); 592 } 593 594 static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk, 595 const struct sock *addr_sk) 596 { 597 int l3index; 598 599 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), 600 addr_sk->sk_bound_dev_if); 601 return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr, 602 l3index); 603 } 604 605 static int tcp_v6_parse_md5_keys(struct sock *sk, int optname, 606 sockptr_t optval, int optlen) 607 { 608 struct tcp_md5sig cmd; 609 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr; 610 union tcp_ao_addr *addr; 611 int l3index = 0; 612 u8 prefixlen; 613 bool l3flag; 614 u8 flags; 615 616 if (optlen < sizeof(cmd)) 617 return -EINVAL; 618 619 if (copy_from_sockptr(&cmd, optval, sizeof(cmd))) 620 return -EFAULT; 621 622 if (sin6->sin6_family != AF_INET6) 623 return -EINVAL; 624 625 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 626 l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 627 628 if (optname == TCP_MD5SIG_EXT && 629 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) { 630 prefixlen = cmd.tcpm_prefixlen; 631 if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) && 632 prefixlen > 32)) 633 return -EINVAL; 634 } else { 635 prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128; 636 } 637 638 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex && 639 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) { 640 struct net_device *dev; 641 642 rcu_read_lock(); 643 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex); 644 if (dev && netif_is_l3_master(dev)) 645 l3index = dev->ifindex; 646 rcu_read_unlock(); 647 648 /* ok to reference set/not set outside of rcu; 649 * right now device MUST be an L3 master 650 */ 651 if (!dev || !l3index) 652 return -EINVAL; 653 } 654 655 if (!cmd.tcpm_keylen) { 656 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) 657 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3], 658 AF_INET, prefixlen, 659 l3index, flags); 660 return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr, 661 AF_INET6, prefixlen, l3index, flags); 662 } 663 664 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN) 665 return -EINVAL; 666 667 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) { 668 addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3]; 669 670 /* Don't allow keys for peers that have a matching TCP-AO key. 671 * See the comment in tcp_ao_add_cmd() 672 */ 673 if (tcp_ao_required(sk, addr, AF_INET, 674 l3flag ? l3index : -1, false)) 675 return -EKEYREJECTED; 676 return tcp_md5_do_add(sk, addr, 677 AF_INET, prefixlen, l3index, flags, 678 cmd.tcpm_key, cmd.tcpm_keylen); 679 } 680 681 addr = (union tcp_md5_addr *)&sin6->sin6_addr; 682 683 /* Don't allow keys for peers that have a matching TCP-AO key. 684 * See the comment in tcp_ao_add_cmd() 685 */ 686 if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false)) 687 return -EKEYREJECTED; 688 689 return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags, 690 cmd.tcpm_key, cmd.tcpm_keylen); 691 } 692 693 static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp, 694 const struct in6_addr *daddr, 695 const struct in6_addr *saddr, 696 const struct tcphdr *th, int nbytes) 697 { 698 struct tcp6_pseudohdr *bp; 699 struct scatterlist sg; 700 struct tcphdr *_th; 701 702 bp = hp->scratch; 703 /* 1. TCP pseudo-header (RFC2460) */ 704 bp->saddr = *saddr; 705 bp->daddr = *daddr; 706 bp->protocol = cpu_to_be32(IPPROTO_TCP); 707 bp->len = cpu_to_be32(nbytes); 708 709 _th = (struct tcphdr *)(bp + 1); 710 memcpy(_th, th, sizeof(*th)); 711 _th->check = 0; 712 713 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); 714 ahash_request_set_crypt(hp->req, &sg, NULL, 715 sizeof(*bp) + sizeof(*th)); 716 return crypto_ahash_update(hp->req); 717 } 718 719 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key, 720 const struct in6_addr *daddr, struct in6_addr *saddr, 721 const struct tcphdr *th) 722 { 723 struct tcp_sigpool hp; 724 725 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 726 goto clear_hash_nostart; 727 728 if (crypto_ahash_init(hp.req)) 729 goto clear_hash; 730 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) 731 goto clear_hash; 732 if (tcp_md5_hash_key(&hp, key)) 733 goto clear_hash; 734 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 735 if (crypto_ahash_final(hp.req)) 736 goto clear_hash; 737 738 tcp_sigpool_end(&hp); 739 return 0; 740 741 clear_hash: 742 tcp_sigpool_end(&hp); 743 clear_hash_nostart: 744 memset(md5_hash, 0, 16); 745 return 1; 746 } 747 748 static int tcp_v6_md5_hash_skb(char *md5_hash, 749 const struct tcp_md5sig_key *key, 750 const struct sock *sk, 751 const struct sk_buff *skb) 752 { 753 const struct tcphdr *th = tcp_hdr(skb); 754 const struct in6_addr *saddr, *daddr; 755 struct tcp_sigpool hp; 756 757 if (sk) { /* valid for establish/request sockets */ 758 saddr = &sk->sk_v6_rcv_saddr; 759 daddr = &sk->sk_v6_daddr; 760 } else { 761 const struct ipv6hdr *ip6h = ipv6_hdr(skb); 762 saddr = &ip6h->saddr; 763 daddr = &ip6h->daddr; 764 } 765 766 if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp)) 767 goto clear_hash_nostart; 768 769 if (crypto_ahash_init(hp.req)) 770 goto clear_hash; 771 772 if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) 773 goto clear_hash; 774 if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2)) 775 goto clear_hash; 776 if (tcp_md5_hash_key(&hp, key)) 777 goto clear_hash; 778 ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); 779 if (crypto_ahash_final(hp.req)) 780 goto clear_hash; 781 782 tcp_sigpool_end(&hp); 783 return 0; 784 785 clear_hash: 786 tcp_sigpool_end(&hp); 787 clear_hash_nostart: 788 memset(md5_hash, 0, 16); 789 return 1; 790 } 791 #endif 792 793 static void tcp_v6_init_req(struct request_sock *req, 794 const struct sock *sk_listener, 795 struct sk_buff *skb, 796 u32 tw_isn) 797 { 798 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 799 struct inet_request_sock *ireq = inet_rsk(req); 800 const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener); 801 802 ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; 803 ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; 804 ireq->ir_rmt_addr = LOOPBACK4_IPV6; 805 ireq->ir_loc_addr = LOOPBACK4_IPV6; 806 807 /* So that link locals have meaning */ 808 if ((!sk_listener->sk_bound_dev_if || l3_slave) && 809 ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL) 810 ireq->ir_iif = tcp_v6_iif(skb); 811 812 if (!tw_isn && 813 (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) || 814 np->rxopt.bits.rxinfo || 815 np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim || 816 np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) { 817 refcount_inc(&skb->users); 818 ireq->pktopts = skb; 819 } 820 } 821 822 static struct dst_entry *tcp_v6_route_req(const struct sock *sk, 823 struct sk_buff *skb, 824 struct flowi *fl, 825 struct request_sock *req, 826 u32 tw_isn) 827 { 828 tcp_v6_init_req(req, sk, skb, tw_isn); 829 830 if (security_inet_conn_request(sk, skb, req)) 831 return NULL; 832 833 return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP); 834 } 835 836 struct request_sock_ops tcp6_request_sock_ops __read_mostly = { 837 .family = AF_INET6, 838 .obj_size = sizeof(struct tcp6_request_sock), 839 .send_ack = tcp_v6_reqsk_send_ack, 840 .destructor = tcp_v6_reqsk_destructor, 841 .send_reset = tcp_v6_send_reset, 842 .syn_ack_timeout = tcp_syn_ack_timeout, 843 }; 844 845 const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { 846 .mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - 847 sizeof(struct ipv6hdr), 848 #ifdef CONFIG_TCP_MD5SIG 849 .req_md5_lookup = tcp_v6_md5_lookup, 850 .calc_md5_hash = tcp_v6_md5_hash_skb, 851 #endif 852 #ifdef CONFIG_TCP_AO 853 .ao_lookup = tcp_v6_ao_lookup_rsk, 854 .ao_calc_key = tcp_v6_ao_calc_key_rsk, 855 .ao_synack_hash = tcp_v6_ao_synack_hash, 856 #endif 857 #ifdef CONFIG_SYN_COOKIES 858 .cookie_init_seq = cookie_v6_init_sequence, 859 #endif 860 .route_req = tcp_v6_route_req, 861 .init_seq = tcp_v6_init_seq, 862 .init_ts_off = tcp_v6_init_ts_off, 863 .send_synack = tcp_v6_send_synack, 864 }; 865 866 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq, 867 u32 ack, u32 win, u32 tsval, u32 tsecr, 868 int oif, int rst, u8 tclass, __be32 label, 869 u32 priority, u32 txhash, struct tcp_key *key) 870 { 871 struct net *net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb); 872 unsigned int tot_len = sizeof(struct tcphdr); 873 struct sock *ctl_sk = net->ipv6.tcp_sk; 874 const struct tcphdr *th = tcp_hdr(skb); 875 __be32 mrst = 0, *topt; 876 struct dst_entry *dst; 877 struct sk_buff *buff; 878 struct tcphdr *t1; 879 struct flowi6 fl6; 880 u32 mark = 0; 881 882 if (tsecr) 883 tot_len += TCPOLEN_TSTAMP_ALIGNED; 884 if (tcp_key_is_md5(key)) 885 tot_len += TCPOLEN_MD5SIG_ALIGNED; 886 if (tcp_key_is_ao(key)) 887 tot_len += tcp_ao_len_aligned(key->ao_key); 888 889 #ifdef CONFIG_MPTCP 890 if (rst && !tcp_key_is_md5(key)) { 891 mrst = mptcp_reset_option(skb); 892 893 if (mrst) 894 tot_len += sizeof(__be32); 895 } 896 #endif 897 898 buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC); 899 if (!buff) 900 return; 901 902 skb_reserve(buff, MAX_TCP_HEADER); 903 904 t1 = skb_push(buff, tot_len); 905 skb_reset_transport_header(buff); 906 907 /* Swap the send and the receive. */ 908 memset(t1, 0, sizeof(*t1)); 909 t1->dest = th->source; 910 t1->source = th->dest; 911 t1->doff = tot_len / 4; 912 t1->seq = htonl(seq); 913 t1->ack_seq = htonl(ack); 914 t1->ack = !rst || !th->ack; 915 t1->rst = rst; 916 t1->window = htons(win); 917 918 topt = (__be32 *)(t1 + 1); 919 920 if (tsecr) { 921 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 922 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP); 923 *topt++ = htonl(tsval); 924 *topt++ = htonl(tsecr); 925 } 926 927 if (mrst) 928 *topt++ = mrst; 929 930 #ifdef CONFIG_TCP_MD5SIG 931 if (tcp_key_is_md5(key)) { 932 *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 933 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG); 934 tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key, 935 &ipv6_hdr(skb)->saddr, 936 &ipv6_hdr(skb)->daddr, t1); 937 } 938 #endif 939 #ifdef CONFIG_TCP_AO 940 if (tcp_key_is_ao(key)) { 941 *topt++ = htonl((TCPOPT_AO << 24) | 942 (tcp_ao_len(key->ao_key) << 16) | 943 (key->ao_key->sndid << 8) | 944 (key->rcv_next)); 945 946 tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key, 947 key->traffic_key, 948 (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr, 949 (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr, 950 t1, key->sne); 951 } 952 #endif 953 954 memset(&fl6, 0, sizeof(fl6)); 955 fl6.daddr = ipv6_hdr(skb)->saddr; 956 fl6.saddr = ipv6_hdr(skb)->daddr; 957 fl6.flowlabel = label; 958 959 buff->ip_summed = CHECKSUM_PARTIAL; 960 961 __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr); 962 963 fl6.flowi6_proto = IPPROTO_TCP; 964 if (rt6_need_strict(&fl6.daddr) && !oif) 965 fl6.flowi6_oif = tcp_v6_iif(skb); 966 else { 967 if (!oif && netif_index_is_l3_master(net, skb->skb_iif)) 968 oif = skb->skb_iif; 969 970 fl6.flowi6_oif = oif; 971 } 972 973 if (sk) { 974 /* unconstify the socket only to attach it to buff with care. */ 975 skb_set_owner_edemux(buff, (struct sock *)sk); 976 977 if (sk->sk_state == TCP_TIME_WAIT) 978 mark = inet_twsk(sk)->tw_mark; 979 else 980 mark = READ_ONCE(sk->sk_mark); 981 skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC); 982 } 983 if (txhash) { 984 /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */ 985 skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4); 986 } 987 fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark; 988 fl6.fl6_dport = t1->dest; 989 fl6.fl6_sport = t1->source; 990 fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL); 991 security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6)); 992 993 /* Pass a socket to ip6_dst_lookup either it is for RST 994 * Underlying function will use this to retrieve the network 995 * namespace 996 */ 997 if (sk && sk->sk_state != TCP_TIME_WAIT) 998 dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/ 999 else 1000 dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL); 1001 if (!IS_ERR(dst)) { 1002 skb_dst_set(buff, dst); 1003 ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL, 1004 tclass, priority); 1005 TCP_INC_STATS(net, TCP_MIB_OUTSEGS); 1006 if (rst) 1007 TCP_INC_STATS(net, TCP_MIB_OUTRSTS); 1008 return; 1009 } 1010 1011 kfree_skb(buff); 1012 } 1013 1014 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, 1015 enum sk_rst_reason reason) 1016 { 1017 const struct tcphdr *th = tcp_hdr(skb); 1018 struct ipv6hdr *ipv6h = ipv6_hdr(skb); 1019 const __u8 *md5_hash_location = NULL; 1020 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1021 bool allocated_traffic_key = false; 1022 #endif 1023 const struct tcp_ao_hdr *aoh; 1024 struct tcp_key key = {}; 1025 u32 seq = 0, ack_seq = 0; 1026 __be32 label = 0; 1027 u32 priority = 0; 1028 struct net *net; 1029 u32 txhash = 0; 1030 int oif = 0; 1031 #ifdef CONFIG_TCP_MD5SIG 1032 unsigned char newhash[16]; 1033 int genhash; 1034 struct sock *sk1 = NULL; 1035 #endif 1036 1037 if (th->rst) 1038 return; 1039 1040 /* If sk not NULL, it means we did a successful lookup and incoming 1041 * route had to be correct. prequeue might have dropped our dst. 1042 */ 1043 if (!sk && !ipv6_unicast_destination(skb)) 1044 return; 1045 1046 net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb); 1047 /* Invalid TCP option size or twice included auth */ 1048 if (tcp_parse_auth_options(th, &md5_hash_location, &aoh)) 1049 return; 1050 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1051 rcu_read_lock(); 1052 #endif 1053 #ifdef CONFIG_TCP_MD5SIG 1054 if (sk && sk_fullsock(sk)) { 1055 int l3index; 1056 1057 /* sdif set, means packet ingressed via a device 1058 * in an L3 domain and inet_iif is set to it. 1059 */ 1060 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1061 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index); 1062 if (key.md5_key) 1063 key.type = TCP_KEY_MD5; 1064 } else if (md5_hash_location) { 1065 int dif = tcp_v6_iif_l3_slave(skb); 1066 int sdif = tcp_v6_sdif(skb); 1067 int l3index; 1068 1069 /* 1070 * active side is lost. Try to find listening socket through 1071 * source port, and then find md5 key through listening socket. 1072 * we are not loose security here: 1073 * Incoming packet is checked with md5 hash with finding key, 1074 * no RST generated if md5 hash doesn't match. 1075 */ 1076 sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, 1077 NULL, 0, &ipv6h->saddr, th->source, 1078 &ipv6h->daddr, ntohs(th->source), 1079 dif, sdif); 1080 if (!sk1) 1081 goto out; 1082 1083 /* sdif set, means packet ingressed via a device 1084 * in an L3 domain and dif is set to it. 1085 */ 1086 l3index = tcp_v6_sdif(skb) ? dif : 0; 1087 1088 key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index); 1089 if (!key.md5_key) 1090 goto out; 1091 key.type = TCP_KEY_MD5; 1092 1093 genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb); 1094 if (genhash || memcmp(md5_hash_location, newhash, 16) != 0) 1095 goto out; 1096 } 1097 #endif 1098 1099 if (th->ack) 1100 seq = ntohl(th->ack_seq); 1101 else 1102 ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len - 1103 (th->doff << 2); 1104 1105 #ifdef CONFIG_TCP_AO 1106 if (aoh) { 1107 int l3index; 1108 1109 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1110 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq, 1111 &key.ao_key, &key.traffic_key, 1112 &allocated_traffic_key, 1113 &key.rcv_next, &key.sne)) 1114 goto out; 1115 key.type = TCP_KEY_AO; 1116 } 1117 #endif 1118 1119 if (sk) { 1120 oif = sk->sk_bound_dev_if; 1121 if (sk_fullsock(sk)) { 1122 if (inet6_test_bit(REPFLOW, sk)) 1123 label = ip6_flowlabel(ipv6h); 1124 priority = READ_ONCE(sk->sk_priority); 1125 txhash = sk->sk_txhash; 1126 } 1127 if (sk->sk_state == TCP_TIME_WAIT) { 1128 label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel); 1129 priority = inet_twsk(sk)->tw_priority; 1130 txhash = inet_twsk(sk)->tw_txhash; 1131 } 1132 } else { 1133 if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) 1134 label = ip6_flowlabel(ipv6h); 1135 } 1136 1137 trace_tcp_send_reset(sk, skb, reason); 1138 1139 tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1, 1140 ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK, 1141 label, priority, txhash, 1142 &key); 1143 1144 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1145 out: 1146 if (allocated_traffic_key) 1147 kfree(key.traffic_key); 1148 rcu_read_unlock(); 1149 #endif 1150 } 1151 1152 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq, 1153 u32 ack, u32 win, u32 tsval, u32 tsecr, int oif, 1154 struct tcp_key *key, u8 tclass, 1155 __be32 label, u32 priority, u32 txhash) 1156 { 1157 tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0, 1158 tclass, label, priority, txhash, key); 1159 } 1160 1161 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb, 1162 enum tcp_tw_status tw_status) 1163 { 1164 struct inet_timewait_sock *tw = inet_twsk(sk); 1165 struct tcp_timewait_sock *tcptw = tcp_twsk(sk); 1166 u8 tclass = tw->tw_tclass; 1167 struct tcp_key key = {}; 1168 1169 if (tw_status == TCP_TW_ACK_OOW) 1170 tclass &= ~INET_ECN_MASK; 1171 #ifdef CONFIG_TCP_AO 1172 struct tcp_ao_info *ao_info; 1173 1174 if (static_branch_unlikely(&tcp_ao_needed.key)) { 1175 1176 /* FIXME: the segment to-be-acked is not verified yet */ 1177 ao_info = rcu_dereference(tcptw->ao_info); 1178 if (ao_info) { 1179 const struct tcp_ao_hdr *aoh; 1180 1181 /* Invalid TCP option size or twice included auth */ 1182 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1183 goto out; 1184 if (aoh) 1185 key.ao_key = tcp_ao_established_key(sk, ao_info, 1186 aoh->rnext_keyid, -1); 1187 } 1188 } 1189 if (key.ao_key) { 1190 struct tcp_ao_key *rnext_key; 1191 1192 key.traffic_key = snd_other_key(key.ao_key); 1193 /* rcv_next switches to our rcv_next */ 1194 rnext_key = READ_ONCE(ao_info->rnext_key); 1195 key.rcv_next = rnext_key->rcvid; 1196 key.sne = READ_ONCE(ao_info->snd_sne); 1197 key.type = TCP_KEY_AO; 1198 #else 1199 if (0) { 1200 #endif 1201 #ifdef CONFIG_TCP_MD5SIG 1202 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1203 key.md5_key = tcp_twsk_md5_key(tcptw); 1204 if (key.md5_key) 1205 key.type = TCP_KEY_MD5; 1206 #endif 1207 } 1208 1209 tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, 1210 READ_ONCE(tcptw->tw_rcv_nxt), 1211 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale, 1212 tcp_tw_tsval(tcptw), 1213 READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if, 1214 &key, tclass, cpu_to_be32(tw->tw_flowlabel), 1215 tw->tw_priority, tw->tw_txhash); 1216 1217 #ifdef CONFIG_TCP_AO 1218 out: 1219 #endif 1220 inet_twsk_put(tw); 1221 } 1222 1223 static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 1224 struct request_sock *req) 1225 { 1226 struct tcp_key key = {}; 1227 1228 #ifdef CONFIG_TCP_AO 1229 if (static_branch_unlikely(&tcp_ao_needed.key) && 1230 tcp_rsk_used_ao(req)) { 1231 const struct in6_addr *addr = &ipv6_hdr(skb)->saddr; 1232 const struct tcp_ao_hdr *aoh; 1233 int l3index; 1234 1235 l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1236 /* Invalid TCP option size or twice included auth */ 1237 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1238 return; 1239 if (!aoh) 1240 return; 1241 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1242 (union tcp_ao_addr *)addr, 1243 AF_INET6, aoh->rnext_keyid, -1); 1244 if (unlikely(!key.ao_key)) { 1245 /* Send ACK with any matching MKT for the peer */ 1246 key.ao_key = tcp_ao_do_lookup(sk, l3index, 1247 (union tcp_ao_addr *)addr, 1248 AF_INET6, -1, -1); 1249 /* Matching key disappeared (user removed the key?) 1250 * let the handshake timeout. 1251 */ 1252 if (!key.ao_key) { 1253 net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n", 1254 addr, 1255 ntohs(tcp_hdr(skb)->source), 1256 &ipv6_hdr(skb)->daddr, 1257 ntohs(tcp_hdr(skb)->dest)); 1258 return; 1259 } 1260 } 1261 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC); 1262 if (!key.traffic_key) 1263 return; 1264 1265 key.type = TCP_KEY_AO; 1266 key.rcv_next = aoh->keyid; 1267 tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req); 1268 #else 1269 if (0) { 1270 #endif 1271 #ifdef CONFIG_TCP_MD5SIG 1272 } else if (static_branch_unlikely(&tcp_md5_needed.key)) { 1273 int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0; 1274 1275 key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, 1276 l3index); 1277 if (key.md5_key) 1278 key.type = TCP_KEY_MD5; 1279 #endif 1280 } 1281 1282 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV 1283 * sk->sk_state == TCP_SYN_RECV -> for Fast Open. 1284 */ 1285 tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ? 1286 tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt, 1287 tcp_rsk(req)->rcv_nxt, 1288 tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale, 1289 tcp_rsk_tsval(tcp_rsk(req)), 1290 req->ts_recent, sk->sk_bound_dev_if, 1291 &key, ipv6_get_dsfield(ipv6_hdr(skb)) & ~INET_ECN_MASK, 1292 0, 1293 READ_ONCE(sk->sk_priority), 1294 READ_ONCE(tcp_rsk(req)->txhash)); 1295 if (tcp_key_is_ao(&key)) 1296 kfree(key.traffic_key); 1297 } 1298 1299 1300 static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb) 1301 { 1302 #ifdef CONFIG_SYN_COOKIES 1303 const struct tcphdr *th = tcp_hdr(skb); 1304 1305 if (!th->syn) 1306 sk = cookie_v6_check(sk, skb); 1307 #endif 1308 return sk; 1309 } 1310 1311 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 1312 struct tcphdr *th, u32 *cookie) 1313 { 1314 u16 mss = 0; 1315 #ifdef CONFIG_SYN_COOKIES 1316 mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops, 1317 &tcp_request_sock_ipv6_ops, sk, th); 1318 if (mss) { 1319 *cookie = __cookie_v6_init_sequence(iph, th, &mss); 1320 tcp_synq_overflow(sk); 1321 } 1322 #endif 1323 return mss; 1324 } 1325 1326 static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb) 1327 { 1328 if (skb->protocol == htons(ETH_P_IP)) 1329 return tcp_v4_conn_request(sk, skb); 1330 1331 if (!ipv6_unicast_destination(skb)) 1332 goto drop; 1333 1334 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 1335 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 1336 return 0; 1337 } 1338 1339 return tcp_conn_request(&tcp6_request_sock_ops, 1340 &tcp_request_sock_ipv6_ops, sk, skb); 1341 1342 drop: 1343 tcp_listendrop(sk); 1344 return 0; /* don't send reset */ 1345 } 1346 1347 static void tcp_v6_restore_cb(struct sk_buff *skb) 1348 { 1349 /* We need to move header back to the beginning if xfrm6_policy_check() 1350 * and tcp_v6_fill_cb() are going to be called again. 1351 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there. 1352 */ 1353 memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6, 1354 sizeof(struct inet6_skb_parm)); 1355 } 1356 1357 static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 1358 struct request_sock *req, 1359 struct dst_entry *dst, 1360 struct request_sock *req_unhash, 1361 bool *own_req) 1362 { 1363 struct inet_request_sock *ireq; 1364 struct ipv6_pinfo *newnp; 1365 const struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1366 struct ipv6_txoptions *opt; 1367 struct inet_sock *newinet; 1368 bool found_dup_sk = false; 1369 struct tcp_sock *newtp; 1370 struct sock *newsk; 1371 #ifdef CONFIG_TCP_MD5SIG 1372 struct tcp_md5sig_key *key; 1373 int l3index; 1374 #endif 1375 struct flowi6 fl6; 1376 1377 if (skb->protocol == htons(ETH_P_IP)) { 1378 /* 1379 * v6 mapped 1380 */ 1381 1382 newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst, 1383 req_unhash, own_req); 1384 1385 if (!newsk) 1386 return NULL; 1387 1388 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1389 1390 newnp = tcp_inet6_sk(newsk); 1391 newtp = tcp_sk(newsk); 1392 1393 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1394 1395 newnp->saddr = newsk->sk_v6_rcv_saddr; 1396 1397 inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; 1398 if (sk_is_mptcp(newsk)) 1399 mptcpv6_handle_mapped(newsk, true); 1400 newsk->sk_backlog_rcv = tcp_v4_do_rcv; 1401 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 1402 newtp->af_specific = &tcp_sock_ipv6_mapped_specific; 1403 #endif 1404 1405 newnp->ipv6_mc_list = NULL; 1406 newnp->ipv6_ac_list = NULL; 1407 newnp->ipv6_fl_list = NULL; 1408 newnp->pktoptions = NULL; 1409 newnp->opt = NULL; 1410 newnp->mcast_oif = inet_iif(skb); 1411 newnp->mcast_hops = ip_hdr(skb)->ttl; 1412 newnp->rcv_flowinfo = 0; 1413 if (inet6_test_bit(REPFLOW, sk)) 1414 newnp->flow_label = 0; 1415 1416 /* 1417 * No need to charge this sock to the relevant IPv6 refcnt debug socks count 1418 * here, tcp_create_openreq_child now does this for us, see the comment in 1419 * that function for the gory details. -acme 1420 */ 1421 1422 /* It is tricky place. Until this moment IPv4 tcp 1423 worked with IPv6 icsk.icsk_af_ops. 1424 Sync it now. 1425 */ 1426 tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie); 1427 1428 return newsk; 1429 } 1430 1431 ireq = inet_rsk(req); 1432 1433 if (sk_acceptq_is_full(sk)) 1434 goto exit_overflow; 1435 1436 if (!dst) { 1437 dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP); 1438 if (!dst) 1439 goto exit; 1440 } 1441 1442 newsk = tcp_create_openreq_child(sk, req, skb); 1443 if (!newsk) 1444 goto exit_nonewsk; 1445 1446 /* 1447 * No need to charge this sock to the relevant IPv6 refcnt debug socks 1448 * count here, tcp_create_openreq_child now does this for us, see the 1449 * comment in that function for the gory details. -acme 1450 */ 1451 1452 newsk->sk_gso_type = SKB_GSO_TCPV6; 1453 inet6_sk_rx_dst_set(newsk, skb); 1454 1455 inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); 1456 1457 newtp = tcp_sk(newsk); 1458 newinet = inet_sk(newsk); 1459 newnp = tcp_inet6_sk(newsk); 1460 1461 memcpy(newnp, np, sizeof(struct ipv6_pinfo)); 1462 1463 ip6_dst_store(newsk, dst, NULL, NULL); 1464 1465 newnp->saddr = ireq->ir_v6_loc_addr; 1466 1467 /* Now IPv6 options... 1468 1469 First: no IPv4 options. 1470 */ 1471 newinet->inet_opt = NULL; 1472 newnp->ipv6_mc_list = NULL; 1473 newnp->ipv6_ac_list = NULL; 1474 newnp->ipv6_fl_list = NULL; 1475 1476 /* Clone RX bits */ 1477 newnp->rxopt.all = np->rxopt.all; 1478 1479 newnp->pktoptions = NULL; 1480 newnp->opt = NULL; 1481 newnp->mcast_oif = tcp_v6_iif(skb); 1482 newnp->mcast_hops = ipv6_hdr(skb)->hop_limit; 1483 newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb)); 1484 if (inet6_test_bit(REPFLOW, sk)) 1485 newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb)); 1486 1487 /* Set ToS of the new socket based upon the value of incoming SYN. 1488 * ECT bits are set later in tcp_init_transfer(). 1489 */ 1490 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)) 1491 newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK; 1492 1493 /* Clone native IPv6 options from listening socket (if any) 1494 1495 Yes, keeping reference count would be much more clever, 1496 but we make one more one thing there: reattach optmem 1497 to newsk. 1498 */ 1499 opt = ireq->ipv6_opt; 1500 if (!opt) 1501 opt = rcu_dereference(np->opt); 1502 if (opt) { 1503 opt = ipv6_dup_options(newsk, opt); 1504 RCU_INIT_POINTER(newnp->opt, opt); 1505 } 1506 inet_csk(newsk)->icsk_ext_hdr_len = 0; 1507 if (opt) 1508 inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen + 1509 opt->opt_flen; 1510 1511 tcp_ca_openreq_child(newsk, dst); 1512 1513 tcp_sync_mss(newsk, dst_mtu(dst)); 1514 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); 1515 1516 tcp_initialize_rcv_mss(newsk); 1517 1518 #ifdef CONFIG_TCP_MD5SIG 1519 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif); 1520 1521 if (!tcp_rsk_used_ao(req)) { 1522 /* Copy over the MD5 key from the original socket */ 1523 key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index); 1524 if (key) { 1525 const union tcp_md5_addr *addr; 1526 1527 addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr; 1528 if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) 1529 goto put_and_exit; 1530 } 1531 } 1532 #endif 1533 #ifdef CONFIG_TCP_AO 1534 /* Copy over tcp_ao_info if any */ 1535 if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6)) 1536 goto put_and_exit; /* OOM */ 1537 #endif 1538 1539 if (__inet_inherit_port(sk, newsk) < 0) 1540 goto put_and_exit; 1541 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash), 1542 &found_dup_sk); 1543 if (*own_req) { 1544 tcp_move_syn(newtp, req); 1545 1546 /* Clone pktoptions received with SYN, if we own the req */ 1547 if (ireq->pktopts) { 1548 newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk); 1549 consume_skb(ireq->pktopts); 1550 ireq->pktopts = NULL; 1551 if (newnp->pktoptions) 1552 tcp_v6_restore_cb(newnp->pktoptions); 1553 } 1554 } else { 1555 if (!req_unhash && found_dup_sk) { 1556 /* This code path should only be executed in the 1557 * syncookie case only 1558 */ 1559 bh_unlock_sock(newsk); 1560 sock_put(newsk); 1561 newsk = NULL; 1562 } 1563 } 1564 1565 return newsk; 1566 1567 exit_overflow: 1568 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS); 1569 exit_nonewsk: 1570 dst_release(dst); 1571 exit: 1572 tcp_listendrop(sk); 1573 return NULL; 1574 put_and_exit: 1575 inet_csk_prepare_forced_close(newsk); 1576 tcp_done(newsk); 1577 goto exit; 1578 } 1579 1580 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 1581 u32)); 1582 /* The socket must have it's spinlock held when we get 1583 * here, unless it is a TCP_LISTEN socket. 1584 * 1585 * We have a potential double-lock case here, so even when 1586 * doing backlog processing we use the BH locking scheme. 1587 * This is because we cannot sleep with the original spinlock 1588 * held. 1589 */ 1590 INDIRECT_CALLABLE_SCOPE 1591 int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) 1592 { 1593 struct ipv6_pinfo *np = tcp_inet6_sk(sk); 1594 struct sk_buff *opt_skb = NULL; 1595 enum skb_drop_reason reason; 1596 struct tcp_sock *tp; 1597 1598 /* Imagine: socket is IPv6. IPv4 packet arrives, 1599 goes to IPv4 receive handler and backlogged. 1600 From backlog it always goes here. Kerboom... 1601 Fortunately, tcp_rcv_established and rcv_established 1602 handle them correctly, but it is not case with 1603 tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK 1604 */ 1605 1606 if (skb->protocol == htons(ETH_P_IP)) 1607 return tcp_v4_do_rcv(sk, skb); 1608 1609 /* 1610 * socket locking is here for SMP purposes as backlog rcv 1611 * is currently called with bh processing disabled. 1612 */ 1613 1614 /* Do Stevens' IPV6_PKTOPTIONS. 1615 1616 Yes, guys, it is the only place in our code, where we 1617 may make it not affecting IPv4. 1618 The rest of code is protocol independent, 1619 and I do not like idea to uglify IPv4. 1620 1621 Actually, all the idea behind IPV6_PKTOPTIONS 1622 looks not very well thought. For now we latch 1623 options, received in the last packet, enqueued 1624 by tcp. Feel free to propose better solution. 1625 --ANK (980728) 1626 */ 1627 if (np->rxopt.all && sk->sk_state != TCP_LISTEN) 1628 opt_skb = skb_clone_and_charge_r(skb, sk); 1629 1630 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */ 1631 struct dst_entry *dst; 1632 1633 dst = rcu_dereference_protected(sk->sk_rx_dst, 1634 lockdep_sock_is_held(sk)); 1635 1636 sock_rps_save_rxhash(sk, skb); 1637 sk_mark_napi_id(sk, skb); 1638 if (dst) { 1639 if (sk->sk_rx_dst_ifindex != skb->skb_iif || 1640 INDIRECT_CALL_1(dst->ops->check, ip6_dst_check, 1641 dst, sk->sk_rx_dst_cookie) == NULL) { 1642 RCU_INIT_POINTER(sk->sk_rx_dst, NULL); 1643 dst_release(dst); 1644 } 1645 } 1646 1647 tcp_rcv_established(sk, skb); 1648 if (opt_skb) 1649 goto ipv6_pktoptions; 1650 return 0; 1651 } 1652 1653 if (tcp_checksum_complete(skb)) 1654 goto csum_err; 1655 1656 if (sk->sk_state == TCP_LISTEN) { 1657 struct sock *nsk = tcp_v6_cookie_check(sk, skb); 1658 1659 if (nsk != sk) { 1660 if (nsk) { 1661 reason = tcp_child_process(sk, nsk, skb); 1662 if (reason) 1663 goto reset; 1664 } 1665 return 0; 1666 } 1667 } else 1668 sock_rps_save_rxhash(sk, skb); 1669 1670 reason = tcp_rcv_state_process(sk, skb); 1671 if (reason) 1672 goto reset; 1673 if (opt_skb) 1674 goto ipv6_pktoptions; 1675 return 0; 1676 1677 reset: 1678 tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason)); 1679 discard: 1680 if (opt_skb) 1681 __kfree_skb(opt_skb); 1682 sk_skb_reason_drop(sk, skb, reason); 1683 return 0; 1684 csum_err: 1685 reason = SKB_DROP_REASON_TCP_CSUM; 1686 trace_tcp_bad_csum(skb); 1687 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS); 1688 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS); 1689 goto discard; 1690 1691 1692 ipv6_pktoptions: 1693 /* Do you ask, what is it? 1694 1695 1. skb was enqueued by tcp. 1696 2. skb is added to tail of read queue, rather than out of order. 1697 3. socket is not in passive state. 1698 4. Finally, it really contains options, which user wants to receive. 1699 */ 1700 tp = tcp_sk(sk); 1701 if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt && 1702 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { 1703 if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo) 1704 WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb)); 1705 if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim) 1706 WRITE_ONCE(np->mcast_hops, 1707 ipv6_hdr(opt_skb)->hop_limit); 1708 if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass) 1709 np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb)); 1710 if (inet6_test_bit(REPFLOW, sk)) 1711 np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb)); 1712 if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) { 1713 tcp_v6_restore_cb(opt_skb); 1714 opt_skb = xchg(&np->pktoptions, opt_skb); 1715 } else { 1716 __kfree_skb(opt_skb); 1717 opt_skb = xchg(&np->pktoptions, NULL); 1718 } 1719 } 1720 1721 consume_skb(opt_skb); 1722 return 0; 1723 } 1724 1725 static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr, 1726 const struct tcphdr *th) 1727 { 1728 /* This is tricky: we move IP6CB at its correct location into 1729 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because 1730 * _decode_session6() uses IP6CB(). 1731 * barrier() makes sure compiler won't play aliasing games. 1732 */ 1733 memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb), 1734 sizeof(struct inet6_skb_parm)); 1735 barrier(); 1736 1737 TCP_SKB_CB(skb)->seq = ntohl(th->seq); 1738 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin + 1739 skb->len - th->doff*4); 1740 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq); 1741 TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th); 1742 TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr); 1743 TCP_SKB_CB(skb)->sacked = 0; 1744 TCP_SKB_CB(skb)->has_rxtstamp = 1745 skb->tstamp || skb_hwtstamps(skb)->hwtstamp; 1746 } 1747 1748 INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb) 1749 { 1750 struct net *net = dev_net_rcu(skb->dev); 1751 enum skb_drop_reason drop_reason; 1752 enum tcp_tw_status tw_status; 1753 int sdif = inet6_sdif(skb); 1754 int dif = inet6_iif(skb); 1755 const struct tcphdr *th; 1756 const struct ipv6hdr *hdr; 1757 struct sock *sk = NULL; 1758 bool refcounted; 1759 int ret; 1760 u32 isn; 1761 1762 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 1763 if (skb->pkt_type != PACKET_HOST) 1764 goto discard_it; 1765 1766 /* 1767 * Count it even if it's bad. 1768 */ 1769 __TCP_INC_STATS(net, TCP_MIB_INSEGS); 1770 1771 if (!pskb_may_pull(skb, sizeof(struct tcphdr))) 1772 goto discard_it; 1773 1774 th = (const struct tcphdr *)skb->data; 1775 1776 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) { 1777 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL; 1778 goto bad_packet; 1779 } 1780 if (!pskb_may_pull(skb, th->doff*4)) 1781 goto discard_it; 1782 1783 if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo)) 1784 goto csum_error; 1785 1786 th = (const struct tcphdr *)skb->data; 1787 hdr = ipv6_hdr(skb); 1788 1789 lookup: 1790 sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th), 1791 th->source, th->dest, inet6_iif(skb), sdif, 1792 &refcounted); 1793 if (!sk) 1794 goto no_tcp_socket; 1795 1796 if (sk->sk_state == TCP_TIME_WAIT) 1797 goto do_time_wait; 1798 1799 if (sk->sk_state == TCP_NEW_SYN_RECV) { 1800 struct request_sock *req = inet_reqsk(sk); 1801 bool req_stolen = false; 1802 struct sock *nsk; 1803 1804 sk = req->rsk_listener; 1805 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) 1806 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1807 else 1808 drop_reason = tcp_inbound_hash(sk, req, skb, 1809 &hdr->saddr, &hdr->daddr, 1810 AF_INET6, dif, sdif); 1811 if (drop_reason) { 1812 sk_drops_add(sk, skb); 1813 reqsk_put(req); 1814 goto discard_it; 1815 } 1816 if (tcp_checksum_complete(skb)) { 1817 reqsk_put(req); 1818 goto csum_error; 1819 } 1820 if (unlikely(sk->sk_state != TCP_LISTEN)) { 1821 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb); 1822 if (!nsk) { 1823 inet_csk_reqsk_queue_drop_and_put(sk, req); 1824 goto lookup; 1825 } 1826 sk = nsk; 1827 /* reuseport_migrate_sock() has already held one sk_refcnt 1828 * before returning. 1829 */ 1830 } else { 1831 sock_hold(sk); 1832 } 1833 refcounted = true; 1834 nsk = NULL; 1835 if (!tcp_filter(sk, skb, &drop_reason)) { 1836 th = (const struct tcphdr *)skb->data; 1837 hdr = ipv6_hdr(skb); 1838 tcp_v6_fill_cb(skb, hdr, th); 1839 nsk = tcp_check_req(sk, skb, req, false, &req_stolen, 1840 &drop_reason); 1841 } 1842 if (!nsk) { 1843 reqsk_put(req); 1844 if (req_stolen) { 1845 /* Another cpu got exclusive access to req 1846 * and created a full blown socket. 1847 * Try to feed this packet to this socket 1848 * instead of discarding it. 1849 */ 1850 tcp_v6_restore_cb(skb); 1851 sock_put(sk); 1852 goto lookup; 1853 } 1854 goto discard_and_relse; 1855 } 1856 nf_reset_ct(skb); 1857 if (nsk == sk) { 1858 reqsk_put(req); 1859 tcp_v6_restore_cb(skb); 1860 } else { 1861 drop_reason = tcp_child_process(sk, nsk, skb); 1862 if (drop_reason) { 1863 enum sk_rst_reason rst_reason; 1864 1865 rst_reason = sk_rst_convert_drop_reason(drop_reason); 1866 tcp_v6_send_reset(nsk, skb, rst_reason); 1867 goto discard_and_relse; 1868 } 1869 sock_put(sk); 1870 return 0; 1871 } 1872 } 1873 1874 process: 1875 if (static_branch_unlikely(&ip6_min_hopcount)) { 1876 /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */ 1877 if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) { 1878 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 1879 drop_reason = SKB_DROP_REASON_TCP_MINTTL; 1880 goto discard_and_relse; 1881 } 1882 } 1883 1884 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) { 1885 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1886 goto discard_and_relse; 1887 } 1888 1889 drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr, 1890 AF_INET6, dif, sdif); 1891 if (drop_reason) 1892 goto discard_and_relse; 1893 1894 nf_reset_ct(skb); 1895 1896 if (tcp_filter(sk, skb, &drop_reason)) 1897 goto discard_and_relse; 1898 1899 th = (const struct tcphdr *)skb->data; 1900 hdr = ipv6_hdr(skb); 1901 tcp_v6_fill_cb(skb, hdr, th); 1902 1903 skb->dev = NULL; 1904 1905 if (sk->sk_state == TCP_LISTEN) { 1906 ret = tcp_v6_do_rcv(sk, skb); 1907 goto put_and_return; 1908 } 1909 1910 sk_incoming_cpu_update(sk); 1911 1912 bh_lock_sock_nested(sk); 1913 tcp_segs_in(tcp_sk(sk), skb); 1914 ret = 0; 1915 if (!sock_owned_by_user(sk)) { 1916 ret = tcp_v6_do_rcv(sk, skb); 1917 } else { 1918 if (tcp_add_backlog(sk, skb, &drop_reason)) 1919 goto discard_and_relse; 1920 } 1921 bh_unlock_sock(sk); 1922 put_and_return: 1923 if (refcounted) 1924 sock_put(sk); 1925 return ret ? -1 : 0; 1926 1927 no_tcp_socket: 1928 drop_reason = SKB_DROP_REASON_NO_SOCKET; 1929 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) 1930 goto discard_it; 1931 1932 tcp_v6_fill_cb(skb, hdr, th); 1933 1934 if (tcp_checksum_complete(skb)) { 1935 csum_error: 1936 drop_reason = SKB_DROP_REASON_TCP_CSUM; 1937 trace_tcp_bad_csum(skb); 1938 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS); 1939 bad_packet: 1940 __TCP_INC_STATS(net, TCP_MIB_INERRS); 1941 } else { 1942 tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason)); 1943 } 1944 1945 discard_it: 1946 SKB_DR_OR(drop_reason, NOT_SPECIFIED); 1947 sk_skb_reason_drop(sk, skb, drop_reason); 1948 return 0; 1949 1950 discard_and_relse: 1951 sk_drops_add(sk, skb); 1952 if (refcounted) 1953 sock_put(sk); 1954 goto discard_it; 1955 1956 do_time_wait: 1957 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) { 1958 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 1959 inet_twsk_put(inet_twsk(sk)); 1960 goto discard_it; 1961 } 1962 1963 tcp_v6_fill_cb(skb, hdr, th); 1964 1965 if (tcp_checksum_complete(skb)) { 1966 inet_twsk_put(inet_twsk(sk)); 1967 goto csum_error; 1968 } 1969 1970 tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn, 1971 &drop_reason); 1972 switch (tw_status) { 1973 case TCP_TW_SYN: 1974 { 1975 struct sock *sk2; 1976 1977 sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, 1978 skb, __tcp_hdrlen(th), 1979 &ipv6_hdr(skb)->saddr, th->source, 1980 &ipv6_hdr(skb)->daddr, 1981 ntohs(th->dest), 1982 tcp_v6_iif_l3_slave(skb), 1983 sdif); 1984 if (sk2) { 1985 struct inet_timewait_sock *tw = inet_twsk(sk); 1986 inet_twsk_deschedule_put(tw); 1987 sk = sk2; 1988 tcp_v6_restore_cb(skb); 1989 refcounted = false; 1990 __this_cpu_write(tcp_tw_isn, isn); 1991 goto process; 1992 } 1993 } 1994 /* to ACK */ 1995 fallthrough; 1996 case TCP_TW_ACK: 1997 case TCP_TW_ACK_OOW: 1998 tcp_v6_timewait_ack(sk, skb, tw_status); 1999 break; 2000 case TCP_TW_RST: 2001 tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET); 2002 inet_twsk_deschedule_put(inet_twsk(sk)); 2003 goto discard_it; 2004 case TCP_TW_SUCCESS: 2005 ; 2006 } 2007 goto discard_it; 2008 } 2009 2010 void tcp_v6_early_demux(struct sk_buff *skb) 2011 { 2012 struct net *net = dev_net_rcu(skb->dev); 2013 const struct ipv6hdr *hdr; 2014 const struct tcphdr *th; 2015 struct sock *sk; 2016 2017 if (skb->pkt_type != PACKET_HOST) 2018 return; 2019 2020 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr))) 2021 return; 2022 2023 hdr = ipv6_hdr(skb); 2024 th = tcp_hdr(skb); 2025 2026 if (th->doff < sizeof(struct tcphdr) / 4) 2027 return; 2028 2029 /* Note : We use inet6_iif() here, not tcp_v6_iif() */ 2030 sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 2031 &hdr->saddr, th->source, 2032 &hdr->daddr, ntohs(th->dest), 2033 inet6_iif(skb), inet6_sdif(skb)); 2034 if (sk) { 2035 skb->sk = sk; 2036 skb->destructor = sock_edemux; 2037 if (sk_fullsock(sk)) { 2038 struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst); 2039 2040 if (dst) 2041 dst = dst_check(dst, sk->sk_rx_dst_cookie); 2042 if (dst && 2043 sk->sk_rx_dst_ifindex == skb->skb_iif) 2044 skb_dst_set_noref(skb, dst); 2045 } 2046 } 2047 } 2048 2049 static struct timewait_sock_ops tcp6_timewait_sock_ops = { 2050 .twsk_obj_size = sizeof(struct tcp6_timewait_sock), 2051 .twsk_destructor = tcp_twsk_destructor, 2052 }; 2053 2054 INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) 2055 { 2056 __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr); 2057 } 2058 2059 const struct inet_connection_sock_af_ops ipv6_specific = { 2060 .queue_xmit = inet6_csk_xmit, 2061 .send_check = tcp_v6_send_check, 2062 .rebuild_header = inet6_sk_rebuild_header, 2063 .sk_rx_dst_set = inet6_sk_rx_dst_set, 2064 .conn_request = tcp_v6_conn_request, 2065 .syn_recv_sock = tcp_v6_syn_recv_sock, 2066 .net_header_len = sizeof(struct ipv6hdr), 2067 .setsockopt = ipv6_setsockopt, 2068 .getsockopt = ipv6_getsockopt, 2069 .mtu_reduced = tcp_v6_mtu_reduced, 2070 }; 2071 2072 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2073 static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = { 2074 #ifdef CONFIG_TCP_MD5SIG 2075 .md5_lookup = tcp_v6_md5_lookup, 2076 .calc_md5_hash = tcp_v6_md5_hash_skb, 2077 .md5_parse = tcp_v6_parse_md5_keys, 2078 #endif 2079 #ifdef CONFIG_TCP_AO 2080 .ao_lookup = tcp_v6_ao_lookup, 2081 .calc_ao_hash = tcp_v6_ao_hash_skb, 2082 .ao_parse = tcp_v6_parse_ao, 2083 .ao_calc_key_sk = tcp_v6_ao_calc_key_sk, 2084 #endif 2085 }; 2086 #endif 2087 2088 /* 2089 * TCP over IPv4 via INET6 API 2090 */ 2091 static const struct inet_connection_sock_af_ops ipv6_mapped = { 2092 .queue_xmit = ip_queue_xmit, 2093 .send_check = tcp_v4_send_check, 2094 .rebuild_header = inet_sk_rebuild_header, 2095 .sk_rx_dst_set = inet_sk_rx_dst_set, 2096 .conn_request = tcp_v6_conn_request, 2097 .syn_recv_sock = tcp_v6_syn_recv_sock, 2098 .net_header_len = sizeof(struct iphdr), 2099 .setsockopt = ipv6_setsockopt, 2100 .getsockopt = ipv6_getsockopt, 2101 .mtu_reduced = tcp_v4_mtu_reduced, 2102 }; 2103 2104 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2105 static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = { 2106 #ifdef CONFIG_TCP_MD5SIG 2107 .md5_lookup = tcp_v4_md5_lookup, 2108 .calc_md5_hash = tcp_v4_md5_hash_skb, 2109 .md5_parse = tcp_v6_parse_md5_keys, 2110 #endif 2111 #ifdef CONFIG_TCP_AO 2112 .ao_lookup = tcp_v6_ao_lookup, 2113 .calc_ao_hash = tcp_v4_ao_hash_skb, 2114 .ao_parse = tcp_v6_parse_ao, 2115 .ao_calc_key_sk = tcp_v4_ao_calc_key_sk, 2116 #endif 2117 }; 2118 #endif 2119 2120 /* NOTE: A lot of things set to zero explicitly by call to 2121 * sk_alloc() so need not be done here. 2122 */ 2123 static int tcp_v6_init_sock(struct sock *sk) 2124 { 2125 struct inet_connection_sock *icsk = inet_csk(sk); 2126 2127 tcp_init_sock(sk); 2128 2129 icsk->icsk_af_ops = &ipv6_specific; 2130 2131 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2132 tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific; 2133 #endif 2134 2135 return 0; 2136 } 2137 2138 #ifdef CONFIG_PROC_FS 2139 /* Proc filesystem TCPv6 sock list dumping. */ 2140 static void get_openreq6(struct seq_file *seq, 2141 const struct request_sock *req, int i) 2142 { 2143 long ttd = req->rsk_timer.expires - jiffies; 2144 const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr; 2145 const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr; 2146 2147 if (ttd < 0) 2148 ttd = 0; 2149 2150 seq_printf(seq, 2151 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2152 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n", 2153 i, 2154 src->s6_addr32[0], src->s6_addr32[1], 2155 src->s6_addr32[2], src->s6_addr32[3], 2156 inet_rsk(req)->ir_num, 2157 dest->s6_addr32[0], dest->s6_addr32[1], 2158 dest->s6_addr32[2], dest->s6_addr32[3], 2159 ntohs(inet_rsk(req)->ir_rmt_port), 2160 TCP_SYN_RECV, 2161 0, 0, /* could print option size, but that is af dependent. */ 2162 1, /* timers active (only the expire timer) */ 2163 jiffies_to_clock_t(ttd), 2164 req->num_timeout, 2165 from_kuid_munged(seq_user_ns(seq), 2166 sk_uid(req->rsk_listener)), 2167 0, /* non standard timer */ 2168 0, /* open_requests have no inode */ 2169 0, req); 2170 } 2171 2172 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i) 2173 { 2174 const struct in6_addr *dest, *src; 2175 __u16 destp, srcp; 2176 int timer_active; 2177 unsigned long timer_expires; 2178 const struct inet_sock *inet = inet_sk(sp); 2179 const struct tcp_sock *tp = tcp_sk(sp); 2180 const struct inet_connection_sock *icsk = inet_csk(sp); 2181 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq; 2182 u8 icsk_pending; 2183 int rx_queue; 2184 int state; 2185 2186 dest = &sp->sk_v6_daddr; 2187 src = &sp->sk_v6_rcv_saddr; 2188 destp = ntohs(inet->inet_dport); 2189 srcp = ntohs(inet->inet_sport); 2190 2191 icsk_pending = smp_load_acquire(&icsk->icsk_pending); 2192 if (icsk_pending == ICSK_TIME_RETRANS || 2193 icsk_pending == ICSK_TIME_REO_TIMEOUT || 2194 icsk_pending == ICSK_TIME_LOSS_PROBE) { 2195 timer_active = 1; 2196 timer_expires = icsk_timeout(icsk); 2197 } else if (icsk_pending == ICSK_TIME_PROBE0) { 2198 timer_active = 4; 2199 timer_expires = icsk_timeout(icsk); 2200 } else if (timer_pending(&sp->sk_timer)) { 2201 timer_active = 2; 2202 timer_expires = sp->sk_timer.expires; 2203 } else { 2204 timer_active = 0; 2205 timer_expires = jiffies; 2206 } 2207 2208 state = inet_sk_state_load(sp); 2209 if (state == TCP_LISTEN) 2210 rx_queue = READ_ONCE(sp->sk_ack_backlog); 2211 else 2212 /* Because we don't lock the socket, 2213 * we might find a transient negative value. 2214 */ 2215 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) - 2216 READ_ONCE(tp->copied_seq), 0); 2217 2218 seq_printf(seq, 2219 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2220 "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n", 2221 i, 2222 src->s6_addr32[0], src->s6_addr32[1], 2223 src->s6_addr32[2], src->s6_addr32[3], srcp, 2224 dest->s6_addr32[0], dest->s6_addr32[1], 2225 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2226 state, 2227 READ_ONCE(tp->write_seq) - tp->snd_una, 2228 rx_queue, 2229 timer_active, 2230 jiffies_delta_to_clock_t(timer_expires - jiffies), 2231 icsk->icsk_retransmits, 2232 from_kuid_munged(seq_user_ns(seq), sk_uid(sp)), 2233 icsk->icsk_probes_out, 2234 sock_i_ino(sp), 2235 refcount_read(&sp->sk_refcnt), sp, 2236 jiffies_to_clock_t(icsk->icsk_rto), 2237 jiffies_to_clock_t(icsk->icsk_ack.ato), 2238 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp), 2239 tcp_snd_cwnd(tp), 2240 state == TCP_LISTEN ? 2241 fastopenq->max_qlen : 2242 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh) 2243 ); 2244 } 2245 2246 static void get_timewait6_sock(struct seq_file *seq, 2247 struct inet_timewait_sock *tw, int i) 2248 { 2249 long delta = tw->tw_timer.expires - jiffies; 2250 const struct in6_addr *dest, *src; 2251 __u16 destp, srcp; 2252 2253 dest = &tw->tw_v6_daddr; 2254 src = &tw->tw_v6_rcv_saddr; 2255 destp = ntohs(tw->tw_dport); 2256 srcp = ntohs(tw->tw_sport); 2257 2258 seq_printf(seq, 2259 "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X " 2260 "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", 2261 i, 2262 src->s6_addr32[0], src->s6_addr32[1], 2263 src->s6_addr32[2], src->s6_addr32[3], srcp, 2264 dest->s6_addr32[0], dest->s6_addr32[1], 2265 dest->s6_addr32[2], dest->s6_addr32[3], destp, 2266 READ_ONCE(tw->tw_substate), 0, 0, 2267 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, 2268 refcount_read(&tw->tw_refcnt), tw); 2269 } 2270 2271 static int tcp6_seq_show(struct seq_file *seq, void *v) 2272 { 2273 struct tcp_iter_state *st; 2274 struct sock *sk = v; 2275 2276 if (v == SEQ_START_TOKEN) { 2277 seq_puts(seq, 2278 " sl " 2279 "local_address " 2280 "remote_address " 2281 "st tx_queue rx_queue tr tm->when retrnsmt" 2282 " uid timeout inode\n"); 2283 goto out; 2284 } 2285 st = seq->private; 2286 2287 if (sk->sk_state == TCP_TIME_WAIT) 2288 get_timewait6_sock(seq, v, st->num); 2289 else if (sk->sk_state == TCP_NEW_SYN_RECV) 2290 get_openreq6(seq, v, st->num); 2291 else 2292 get_tcp6_sock(seq, v, st->num); 2293 out: 2294 return 0; 2295 } 2296 2297 static const struct seq_operations tcp6_seq_ops = { 2298 .show = tcp6_seq_show, 2299 .start = tcp_seq_start, 2300 .next = tcp_seq_next, 2301 .stop = tcp_seq_stop, 2302 }; 2303 2304 static struct tcp_seq_afinfo tcp6_seq_afinfo = { 2305 .family = AF_INET6, 2306 }; 2307 2308 int __net_init tcp6_proc_init(struct net *net) 2309 { 2310 if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops, 2311 sizeof(struct tcp_iter_state), &tcp6_seq_afinfo)) 2312 return -ENOMEM; 2313 return 0; 2314 } 2315 2316 void tcp6_proc_exit(struct net *net) 2317 { 2318 remove_proc_entry("tcp6", net->proc_net); 2319 } 2320 #endif 2321 2322 struct proto tcpv6_prot = { 2323 .name = "TCPv6", 2324 .owner = THIS_MODULE, 2325 .close = tcp_close, 2326 .pre_connect = tcp_v6_pre_connect, 2327 .connect = tcp_v6_connect, 2328 .disconnect = tcp_disconnect, 2329 .accept = inet_csk_accept, 2330 .ioctl = tcp_ioctl, 2331 .init = tcp_v6_init_sock, 2332 .destroy = tcp_v4_destroy_sock, 2333 .shutdown = tcp_shutdown, 2334 .setsockopt = tcp_setsockopt, 2335 .getsockopt = tcp_getsockopt, 2336 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, 2337 .keepalive = tcp_set_keepalive, 2338 .recvmsg = tcp_recvmsg, 2339 .sendmsg = tcp_sendmsg, 2340 .splice_eof = tcp_splice_eof, 2341 .backlog_rcv = tcp_v6_do_rcv, 2342 .release_cb = tcp_release_cb, 2343 .hash = inet6_hash, 2344 .unhash = inet_unhash, 2345 .get_port = inet_csk_get_port, 2346 .put_port = inet_put_port, 2347 #ifdef CONFIG_BPF_SYSCALL 2348 .psock_update_sk_prot = tcp_bpf_update_proto, 2349 #endif 2350 .enter_memory_pressure = tcp_enter_memory_pressure, 2351 .leave_memory_pressure = tcp_leave_memory_pressure, 2352 .stream_memory_free = tcp_stream_memory_free, 2353 .sockets_allocated = &tcp_sockets_allocated, 2354 2355 .memory_allocated = &net_aligned_data.tcp_memory_allocated, 2356 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 2357 2358 .memory_pressure = &tcp_memory_pressure, 2359 .orphan_count = &tcp_orphan_count, 2360 .sysctl_mem = sysctl_tcp_mem, 2361 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 2362 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 2363 .max_header = MAX_TCP_HEADER, 2364 .obj_size = sizeof(struct tcp6_sock), 2365 .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6), 2366 .slab_flags = SLAB_TYPESAFE_BY_RCU, 2367 .twsk_prot = &tcp6_timewait_sock_ops, 2368 .rsk_prot = &tcp6_request_sock_ops, 2369 .h.hashinfo = NULL, 2370 .no_autobind = true, 2371 .diag_destroy = tcp_abort, 2372 }; 2373 EXPORT_SYMBOL_GPL(tcpv6_prot); 2374 2375 2376 static struct inet_protosw tcpv6_protosw = { 2377 .type = SOCK_STREAM, 2378 .protocol = IPPROTO_TCP, 2379 .prot = &tcpv6_prot, 2380 .ops = &inet6_stream_ops, 2381 .flags = INET_PROTOSW_PERMANENT | 2382 INET_PROTOSW_ICSK, 2383 }; 2384 2385 static int __net_init tcpv6_net_init(struct net *net) 2386 { 2387 int res; 2388 2389 res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6, 2390 SOCK_RAW, IPPROTO_TCP, net); 2391 if (!res) 2392 net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC; 2393 2394 return res; 2395 } 2396 2397 static void __net_exit tcpv6_net_exit(struct net *net) 2398 { 2399 inet_ctl_sock_destroy(net->ipv6.tcp_sk); 2400 } 2401 2402 static struct pernet_operations tcpv6_net_ops = { 2403 .init = tcpv6_net_init, 2404 .exit = tcpv6_net_exit, 2405 }; 2406 2407 int __init tcpv6_init(void) 2408 { 2409 int ret; 2410 2411 net_hotdata.tcpv6_protocol = (struct inet6_protocol) { 2412 .handler = tcp_v6_rcv, 2413 .err_handler = tcp_v6_err, 2414 .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL, 2415 }; 2416 ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2417 if (ret) 2418 goto out; 2419 2420 /* register inet6 protocol */ 2421 ret = inet6_register_protosw(&tcpv6_protosw); 2422 if (ret) 2423 goto out_tcpv6_protocol; 2424 2425 ret = register_pernet_subsys(&tcpv6_net_ops); 2426 if (ret) 2427 goto out_tcpv6_protosw; 2428 2429 ret = mptcpv6_init(); 2430 if (ret) 2431 goto out_tcpv6_pernet_subsys; 2432 2433 out: 2434 return ret; 2435 2436 out_tcpv6_pernet_subsys: 2437 unregister_pernet_subsys(&tcpv6_net_ops); 2438 out_tcpv6_protosw: 2439 inet6_unregister_protosw(&tcpv6_protosw); 2440 out_tcpv6_protocol: 2441 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2442 goto out; 2443 } 2444 2445 void tcpv6_exit(void) 2446 { 2447 unregister_pernet_subsys(&tcpv6_net_ops); 2448 inet6_unregister_protosw(&tcpv6_protosw); 2449 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2450 } 2451