177d8bf9cSArnaldo Carvalho de Melo /* 277d8bf9cSArnaldo Carvalho de Melo * INET An implementation of the TCP/IP protocol suite for the LINUX 377d8bf9cSArnaldo Carvalho de Melo * operating system. INET is implemented using the BSD Socket 477d8bf9cSArnaldo Carvalho de Melo * interface as the means of communication with the user level. 577d8bf9cSArnaldo Carvalho de Melo * 677d8bf9cSArnaldo Carvalho de Melo * Generic INET transport hashtables 777d8bf9cSArnaldo Carvalho de Melo * 877d8bf9cSArnaldo Carvalho de Melo * Authors: Lotsa people, from code originally in tcp 977d8bf9cSArnaldo Carvalho de Melo * 1077d8bf9cSArnaldo Carvalho de Melo * This program is free software; you can redistribute it and/or 1177d8bf9cSArnaldo Carvalho de Melo * modify it under the terms of the GNU General Public License 1277d8bf9cSArnaldo Carvalho de Melo * as published by the Free Software Foundation; either version 1377d8bf9cSArnaldo Carvalho de Melo * 2 of the License, or (at your option) any later version. 1477d8bf9cSArnaldo Carvalho de Melo */ 1577d8bf9cSArnaldo Carvalho de Melo 162d8c4ce5SArnaldo Carvalho de Melo #include <linux/module.h> 17a7f5e7f1SArnaldo Carvalho de Melo #include <linux/random.h> 18f3f05f70SArnaldo Carvalho de Melo #include <linux/sched.h> 1977d8bf9cSArnaldo Carvalho de Melo #include <linux/slab.h> 20f3f05f70SArnaldo Carvalho de Melo #include <linux/wait.h> 2177d8bf9cSArnaldo Carvalho de Melo 22463c84b9SArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 2377d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 246e5714eaSDavid S. Miller #include <net/secure_seq.h> 25a7f5e7f1SArnaldo Carvalho de Melo #include <net/ip.h> 2677d8bf9cSArnaldo Carvalho de Melo 276eada011SEric Dumazet static u32 inet_ehashfn(const struct net *net, const __be32 laddr, 2865cd8033SHannes Frederic Sowa const __u16 lport, const __be32 faddr, 2965cd8033SHannes Frederic Sowa const __be16 fport) 3065cd8033SHannes Frederic Sowa { 311bbdceefSHannes Frederic Sowa static u32 inet_ehash_secret __read_mostly; 321bbdceefSHannes Frederic Sowa 331bbdceefSHannes Frederic Sowa net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret)); 341bbdceefSHannes Frederic Sowa 3565cd8033SHannes Frederic Sowa return __inet_ehashfn(laddr, lport, faddr, fport, 3665cd8033SHannes Frederic Sowa inet_ehash_secret + net_hash_mix(net)); 3765cd8033SHannes Frederic Sowa } 3865cd8033SHannes Frederic Sowa 39*d1e559d0SEric Dumazet /* This function handles inet_sock, but also timewait and request sockets 40*d1e559d0SEric Dumazet * for IPv4/IPv6. 41*d1e559d0SEric Dumazet */ 425b441f76SEric Dumazet u32 sk_ehashfn(const struct sock *sk) 4365cd8033SHannes Frederic Sowa { 44*d1e559d0SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 45*d1e559d0SEric Dumazet if (sk->sk_family == AF_INET6 && 46*d1e559d0SEric Dumazet !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 47*d1e559d0SEric Dumazet return inet6_ehashfn(sock_net(sk), 48*d1e559d0SEric Dumazet &sk->sk_v6_rcv_saddr, sk->sk_num, 49*d1e559d0SEric Dumazet &sk->sk_v6_daddr, sk->sk_dport); 50*d1e559d0SEric Dumazet #endif 515b441f76SEric Dumazet return inet_ehashfn(sock_net(sk), 525b441f76SEric Dumazet sk->sk_rcv_saddr, sk->sk_num, 535b441f76SEric Dumazet sk->sk_daddr, sk->sk_dport); 5465cd8033SHannes Frederic Sowa } 5565cd8033SHannes Frederic Sowa 5677d8bf9cSArnaldo Carvalho de Melo /* 5777d8bf9cSArnaldo Carvalho de Melo * Allocate and initialize a new local port bind bucket. 5877d8bf9cSArnaldo Carvalho de Melo * The bindhash mutex for snum's hash chain must be held here. 5977d8bf9cSArnaldo Carvalho de Melo */ 60e18b890bSChristoph Lameter struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, 61941b1d22SPavel Emelyanov struct net *net, 6277d8bf9cSArnaldo Carvalho de Melo struct inet_bind_hashbucket *head, 6377d8bf9cSArnaldo Carvalho de Melo const unsigned short snum) 6477d8bf9cSArnaldo Carvalho de Melo { 6554e6ecb2SChristoph Lameter struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); 6677d8bf9cSArnaldo Carvalho de Melo 6777d8bf9cSArnaldo Carvalho de Melo if (tb != NULL) { 68efd7ef1cSEric W. Biederman write_pnet(&tb->ib_net, net); 6977d8bf9cSArnaldo Carvalho de Melo tb->port = snum; 7077d8bf9cSArnaldo Carvalho de Melo tb->fastreuse = 0; 71da5e3630STom Herbert tb->fastreuseport = 0; 72a9d8f911SEvgeniy Polyakov tb->num_owners = 0; 7377d8bf9cSArnaldo Carvalho de Melo INIT_HLIST_HEAD(&tb->owners); 7477d8bf9cSArnaldo Carvalho de Melo hlist_add_head(&tb->node, &head->chain); 7577d8bf9cSArnaldo Carvalho de Melo } 7677d8bf9cSArnaldo Carvalho de Melo return tb; 7777d8bf9cSArnaldo Carvalho de Melo } 7877d8bf9cSArnaldo Carvalho de Melo 7977d8bf9cSArnaldo Carvalho de Melo /* 8077d8bf9cSArnaldo Carvalho de Melo * Caller must hold hashbucket lock for this tb with local BH disabled 8177d8bf9cSArnaldo Carvalho de Melo */ 82e18b890bSChristoph Lameter void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb) 8377d8bf9cSArnaldo Carvalho de Melo { 8477d8bf9cSArnaldo Carvalho de Melo if (hlist_empty(&tb->owners)) { 8577d8bf9cSArnaldo Carvalho de Melo __hlist_del(&tb->node); 8677d8bf9cSArnaldo Carvalho de Melo kmem_cache_free(cachep, tb); 8777d8bf9cSArnaldo Carvalho de Melo } 8877d8bf9cSArnaldo Carvalho de Melo } 892d8c4ce5SArnaldo Carvalho de Melo 902d8c4ce5SArnaldo Carvalho de Melo void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, 912d8c4ce5SArnaldo Carvalho de Melo const unsigned short snum) 922d8c4ce5SArnaldo Carvalho de Melo { 93a9d8f911SEvgeniy Polyakov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 94a9d8f911SEvgeniy Polyakov 9524dd1fa1SEric Dumazet atomic_inc(&hashinfo->bsockets); 96a9d8f911SEvgeniy Polyakov 97c720c7e8SEric Dumazet inet_sk(sk)->inet_num = snum; 982d8c4ce5SArnaldo Carvalho de Melo sk_add_bind_node(sk, &tb->owners); 99a9d8f911SEvgeniy Polyakov tb->num_owners++; 100463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = tb; 1012d8c4ce5SArnaldo Carvalho de Melo } 1022d8c4ce5SArnaldo Carvalho de Melo 1032d8c4ce5SArnaldo Carvalho de Melo /* 1042d8c4ce5SArnaldo Carvalho de Melo * Get rid of any references to a local port held by the given sock. 1052d8c4ce5SArnaldo Carvalho de Melo */ 106ab1e0a13SArnaldo Carvalho de Melo static void __inet_put_port(struct sock *sk) 1072d8c4ce5SArnaldo Carvalho de Melo { 10839d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 109c720c7e8SEric Dumazet const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num, 1107f635ab7SPavel Emelyanov hashinfo->bhash_size); 1112d8c4ce5SArnaldo Carvalho de Melo struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; 1122d8c4ce5SArnaldo Carvalho de Melo struct inet_bind_bucket *tb; 1132d8c4ce5SArnaldo Carvalho de Melo 11424dd1fa1SEric Dumazet atomic_dec(&hashinfo->bsockets); 115a9d8f911SEvgeniy Polyakov 1162d8c4ce5SArnaldo Carvalho de Melo spin_lock(&head->lock); 117463c84b9SArnaldo Carvalho de Melo tb = inet_csk(sk)->icsk_bind_hash; 1182d8c4ce5SArnaldo Carvalho de Melo __sk_del_bind_node(sk); 119a9d8f911SEvgeniy Polyakov tb->num_owners--; 120463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = NULL; 121c720c7e8SEric Dumazet inet_sk(sk)->inet_num = 0; 1222d8c4ce5SArnaldo Carvalho de Melo inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 1232d8c4ce5SArnaldo Carvalho de Melo spin_unlock(&head->lock); 1242d8c4ce5SArnaldo Carvalho de Melo } 1252d8c4ce5SArnaldo Carvalho de Melo 126ab1e0a13SArnaldo Carvalho de Melo void inet_put_port(struct sock *sk) 1272d8c4ce5SArnaldo Carvalho de Melo { 1282d8c4ce5SArnaldo Carvalho de Melo local_bh_disable(); 129ab1e0a13SArnaldo Carvalho de Melo __inet_put_port(sk); 1302d8c4ce5SArnaldo Carvalho de Melo local_bh_enable(); 1312d8c4ce5SArnaldo Carvalho de Melo } 1322d8c4ce5SArnaldo Carvalho de Melo EXPORT_SYMBOL(inet_put_port); 133f3f05f70SArnaldo Carvalho de Melo 134093d2823SBalazs Scheidler int __inet_inherit_port(struct sock *sk, struct sock *child) 13553083773SPavel Emelyanov { 13653083773SPavel Emelyanov struct inet_hashinfo *table = sk->sk_prot->h.hashinfo; 137093d2823SBalazs Scheidler unsigned short port = inet_sk(child)->inet_num; 138093d2823SBalazs Scheidler const int bhash = inet_bhashfn(sock_net(sk), port, 1397f635ab7SPavel Emelyanov table->bhash_size); 14053083773SPavel Emelyanov struct inet_bind_hashbucket *head = &table->bhash[bhash]; 14153083773SPavel Emelyanov struct inet_bind_bucket *tb; 14253083773SPavel Emelyanov 14353083773SPavel Emelyanov spin_lock(&head->lock); 14453083773SPavel Emelyanov tb = inet_csk(sk)->icsk_bind_hash; 145093d2823SBalazs Scheidler if (tb->port != port) { 146093d2823SBalazs Scheidler /* NOTE: using tproxy and redirecting skbs to a proxy 147093d2823SBalazs Scheidler * on a different listener port breaks the assumption 148093d2823SBalazs Scheidler * that the listener socket's icsk_bind_hash is the same 149093d2823SBalazs Scheidler * as that of the child socket. We have to look up or 150093d2823SBalazs Scheidler * create a new bind bucket for the child here. */ 151b67bfe0dSSasha Levin inet_bind_bucket_for_each(tb, &head->chain) { 152093d2823SBalazs Scheidler if (net_eq(ib_net(tb), sock_net(sk)) && 153093d2823SBalazs Scheidler tb->port == port) 154093d2823SBalazs Scheidler break; 155093d2823SBalazs Scheidler } 156b67bfe0dSSasha Levin if (!tb) { 157093d2823SBalazs Scheidler tb = inet_bind_bucket_create(table->bind_bucket_cachep, 158093d2823SBalazs Scheidler sock_net(sk), head, port); 159093d2823SBalazs Scheidler if (!tb) { 160093d2823SBalazs Scheidler spin_unlock(&head->lock); 161093d2823SBalazs Scheidler return -ENOMEM; 162093d2823SBalazs Scheidler } 163093d2823SBalazs Scheidler } 164093d2823SBalazs Scheidler } 165b4ff3c90SNagendra Tomar inet_bind_hash(child, tb, port); 16653083773SPavel Emelyanov spin_unlock(&head->lock); 167093d2823SBalazs Scheidler 168093d2823SBalazs Scheidler return 0; 16953083773SPavel Emelyanov } 17053083773SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_inherit_port); 17153083773SPavel Emelyanov 172c25eb3bfSEric Dumazet static inline int compute_score(struct sock *sk, struct net *net, 173c25eb3bfSEric Dumazet const unsigned short hnum, const __be32 daddr, 174c25eb3bfSEric Dumazet const int dif) 175c25eb3bfSEric Dumazet { 176c25eb3bfSEric Dumazet int score = -1; 177c25eb3bfSEric Dumazet struct inet_sock *inet = inet_sk(sk); 178c25eb3bfSEric Dumazet 179c720c7e8SEric Dumazet if (net_eq(sock_net(sk), net) && inet->inet_num == hnum && 180c25eb3bfSEric Dumazet !ipv6_only_sock(sk)) { 181c720c7e8SEric Dumazet __be32 rcv_saddr = inet->inet_rcv_saddr; 182da5e3630STom Herbert score = sk->sk_family == PF_INET ? 2 : 1; 183c25eb3bfSEric Dumazet if (rcv_saddr) { 184c25eb3bfSEric Dumazet if (rcv_saddr != daddr) 185c25eb3bfSEric Dumazet return -1; 186da5e3630STom Herbert score += 4; 187c25eb3bfSEric Dumazet } 188c25eb3bfSEric Dumazet if (sk->sk_bound_dev_if) { 189c25eb3bfSEric Dumazet if (sk->sk_bound_dev_if != dif) 190c25eb3bfSEric Dumazet return -1; 191da5e3630STom Herbert score += 4; 192c25eb3bfSEric Dumazet } 193c25eb3bfSEric Dumazet } 194c25eb3bfSEric Dumazet return score; 195c25eb3bfSEric Dumazet } 196c25eb3bfSEric Dumazet 197f3f05f70SArnaldo Carvalho de Melo /* 19833b62231SArnaldo Carvalho de Melo * Don't inline this cruft. Here are some nice properties to exploit here. The 19933b62231SArnaldo Carvalho de Melo * BSD API does not allow a listening sock to specify the remote port nor the 20033b62231SArnaldo Carvalho de Melo * remote address for the connection. So always assume those are both 20133b62231SArnaldo Carvalho de Melo * wildcarded during the search since they can never be otherwise. 20233b62231SArnaldo Carvalho de Melo */ 20333b62231SArnaldo Carvalho de Melo 20433b62231SArnaldo Carvalho de Melo 205c67499c0SPavel Emelyanov struct sock *__inet_lookup_listener(struct net *net, 206c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 207da5e3630STom Herbert const __be32 saddr, __be16 sport, 208fb99c848SAl Viro const __be32 daddr, const unsigned short hnum, 20999a92ff5SHerbert Xu const int dif) 21099a92ff5SHerbert Xu { 211c25eb3bfSEric Dumazet struct sock *sk, *result; 212c25eb3bfSEric Dumazet struct hlist_nulls_node *node; 213c25eb3bfSEric Dumazet unsigned int hash = inet_lhashfn(net, hnum); 214c25eb3bfSEric Dumazet struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; 215da5e3630STom Herbert int score, hiscore, matches = 0, reuseport = 0; 216da5e3630STom Herbert u32 phash = 0; 21799a92ff5SHerbert Xu 218c25eb3bfSEric Dumazet rcu_read_lock(); 219c25eb3bfSEric Dumazet begin: 220c25eb3bfSEric Dumazet result = NULL; 221da5e3630STom Herbert hiscore = 0; 222c25eb3bfSEric Dumazet sk_nulls_for_each_rcu(sk, node, &ilb->head) { 223c25eb3bfSEric Dumazet score = compute_score(sk, net, hnum, daddr, dif); 224c25eb3bfSEric Dumazet if (score > hiscore) { 225c25eb3bfSEric Dumazet result = sk; 226c25eb3bfSEric Dumazet hiscore = score; 227da5e3630STom Herbert reuseport = sk->sk_reuseport; 228da5e3630STom Herbert if (reuseport) { 229da5e3630STom Herbert phash = inet_ehashfn(net, daddr, hnum, 230da5e3630STom Herbert saddr, sport); 231da5e3630STom Herbert matches = 1; 232da5e3630STom Herbert } 233da5e3630STom Herbert } else if (score == hiscore && reuseport) { 234da5e3630STom Herbert matches++; 2358fc54f68SDaniel Borkmann if (reciprocal_scale(phash, matches) == 0) 236da5e3630STom Herbert result = sk; 237da5e3630STom Herbert phash = next_pseudo_random32(phash); 23899a92ff5SHerbert Xu } 23999a92ff5SHerbert Xu } 240c25eb3bfSEric Dumazet /* 241c25eb3bfSEric Dumazet * if the nulls value we got at the end of this lookup is 242c25eb3bfSEric Dumazet * not the expected one, we must restart lookup. 243c25eb3bfSEric Dumazet * We probably met an item that was moved to another chain. 244c25eb3bfSEric Dumazet */ 245c25eb3bfSEric Dumazet if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE) 246c25eb3bfSEric Dumazet goto begin; 247c25eb3bfSEric Dumazet if (result) { 248c25eb3bfSEric Dumazet if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt))) 249c25eb3bfSEric Dumazet result = NULL; 250c25eb3bfSEric Dumazet else if (unlikely(compute_score(result, net, hnum, daddr, 251c25eb3bfSEric Dumazet dif) < hiscore)) { 252c25eb3bfSEric Dumazet sock_put(result); 253c25eb3bfSEric Dumazet goto begin; 254c25eb3bfSEric Dumazet } 255c25eb3bfSEric Dumazet } 256c25eb3bfSEric Dumazet rcu_read_unlock(); 257c25eb3bfSEric Dumazet return result; 25899a92ff5SHerbert Xu } 2598f491069SHerbert Xu EXPORT_SYMBOL_GPL(__inet_lookup_listener); 260a7f5e7f1SArnaldo Carvalho de Melo 26105dbc7b5SEric Dumazet /* All sockets share common refcount, but have different destructors */ 26205dbc7b5SEric Dumazet void sock_gen_put(struct sock *sk) 26305dbc7b5SEric Dumazet { 26405dbc7b5SEric Dumazet if (!atomic_dec_and_test(&sk->sk_refcnt)) 26505dbc7b5SEric Dumazet return; 26605dbc7b5SEric Dumazet 26705dbc7b5SEric Dumazet if (sk->sk_state == TCP_TIME_WAIT) 26805dbc7b5SEric Dumazet inet_twsk_free(inet_twsk(sk)); 26941b822c5SEric Dumazet else if (sk->sk_state == TCP_NEW_SYN_RECV) 27041b822c5SEric Dumazet reqsk_free(inet_reqsk(sk)); 27105dbc7b5SEric Dumazet else 27205dbc7b5SEric Dumazet sk_free(sk); 27305dbc7b5SEric Dumazet } 27405dbc7b5SEric Dumazet EXPORT_SYMBOL_GPL(sock_gen_put); 27505dbc7b5SEric Dumazet 2762c13270bSEric Dumazet void sock_edemux(struct sk_buff *skb) 2772c13270bSEric Dumazet { 2782c13270bSEric Dumazet sock_gen_put(skb->sk); 2792c13270bSEric Dumazet } 2802c13270bSEric Dumazet EXPORT_SYMBOL(sock_edemux); 2812c13270bSEric Dumazet 282c67499c0SPavel Emelyanov struct sock *__inet_lookup_established(struct net *net, 283c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 28477a5ba55SPavel Emelyanov const __be32 saddr, const __be16 sport, 28577a5ba55SPavel Emelyanov const __be32 daddr, const u16 hnum, 28677a5ba55SPavel Emelyanov const int dif) 28777a5ba55SPavel Emelyanov { 288c7228317SJoe Perches INET_ADDR_COOKIE(acookie, saddr, daddr); 28977a5ba55SPavel Emelyanov const __portpair ports = INET_COMBINED_PORTS(sport, hnum); 29077a5ba55SPavel Emelyanov struct sock *sk; 2913ab5aee7SEric Dumazet const struct hlist_nulls_node *node; 29277a5ba55SPavel Emelyanov /* Optimize here for direct hit, only listening connections can 29377a5ba55SPavel Emelyanov * have wildcards anyways. 29477a5ba55SPavel Emelyanov */ 2959f26b3adSPavel Emelyanov unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport); 296f373b53bSEric Dumazet unsigned int slot = hash & hashinfo->ehash_mask; 2973ab5aee7SEric Dumazet struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; 29877a5ba55SPavel Emelyanov 2993ab5aee7SEric Dumazet rcu_read_lock(); 3003ab5aee7SEric Dumazet begin: 3013ab5aee7SEric Dumazet sk_nulls_for_each_rcu(sk, node, &head->chain) { 302ce43b03eSEric Dumazet if (sk->sk_hash != hash) 303ce43b03eSEric Dumazet continue; 304ce43b03eSEric Dumazet if (likely(INET_MATCH(sk, net, acookie, 305ce43b03eSEric Dumazet saddr, daddr, ports, dif))) { 3063ab5aee7SEric Dumazet if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) 30705dbc7b5SEric Dumazet goto out; 308ce43b03eSEric Dumazet if (unlikely(!INET_MATCH(sk, net, acookie, 3093ab5aee7SEric Dumazet saddr, daddr, ports, dif))) { 31005dbc7b5SEric Dumazet sock_gen_put(sk); 3113ab5aee7SEric Dumazet goto begin; 31277a5ba55SPavel Emelyanov } 31305dbc7b5SEric Dumazet goto found; 3143ab5aee7SEric Dumazet } 3153ab5aee7SEric Dumazet } 3163ab5aee7SEric Dumazet /* 3173ab5aee7SEric Dumazet * if the nulls value we got at the end of this lookup is 3183ab5aee7SEric Dumazet * not the expected one, we must restart lookup. 3193ab5aee7SEric Dumazet * We probably met an item that was moved to another chain. 3203ab5aee7SEric Dumazet */ 3213ab5aee7SEric Dumazet if (get_nulls_value(node) != slot) 3223ab5aee7SEric Dumazet goto begin; 32377a5ba55SPavel Emelyanov out: 32405dbc7b5SEric Dumazet sk = NULL; 32505dbc7b5SEric Dumazet found: 3263ab5aee7SEric Dumazet rcu_read_unlock(); 32777a5ba55SPavel Emelyanov return sk; 32877a5ba55SPavel Emelyanov } 32977a5ba55SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_lookup_established); 33077a5ba55SPavel Emelyanov 331a7f5e7f1SArnaldo Carvalho de Melo /* called with local bh disabled */ 332a7f5e7f1SArnaldo Carvalho de Melo static int __inet_check_established(struct inet_timewait_death_row *death_row, 333a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk, __u16 lport, 334a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock **twp) 335a7f5e7f1SArnaldo Carvalho de Melo { 336a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 337a7f5e7f1SArnaldo Carvalho de Melo struct inet_sock *inet = inet_sk(sk); 338c720c7e8SEric Dumazet __be32 daddr = inet->inet_rcv_saddr; 339c720c7e8SEric Dumazet __be32 saddr = inet->inet_daddr; 340a7f5e7f1SArnaldo Carvalho de Melo int dif = sk->sk_bound_dev_if; 341c7228317SJoe Perches INET_ADDR_COOKIE(acookie, saddr, daddr); 342c720c7e8SEric Dumazet const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); 3439f26b3adSPavel Emelyanov struct net *net = sock_net(sk); 344c720c7e8SEric Dumazet unsigned int hash = inet_ehashfn(net, daddr, lport, 345c720c7e8SEric Dumazet saddr, inet->inet_dport); 346a7f5e7f1SArnaldo Carvalho de Melo struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); 3479db66bdcSEric Dumazet spinlock_t *lock = inet_ehash_lockp(hinfo, hash); 348a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk2; 3493ab5aee7SEric Dumazet const struct hlist_nulls_node *node; 35005dbc7b5SEric Dumazet struct inet_timewait_sock *tw = NULL; 35113475a30SEric Dumazet int twrefcnt = 0; 352a7f5e7f1SArnaldo Carvalho de Melo 3539db66bdcSEric Dumazet spin_lock(lock); 354a7f5e7f1SArnaldo Carvalho de Melo 3553ab5aee7SEric Dumazet sk_nulls_for_each(sk2, node, &head->chain) { 356ce43b03eSEric Dumazet if (sk2->sk_hash != hash) 357ce43b03eSEric Dumazet continue; 35805dbc7b5SEric Dumazet 359ce43b03eSEric Dumazet if (likely(INET_MATCH(sk2, net, acookie, 36005dbc7b5SEric Dumazet saddr, daddr, ports, dif))) { 36105dbc7b5SEric Dumazet if (sk2->sk_state == TCP_TIME_WAIT) { 36205dbc7b5SEric Dumazet tw = inet_twsk(sk2); 36305dbc7b5SEric Dumazet if (twsk_unique(sk, sk2, twp)) 36405dbc7b5SEric Dumazet break; 36505dbc7b5SEric Dumazet } 366a7f5e7f1SArnaldo Carvalho de Melo goto not_unique; 367a7f5e7f1SArnaldo Carvalho de Melo } 36805dbc7b5SEric Dumazet } 369a7f5e7f1SArnaldo Carvalho de Melo 370a7f5e7f1SArnaldo Carvalho de Melo /* Must record num and sport now. Otherwise we will see 37105dbc7b5SEric Dumazet * in hash table socket with a funny identity. 37205dbc7b5SEric Dumazet */ 373c720c7e8SEric Dumazet inet->inet_num = lport; 374c720c7e8SEric Dumazet inet->inet_sport = htons(lport); 375a7f5e7f1SArnaldo Carvalho de Melo sk->sk_hash = hash; 376547b792cSIlpo Järvinen WARN_ON(!sk_unhashed(sk)); 3773ab5aee7SEric Dumazet __sk_nulls_add_node_rcu(sk, &head->chain); 37813475a30SEric Dumazet if (tw) { 37913475a30SEric Dumazet twrefcnt = inet_twsk_unhash(tw); 38013475a30SEric Dumazet NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED); 38113475a30SEric Dumazet } 3829db66bdcSEric Dumazet spin_unlock(lock); 38313475a30SEric Dumazet if (twrefcnt) 38413475a30SEric Dumazet inet_twsk_put(tw); 385c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 386a7f5e7f1SArnaldo Carvalho de Melo 387a7f5e7f1SArnaldo Carvalho de Melo if (twp) { 388a7f5e7f1SArnaldo Carvalho de Melo *twp = tw; 389a7f5e7f1SArnaldo Carvalho de Melo } else if (tw) { 390a7f5e7f1SArnaldo Carvalho de Melo /* Silly. Should hash-dance instead... */ 391a7f5e7f1SArnaldo Carvalho de Melo inet_twsk_deschedule(tw, death_row); 392a7f5e7f1SArnaldo Carvalho de Melo 393a7f5e7f1SArnaldo Carvalho de Melo inet_twsk_put(tw); 394a7f5e7f1SArnaldo Carvalho de Melo } 395a7f5e7f1SArnaldo Carvalho de Melo return 0; 396a7f5e7f1SArnaldo Carvalho de Melo 397a7f5e7f1SArnaldo Carvalho de Melo not_unique: 3989db66bdcSEric Dumazet spin_unlock(lock); 399a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 400a7f5e7f1SArnaldo Carvalho de Melo } 401a7f5e7f1SArnaldo Carvalho de Melo 402a7f5e7f1SArnaldo Carvalho de Melo static inline u32 inet_sk_port_offset(const struct sock *sk) 403a7f5e7f1SArnaldo Carvalho de Melo { 404a7f5e7f1SArnaldo Carvalho de Melo const struct inet_sock *inet = inet_sk(sk); 405c720c7e8SEric Dumazet return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr, 406c720c7e8SEric Dumazet inet->inet_daddr, 407c720c7e8SEric Dumazet inet->inet_dport); 408a7f5e7f1SArnaldo Carvalho de Melo } 409a7f5e7f1SArnaldo Carvalho de Melo 4109327f705SEric Dumazet int __inet_hash_nolisten(struct sock *sk, struct inet_timewait_sock *tw) 411152da81dSPavel Emelyanov { 41239d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 4133ab5aee7SEric Dumazet struct hlist_nulls_head *list; 414152da81dSPavel Emelyanov struct inet_ehash_bucket *head; 4155b441f76SEric Dumazet spinlock_t *lock; 4169327f705SEric Dumazet int twrefcnt = 0; 417152da81dSPavel Emelyanov 418547b792cSIlpo Järvinen WARN_ON(!sk_unhashed(sk)); 419152da81dSPavel Emelyanov 4205b441f76SEric Dumazet sk->sk_hash = sk_ehashfn(sk); 421152da81dSPavel Emelyanov head = inet_ehash_bucket(hashinfo, sk->sk_hash); 422152da81dSPavel Emelyanov list = &head->chain; 423152da81dSPavel Emelyanov lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 424152da81dSPavel Emelyanov 4259db66bdcSEric Dumazet spin_lock(lock); 4263ab5aee7SEric Dumazet __sk_nulls_add_node_rcu(sk, list); 4279327f705SEric Dumazet if (tw) { 4289327f705SEric Dumazet WARN_ON(sk->sk_hash != tw->tw_hash); 4299327f705SEric Dumazet twrefcnt = inet_twsk_unhash(tw); 4309327f705SEric Dumazet } 4319db66bdcSEric Dumazet spin_unlock(lock); 432c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 4339327f705SEric Dumazet return twrefcnt; 434152da81dSPavel Emelyanov } 435152da81dSPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_hash_nolisten); 436152da81dSPavel Emelyanov 437ab1e0a13SArnaldo Carvalho de Melo static void __inet_hash(struct sock *sk) 438152da81dSPavel Emelyanov { 43939d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 4405caea4eaSEric Dumazet struct inet_listen_hashbucket *ilb; 441152da81dSPavel Emelyanov 442152da81dSPavel Emelyanov if (sk->sk_state != TCP_LISTEN) { 4439327f705SEric Dumazet __inet_hash_nolisten(sk, NULL); 444152da81dSPavel Emelyanov return; 445152da81dSPavel Emelyanov } 446152da81dSPavel Emelyanov 447547b792cSIlpo Järvinen WARN_ON(!sk_unhashed(sk)); 4485caea4eaSEric Dumazet ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)]; 449152da81dSPavel Emelyanov 4505caea4eaSEric Dumazet spin_lock(&ilb->lock); 451c25eb3bfSEric Dumazet __sk_nulls_add_node_rcu(sk, &ilb->head); 452c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 4535caea4eaSEric Dumazet spin_unlock(&ilb->lock); 454152da81dSPavel Emelyanov } 455ab1e0a13SArnaldo Carvalho de Melo 456ab1e0a13SArnaldo Carvalho de Melo void inet_hash(struct sock *sk) 457ab1e0a13SArnaldo Carvalho de Melo { 458ab1e0a13SArnaldo Carvalho de Melo if (sk->sk_state != TCP_CLOSE) { 459ab1e0a13SArnaldo Carvalho de Melo local_bh_disable(); 460ab1e0a13SArnaldo Carvalho de Melo __inet_hash(sk); 461ab1e0a13SArnaldo Carvalho de Melo local_bh_enable(); 462ab1e0a13SArnaldo Carvalho de Melo } 463ab1e0a13SArnaldo Carvalho de Melo } 464ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash); 465ab1e0a13SArnaldo Carvalho de Melo 466ab1e0a13SArnaldo Carvalho de Melo void inet_unhash(struct sock *sk) 467ab1e0a13SArnaldo Carvalho de Melo { 46839d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 469c25eb3bfSEric Dumazet spinlock_t *lock; 470c25eb3bfSEric Dumazet int done; 471ab1e0a13SArnaldo Carvalho de Melo 472ab1e0a13SArnaldo Carvalho de Melo if (sk_unhashed(sk)) 4735caea4eaSEric Dumazet return; 474ab1e0a13SArnaldo Carvalho de Melo 475c25eb3bfSEric Dumazet if (sk->sk_state == TCP_LISTEN) 476c25eb3bfSEric Dumazet lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock; 477c25eb3bfSEric Dumazet else 478c25eb3bfSEric Dumazet lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 4795caea4eaSEric Dumazet 4809db66bdcSEric Dumazet spin_lock_bh(lock); 481c25eb3bfSEric Dumazet done = __sk_nulls_del_node_init_rcu(sk); 482c25eb3bfSEric Dumazet if (done) 483c25eb3bfSEric Dumazet sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 484920de804SEric Dumazet spin_unlock_bh(lock); 485ab1e0a13SArnaldo Carvalho de Melo } 486ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_unhash); 487152da81dSPavel Emelyanov 4885ee31fc1SPavel Emelyanov int __inet_hash_connect(struct inet_timewait_death_row *death_row, 4895d8c0aa9SPavel Emelyanov struct sock *sk, u32 port_offset, 4905ee31fc1SPavel Emelyanov int (*check_established)(struct inet_timewait_death_row *, 4915ee31fc1SPavel Emelyanov struct sock *, __u16, struct inet_timewait_sock **), 4929327f705SEric Dumazet int (*hash)(struct sock *sk, struct inet_timewait_sock *twp)) 493a7f5e7f1SArnaldo Carvalho de Melo { 494a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 495c720c7e8SEric Dumazet const unsigned short snum = inet_sk(sk)->inet_num; 496a7f5e7f1SArnaldo Carvalho de Melo struct inet_bind_hashbucket *head; 497a7f5e7f1SArnaldo Carvalho de Melo struct inet_bind_bucket *tb; 498a7f5e7f1SArnaldo Carvalho de Melo int ret; 4993b1e0a65SYOSHIFUJI Hideaki struct net *net = sock_net(sk); 5009327f705SEric Dumazet int twrefcnt = 1; 501a7f5e7f1SArnaldo Carvalho de Melo 502a7f5e7f1SArnaldo Carvalho de Melo if (!snum) { 503227b60f5SStephen Hemminger int i, remaining, low, high, port; 504a7f5e7f1SArnaldo Carvalho de Melo static u32 hint; 5055d8c0aa9SPavel Emelyanov u32 offset = hint + port_offset; 506a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock *tw = NULL; 507a7f5e7f1SArnaldo Carvalho de Melo 5080bbf87d8SEric W. Biederman inet_get_local_port_range(net, &low, &high); 509a25de534SAnton Arapov remaining = (high - low) + 1; 510227b60f5SStephen Hemminger 511a7f5e7f1SArnaldo Carvalho de Melo local_bh_disable(); 512227b60f5SStephen Hemminger for (i = 1; i <= remaining; i++) { 513227b60f5SStephen Hemminger port = low + (i + offset) % remaining; 514122ff243SWANG Cong if (inet_is_local_reserved_port(net, port)) 515e3826f1eSAmerigo Wang continue; 5167f635ab7SPavel Emelyanov head = &hinfo->bhash[inet_bhashfn(net, port, 5177f635ab7SPavel Emelyanov hinfo->bhash_size)]; 518a7f5e7f1SArnaldo Carvalho de Melo spin_lock(&head->lock); 519a7f5e7f1SArnaldo Carvalho de Melo 520a7f5e7f1SArnaldo Carvalho de Melo /* Does not bother with rcv_saddr checks, 521a7f5e7f1SArnaldo Carvalho de Melo * because the established check is already 522a7f5e7f1SArnaldo Carvalho de Melo * unique enough. 523a7f5e7f1SArnaldo Carvalho de Melo */ 524b67bfe0dSSasha Levin inet_bind_bucket_for_each(tb, &head->chain) { 52509ad9bc7SOctavian Purdila if (net_eq(ib_net(tb), net) && 52609ad9bc7SOctavian Purdila tb->port == port) { 527da5e3630STom Herbert if (tb->fastreuse >= 0 || 528da5e3630STom Herbert tb->fastreuseport >= 0) 529a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 530a9d8f911SEvgeniy Polyakov WARN_ON(hlist_empty(&tb->owners)); 5315ee31fc1SPavel Emelyanov if (!check_established(death_row, sk, 5325ee31fc1SPavel Emelyanov port, &tw)) 533a7f5e7f1SArnaldo Carvalho de Melo goto ok; 534a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 535a7f5e7f1SArnaldo Carvalho de Melo } 536a7f5e7f1SArnaldo Carvalho de Melo } 537a7f5e7f1SArnaldo Carvalho de Melo 538941b1d22SPavel Emelyanov tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, 539941b1d22SPavel Emelyanov net, head, port); 540a7f5e7f1SArnaldo Carvalho de Melo if (!tb) { 541a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 542a7f5e7f1SArnaldo Carvalho de Melo break; 543a7f5e7f1SArnaldo Carvalho de Melo } 544a7f5e7f1SArnaldo Carvalho de Melo tb->fastreuse = -1; 545da5e3630STom Herbert tb->fastreuseport = -1; 546a7f5e7f1SArnaldo Carvalho de Melo goto ok; 547a7f5e7f1SArnaldo Carvalho de Melo 548a7f5e7f1SArnaldo Carvalho de Melo next_port: 549a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 550a7f5e7f1SArnaldo Carvalho de Melo } 551a7f5e7f1SArnaldo Carvalho de Melo local_bh_enable(); 552a7f5e7f1SArnaldo Carvalho de Melo 553a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 554a7f5e7f1SArnaldo Carvalho de Melo 555a7f5e7f1SArnaldo Carvalho de Melo ok: 556a7f5e7f1SArnaldo Carvalho de Melo hint += i; 557a7f5e7f1SArnaldo Carvalho de Melo 558a7f5e7f1SArnaldo Carvalho de Melo /* Head lock still held and bh's disabled */ 559a7f5e7f1SArnaldo Carvalho de Melo inet_bind_hash(sk, tb, port); 560a7f5e7f1SArnaldo Carvalho de Melo if (sk_unhashed(sk)) { 561c720c7e8SEric Dumazet inet_sk(sk)->inet_sport = htons(port); 5629327f705SEric Dumazet twrefcnt += hash(sk, tw); 563a7f5e7f1SArnaldo Carvalho de Melo } 5643cdaedaeSEric Dumazet if (tw) 5653cdaedaeSEric Dumazet twrefcnt += inet_twsk_bind_unhash(tw, hinfo); 566a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 567a7f5e7f1SArnaldo Carvalho de Melo 568a7f5e7f1SArnaldo Carvalho de Melo if (tw) { 56953b3531bSAlexey Dobriyan inet_twsk_deschedule(tw, death_row); 5709327f705SEric Dumazet while (twrefcnt) { 5719327f705SEric Dumazet twrefcnt--; 572a7f5e7f1SArnaldo Carvalho de Melo inet_twsk_put(tw); 573a7f5e7f1SArnaldo Carvalho de Melo } 5749327f705SEric Dumazet } 575a7f5e7f1SArnaldo Carvalho de Melo 576a7f5e7f1SArnaldo Carvalho de Melo ret = 0; 577a7f5e7f1SArnaldo Carvalho de Melo goto out; 578a7f5e7f1SArnaldo Carvalho de Melo } 579a7f5e7f1SArnaldo Carvalho de Melo 5807f635ab7SPavel Emelyanov head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)]; 581a7f5e7f1SArnaldo Carvalho de Melo tb = inet_csk(sk)->icsk_bind_hash; 582a7f5e7f1SArnaldo Carvalho de Melo spin_lock_bh(&head->lock); 583a7f5e7f1SArnaldo Carvalho de Melo if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) { 5849327f705SEric Dumazet hash(sk, NULL); 585a7f5e7f1SArnaldo Carvalho de Melo spin_unlock_bh(&head->lock); 586a7f5e7f1SArnaldo Carvalho de Melo return 0; 587a7f5e7f1SArnaldo Carvalho de Melo } else { 588a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 589a7f5e7f1SArnaldo Carvalho de Melo /* No definite answer... Walk to established hash table */ 5905ee31fc1SPavel Emelyanov ret = check_established(death_row, sk, snum, NULL); 591a7f5e7f1SArnaldo Carvalho de Melo out: 592a7f5e7f1SArnaldo Carvalho de Melo local_bh_enable(); 593a7f5e7f1SArnaldo Carvalho de Melo return ret; 594a7f5e7f1SArnaldo Carvalho de Melo } 595a7f5e7f1SArnaldo Carvalho de Melo } 5965ee31fc1SPavel Emelyanov 5975ee31fc1SPavel Emelyanov /* 5985ee31fc1SPavel Emelyanov * Bind a port for a connect operation and hash it. 5995ee31fc1SPavel Emelyanov */ 6005ee31fc1SPavel Emelyanov int inet_hash_connect(struct inet_timewait_death_row *death_row, 6015ee31fc1SPavel Emelyanov struct sock *sk) 6025ee31fc1SPavel Emelyanov { 6035d8c0aa9SPavel Emelyanov return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk), 6045ee31fc1SPavel Emelyanov __inet_check_established, __inet_hash_nolisten); 6055ee31fc1SPavel Emelyanov } 606a7f5e7f1SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash_connect); 6075caea4eaSEric Dumazet 6085caea4eaSEric Dumazet void inet_hashinfo_init(struct inet_hashinfo *h) 6095caea4eaSEric Dumazet { 6105caea4eaSEric Dumazet int i; 6115caea4eaSEric Dumazet 61224dd1fa1SEric Dumazet atomic_set(&h->bsockets, 0); 613c25eb3bfSEric Dumazet for (i = 0; i < INET_LHTABLE_SIZE; i++) { 6145caea4eaSEric Dumazet spin_lock_init(&h->listening_hash[i].lock); 615c25eb3bfSEric Dumazet INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head, 616c25eb3bfSEric Dumazet i + LISTENING_NULLS_BASE); 617c25eb3bfSEric Dumazet } 6185caea4eaSEric Dumazet } 6195caea4eaSEric Dumazet EXPORT_SYMBOL_GPL(inet_hashinfo_init); 620