177d8bf9cSArnaldo Carvalho de Melo /* 277d8bf9cSArnaldo Carvalho de Melo * INET An implementation of the TCP/IP protocol suite for the LINUX 377d8bf9cSArnaldo Carvalho de Melo * operating system. INET is implemented using the BSD Socket 477d8bf9cSArnaldo Carvalho de Melo * interface as the means of communication with the user level. 577d8bf9cSArnaldo Carvalho de Melo * 677d8bf9cSArnaldo Carvalho de Melo * Generic INET transport hashtables 777d8bf9cSArnaldo Carvalho de Melo * 877d8bf9cSArnaldo Carvalho de Melo * Authors: Lotsa people, from code originally in tcp 977d8bf9cSArnaldo Carvalho de Melo * 1077d8bf9cSArnaldo Carvalho de Melo * This program is free software; you can redistribute it and/or 1177d8bf9cSArnaldo Carvalho de Melo * modify it under the terms of the GNU General Public License 1277d8bf9cSArnaldo Carvalho de Melo * as published by the Free Software Foundation; either version 1377d8bf9cSArnaldo Carvalho de Melo * 2 of the License, or (at your option) any later version. 1477d8bf9cSArnaldo Carvalho de Melo */ 1577d8bf9cSArnaldo Carvalho de Melo 162d8c4ce5SArnaldo Carvalho de Melo #include <linux/module.h> 17a7f5e7f1SArnaldo Carvalho de Melo #include <linux/random.h> 18f3f05f70SArnaldo Carvalho de Melo #include <linux/sched.h> 1977d8bf9cSArnaldo Carvalho de Melo #include <linux/slab.h> 20f3f05f70SArnaldo Carvalho de Melo #include <linux/wait.h> 21095dc8e0SEric Dumazet #include <linux/vmalloc.h> 2277d8bf9cSArnaldo Carvalho de Melo 23c125e80bSCraig Gallek #include <net/addrconf.h> 24463c84b9SArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 2577d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 266e5714eaSDavid S. Miller #include <net/secure_seq.h> 27a7f5e7f1SArnaldo Carvalho de Melo #include <net/ip.h> 28c125e80bSCraig Gallek #include <net/sock_reuseport.h> 2977d8bf9cSArnaldo Carvalho de Melo 306eada011SEric Dumazet static u32 inet_ehashfn(const struct net *net, const __be32 laddr, 3165cd8033SHannes Frederic Sowa const __u16 lport, const __be32 faddr, 3265cd8033SHannes Frederic Sowa const __be16 fport) 3365cd8033SHannes Frederic Sowa { 341bbdceefSHannes Frederic Sowa static u32 inet_ehash_secret __read_mostly; 351bbdceefSHannes Frederic Sowa 361bbdceefSHannes Frederic Sowa net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret)); 371bbdceefSHannes Frederic Sowa 3865cd8033SHannes Frederic Sowa return __inet_ehashfn(laddr, lport, faddr, fport, 3965cd8033SHannes Frederic Sowa inet_ehash_secret + net_hash_mix(net)); 4065cd8033SHannes Frederic Sowa } 4165cd8033SHannes Frederic Sowa 42d1e559d0SEric Dumazet /* This function handles inet_sock, but also timewait and request sockets 43d1e559d0SEric Dumazet * for IPv4/IPv6. 44d1e559d0SEric Dumazet */ 455b441f76SEric Dumazet u32 sk_ehashfn(const struct sock *sk) 4665cd8033SHannes Frederic Sowa { 47d1e559d0SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 48d1e559d0SEric Dumazet if (sk->sk_family == AF_INET6 && 49d1e559d0SEric Dumazet !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 50d1e559d0SEric Dumazet return inet6_ehashfn(sock_net(sk), 51d1e559d0SEric Dumazet &sk->sk_v6_rcv_saddr, sk->sk_num, 52d1e559d0SEric Dumazet &sk->sk_v6_daddr, sk->sk_dport); 53d1e559d0SEric Dumazet #endif 545b441f76SEric Dumazet return inet_ehashfn(sock_net(sk), 555b441f76SEric Dumazet sk->sk_rcv_saddr, sk->sk_num, 565b441f76SEric Dumazet sk->sk_daddr, sk->sk_dport); 5765cd8033SHannes Frederic Sowa } 5865cd8033SHannes Frederic Sowa 5977d8bf9cSArnaldo Carvalho de Melo /* 6077d8bf9cSArnaldo Carvalho de Melo * Allocate and initialize a new local port bind bucket. 6177d8bf9cSArnaldo Carvalho de Melo * The bindhash mutex for snum's hash chain must be held here. 6277d8bf9cSArnaldo Carvalho de Melo */ 63e18b890bSChristoph Lameter struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, 64941b1d22SPavel Emelyanov struct net *net, 6577d8bf9cSArnaldo Carvalho de Melo struct inet_bind_hashbucket *head, 6677d8bf9cSArnaldo Carvalho de Melo const unsigned short snum) 6777d8bf9cSArnaldo Carvalho de Melo { 6854e6ecb2SChristoph Lameter struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); 6977d8bf9cSArnaldo Carvalho de Melo 7000db4124SIan Morris if (tb) { 71efd7ef1cSEric W. Biederman write_pnet(&tb->ib_net, net); 7277d8bf9cSArnaldo Carvalho de Melo tb->port = snum; 7377d8bf9cSArnaldo Carvalho de Melo tb->fastreuse = 0; 74da5e3630STom Herbert tb->fastreuseport = 0; 75a9d8f911SEvgeniy Polyakov tb->num_owners = 0; 7677d8bf9cSArnaldo Carvalho de Melo INIT_HLIST_HEAD(&tb->owners); 7777d8bf9cSArnaldo Carvalho de Melo hlist_add_head(&tb->node, &head->chain); 7877d8bf9cSArnaldo Carvalho de Melo } 7977d8bf9cSArnaldo Carvalho de Melo return tb; 8077d8bf9cSArnaldo Carvalho de Melo } 8177d8bf9cSArnaldo Carvalho de Melo 8277d8bf9cSArnaldo Carvalho de Melo /* 8377d8bf9cSArnaldo Carvalho de Melo * Caller must hold hashbucket lock for this tb with local BH disabled 8477d8bf9cSArnaldo Carvalho de Melo */ 85e18b890bSChristoph Lameter void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb) 8677d8bf9cSArnaldo Carvalho de Melo { 8777d8bf9cSArnaldo Carvalho de Melo if (hlist_empty(&tb->owners)) { 8877d8bf9cSArnaldo Carvalho de Melo __hlist_del(&tb->node); 8977d8bf9cSArnaldo Carvalho de Melo kmem_cache_free(cachep, tb); 9077d8bf9cSArnaldo Carvalho de Melo } 9177d8bf9cSArnaldo Carvalho de Melo } 922d8c4ce5SArnaldo Carvalho de Melo 932d8c4ce5SArnaldo Carvalho de Melo void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, 942d8c4ce5SArnaldo Carvalho de Melo const unsigned short snum) 952d8c4ce5SArnaldo Carvalho de Melo { 96c720c7e8SEric Dumazet inet_sk(sk)->inet_num = snum; 972d8c4ce5SArnaldo Carvalho de Melo sk_add_bind_node(sk, &tb->owners); 98a9d8f911SEvgeniy Polyakov tb->num_owners++; 99463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = tb; 1002d8c4ce5SArnaldo Carvalho de Melo } 1012d8c4ce5SArnaldo Carvalho de Melo 1022d8c4ce5SArnaldo Carvalho de Melo /* 1032d8c4ce5SArnaldo Carvalho de Melo * Get rid of any references to a local port held by the given sock. 1042d8c4ce5SArnaldo Carvalho de Melo */ 105ab1e0a13SArnaldo Carvalho de Melo static void __inet_put_port(struct sock *sk) 1062d8c4ce5SArnaldo Carvalho de Melo { 10739d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 108c720c7e8SEric Dumazet const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num, 1097f635ab7SPavel Emelyanov hashinfo->bhash_size); 1102d8c4ce5SArnaldo Carvalho de Melo struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; 1112d8c4ce5SArnaldo Carvalho de Melo struct inet_bind_bucket *tb; 1122d8c4ce5SArnaldo Carvalho de Melo 1132d8c4ce5SArnaldo Carvalho de Melo spin_lock(&head->lock); 114463c84b9SArnaldo Carvalho de Melo tb = inet_csk(sk)->icsk_bind_hash; 1152d8c4ce5SArnaldo Carvalho de Melo __sk_del_bind_node(sk); 116a9d8f911SEvgeniy Polyakov tb->num_owners--; 117463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = NULL; 118c720c7e8SEric Dumazet inet_sk(sk)->inet_num = 0; 1192d8c4ce5SArnaldo Carvalho de Melo inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 1202d8c4ce5SArnaldo Carvalho de Melo spin_unlock(&head->lock); 1212d8c4ce5SArnaldo Carvalho de Melo } 1222d8c4ce5SArnaldo Carvalho de Melo 123ab1e0a13SArnaldo Carvalho de Melo void inet_put_port(struct sock *sk) 1242d8c4ce5SArnaldo Carvalho de Melo { 1252d8c4ce5SArnaldo Carvalho de Melo local_bh_disable(); 126ab1e0a13SArnaldo Carvalho de Melo __inet_put_port(sk); 1272d8c4ce5SArnaldo Carvalho de Melo local_bh_enable(); 1282d8c4ce5SArnaldo Carvalho de Melo } 1292d8c4ce5SArnaldo Carvalho de Melo EXPORT_SYMBOL(inet_put_port); 130f3f05f70SArnaldo Carvalho de Melo 1311ce31c9eSEric Dumazet int __inet_inherit_port(const struct sock *sk, struct sock *child) 13253083773SPavel Emelyanov { 13353083773SPavel Emelyanov struct inet_hashinfo *table = sk->sk_prot->h.hashinfo; 134093d2823SBalazs Scheidler unsigned short port = inet_sk(child)->inet_num; 135093d2823SBalazs Scheidler const int bhash = inet_bhashfn(sock_net(sk), port, 1367f635ab7SPavel Emelyanov table->bhash_size); 13753083773SPavel Emelyanov struct inet_bind_hashbucket *head = &table->bhash[bhash]; 13853083773SPavel Emelyanov struct inet_bind_bucket *tb; 13953083773SPavel Emelyanov 14053083773SPavel Emelyanov spin_lock(&head->lock); 14153083773SPavel Emelyanov tb = inet_csk(sk)->icsk_bind_hash; 142c2f34a65SEric Dumazet if (unlikely(!tb)) { 143c2f34a65SEric Dumazet spin_unlock(&head->lock); 144c2f34a65SEric Dumazet return -ENOENT; 145c2f34a65SEric Dumazet } 146093d2823SBalazs Scheidler if (tb->port != port) { 147093d2823SBalazs Scheidler /* NOTE: using tproxy and redirecting skbs to a proxy 148093d2823SBalazs Scheidler * on a different listener port breaks the assumption 149093d2823SBalazs Scheidler * that the listener socket's icsk_bind_hash is the same 150093d2823SBalazs Scheidler * as that of the child socket. We have to look up or 151093d2823SBalazs Scheidler * create a new bind bucket for the child here. */ 152b67bfe0dSSasha Levin inet_bind_bucket_for_each(tb, &head->chain) { 153093d2823SBalazs Scheidler if (net_eq(ib_net(tb), sock_net(sk)) && 154093d2823SBalazs Scheidler tb->port == port) 155093d2823SBalazs Scheidler break; 156093d2823SBalazs Scheidler } 157b67bfe0dSSasha Levin if (!tb) { 158093d2823SBalazs Scheidler tb = inet_bind_bucket_create(table->bind_bucket_cachep, 159093d2823SBalazs Scheidler sock_net(sk), head, port); 160093d2823SBalazs Scheidler if (!tb) { 161093d2823SBalazs Scheidler spin_unlock(&head->lock); 162093d2823SBalazs Scheidler return -ENOMEM; 163093d2823SBalazs Scheidler } 164093d2823SBalazs Scheidler } 165093d2823SBalazs Scheidler } 166b4ff3c90SNagendra Tomar inet_bind_hash(child, tb, port); 16753083773SPavel Emelyanov spin_unlock(&head->lock); 168093d2823SBalazs Scheidler 169093d2823SBalazs Scheidler return 0; 17053083773SPavel Emelyanov } 17153083773SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_inherit_port); 17253083773SPavel Emelyanov 173c25eb3bfSEric Dumazet static inline int compute_score(struct sock *sk, struct net *net, 174c25eb3bfSEric Dumazet const unsigned short hnum, const __be32 daddr, 175c25eb3bfSEric Dumazet const int dif) 176c25eb3bfSEric Dumazet { 177c25eb3bfSEric Dumazet int score = -1; 178c25eb3bfSEric Dumazet struct inet_sock *inet = inet_sk(sk); 179c25eb3bfSEric Dumazet 180c720c7e8SEric Dumazet if (net_eq(sock_net(sk), net) && inet->inet_num == hnum && 181c25eb3bfSEric Dumazet !ipv6_only_sock(sk)) { 182c720c7e8SEric Dumazet __be32 rcv_saddr = inet->inet_rcv_saddr; 183da5e3630STom Herbert score = sk->sk_family == PF_INET ? 2 : 1; 184c25eb3bfSEric Dumazet if (rcv_saddr) { 185c25eb3bfSEric Dumazet if (rcv_saddr != daddr) 186c25eb3bfSEric Dumazet return -1; 187da5e3630STom Herbert score += 4; 188c25eb3bfSEric Dumazet } 189c25eb3bfSEric Dumazet if (sk->sk_bound_dev_if) { 190c25eb3bfSEric Dumazet if (sk->sk_bound_dev_if != dif) 191c25eb3bfSEric Dumazet return -1; 192da5e3630STom Herbert score += 4; 193c25eb3bfSEric Dumazet } 19470da268bSEric Dumazet if (sk->sk_incoming_cpu == raw_smp_processor_id()) 19570da268bSEric Dumazet score++; 196c25eb3bfSEric Dumazet } 197c25eb3bfSEric Dumazet return score; 198c25eb3bfSEric Dumazet } 199c25eb3bfSEric Dumazet 200f3f05f70SArnaldo Carvalho de Melo /* 20133b62231SArnaldo Carvalho de Melo * Don't inline this cruft. Here are some nice properties to exploit here. The 20233b62231SArnaldo Carvalho de Melo * BSD API does not allow a listening sock to specify the remote port nor the 20333b62231SArnaldo Carvalho de Melo * remote address for the connection. So always assume those are both 20433b62231SArnaldo Carvalho de Melo * wildcarded during the search since they can never be otherwise. 20533b62231SArnaldo Carvalho de Melo */ 20633b62231SArnaldo Carvalho de Melo 20733b62231SArnaldo Carvalho de Melo 208c67499c0SPavel Emelyanov struct sock *__inet_lookup_listener(struct net *net, 209c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 210a583636aSCraig Gallek struct sk_buff *skb, int doff, 211da5e3630STom Herbert const __be32 saddr, __be16 sport, 212fb99c848SAl Viro const __be32 daddr, const unsigned short hnum, 21399a92ff5SHerbert Xu const int dif) 21499a92ff5SHerbert Xu { 215c25eb3bfSEric Dumazet struct sock *sk, *result; 216c25eb3bfSEric Dumazet struct hlist_nulls_node *node; 217c25eb3bfSEric Dumazet unsigned int hash = inet_lhashfn(net, hnum); 218c25eb3bfSEric Dumazet struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; 219da5e3630STom Herbert int score, hiscore, matches = 0, reuseport = 0; 220c125e80bSCraig Gallek bool select_ok = true; 221da5e3630STom Herbert u32 phash = 0; 22299a92ff5SHerbert Xu 223c25eb3bfSEric Dumazet rcu_read_lock(); 224c25eb3bfSEric Dumazet begin: 225c25eb3bfSEric Dumazet result = NULL; 226da5e3630STom Herbert hiscore = 0; 227c25eb3bfSEric Dumazet sk_nulls_for_each_rcu(sk, node, &ilb->head) { 228c25eb3bfSEric Dumazet score = compute_score(sk, net, hnum, daddr, dif); 229c25eb3bfSEric Dumazet if (score > hiscore) { 230c25eb3bfSEric Dumazet result = sk; 231c25eb3bfSEric Dumazet hiscore = score; 232da5e3630STom Herbert reuseport = sk->sk_reuseport; 233da5e3630STom Herbert if (reuseport) { 234da5e3630STom Herbert phash = inet_ehashfn(net, daddr, hnum, 235da5e3630STom Herbert saddr, sport); 236c125e80bSCraig Gallek if (select_ok) { 237c125e80bSCraig Gallek struct sock *sk2; 238c125e80bSCraig Gallek sk2 = reuseport_select_sock(sk, phash, 239c125e80bSCraig Gallek skb, doff); 240c125e80bSCraig Gallek if (sk2) { 241c125e80bSCraig Gallek result = sk2; 242c125e80bSCraig Gallek goto found; 243c125e80bSCraig Gallek } 244c125e80bSCraig Gallek } 245da5e3630STom Herbert matches = 1; 246da5e3630STom Herbert } 247da5e3630STom Herbert } else if (score == hiscore && reuseport) { 248da5e3630STom Herbert matches++; 2498fc54f68SDaniel Borkmann if (reciprocal_scale(phash, matches) == 0) 250da5e3630STom Herbert result = sk; 251da5e3630STom Herbert phash = next_pseudo_random32(phash); 25299a92ff5SHerbert Xu } 25399a92ff5SHerbert Xu } 254c25eb3bfSEric Dumazet /* 255c25eb3bfSEric Dumazet * if the nulls value we got at the end of this lookup is 256c25eb3bfSEric Dumazet * not the expected one, we must restart lookup. 257c25eb3bfSEric Dumazet * We probably met an item that was moved to another chain. 258c25eb3bfSEric Dumazet */ 259c25eb3bfSEric Dumazet if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE) 260c25eb3bfSEric Dumazet goto begin; 261c25eb3bfSEric Dumazet if (result) { 262c125e80bSCraig Gallek found: 263c25eb3bfSEric Dumazet if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt))) 264c25eb3bfSEric Dumazet result = NULL; 265c25eb3bfSEric Dumazet else if (unlikely(compute_score(result, net, hnum, daddr, 266c25eb3bfSEric Dumazet dif) < hiscore)) { 267c25eb3bfSEric Dumazet sock_put(result); 268c125e80bSCraig Gallek select_ok = false; 269c25eb3bfSEric Dumazet goto begin; 270c25eb3bfSEric Dumazet } 271c25eb3bfSEric Dumazet } 272c25eb3bfSEric Dumazet rcu_read_unlock(); 273c25eb3bfSEric Dumazet return result; 27499a92ff5SHerbert Xu } 2758f491069SHerbert Xu EXPORT_SYMBOL_GPL(__inet_lookup_listener); 276a7f5e7f1SArnaldo Carvalho de Melo 27705dbc7b5SEric Dumazet /* All sockets share common refcount, but have different destructors */ 27805dbc7b5SEric Dumazet void sock_gen_put(struct sock *sk) 27905dbc7b5SEric Dumazet { 28005dbc7b5SEric Dumazet if (!atomic_dec_and_test(&sk->sk_refcnt)) 28105dbc7b5SEric Dumazet return; 28205dbc7b5SEric Dumazet 28305dbc7b5SEric Dumazet if (sk->sk_state == TCP_TIME_WAIT) 28405dbc7b5SEric Dumazet inet_twsk_free(inet_twsk(sk)); 28541b822c5SEric Dumazet else if (sk->sk_state == TCP_NEW_SYN_RECV) 28641b822c5SEric Dumazet reqsk_free(inet_reqsk(sk)); 28705dbc7b5SEric Dumazet else 28805dbc7b5SEric Dumazet sk_free(sk); 28905dbc7b5SEric Dumazet } 29005dbc7b5SEric Dumazet EXPORT_SYMBOL_GPL(sock_gen_put); 29105dbc7b5SEric Dumazet 2922c13270bSEric Dumazet void sock_edemux(struct sk_buff *skb) 2932c13270bSEric Dumazet { 2942c13270bSEric Dumazet sock_gen_put(skb->sk); 2952c13270bSEric Dumazet } 2962c13270bSEric Dumazet EXPORT_SYMBOL(sock_edemux); 2972c13270bSEric Dumazet 298c67499c0SPavel Emelyanov struct sock *__inet_lookup_established(struct net *net, 299c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 30077a5ba55SPavel Emelyanov const __be32 saddr, const __be16 sport, 30177a5ba55SPavel Emelyanov const __be32 daddr, const u16 hnum, 30277a5ba55SPavel Emelyanov const int dif) 30377a5ba55SPavel Emelyanov { 304c7228317SJoe Perches INET_ADDR_COOKIE(acookie, saddr, daddr); 30577a5ba55SPavel Emelyanov const __portpair ports = INET_COMBINED_PORTS(sport, hnum); 30677a5ba55SPavel Emelyanov struct sock *sk; 3073ab5aee7SEric Dumazet const struct hlist_nulls_node *node; 30877a5ba55SPavel Emelyanov /* Optimize here for direct hit, only listening connections can 30977a5ba55SPavel Emelyanov * have wildcards anyways. 31077a5ba55SPavel Emelyanov */ 3119f26b3adSPavel Emelyanov unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport); 312f373b53bSEric Dumazet unsigned int slot = hash & hashinfo->ehash_mask; 3133ab5aee7SEric Dumazet struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; 31477a5ba55SPavel Emelyanov 3153ab5aee7SEric Dumazet rcu_read_lock(); 3163ab5aee7SEric Dumazet begin: 3173ab5aee7SEric Dumazet sk_nulls_for_each_rcu(sk, node, &head->chain) { 318ce43b03eSEric Dumazet if (sk->sk_hash != hash) 319ce43b03eSEric Dumazet continue; 320ce43b03eSEric Dumazet if (likely(INET_MATCH(sk, net, acookie, 321ce43b03eSEric Dumazet saddr, daddr, ports, dif))) { 3223ab5aee7SEric Dumazet if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) 32305dbc7b5SEric Dumazet goto out; 324ce43b03eSEric Dumazet if (unlikely(!INET_MATCH(sk, net, acookie, 3253ab5aee7SEric Dumazet saddr, daddr, ports, dif))) { 32605dbc7b5SEric Dumazet sock_gen_put(sk); 3273ab5aee7SEric Dumazet goto begin; 32877a5ba55SPavel Emelyanov } 32905dbc7b5SEric Dumazet goto found; 3303ab5aee7SEric Dumazet } 3313ab5aee7SEric Dumazet } 3323ab5aee7SEric Dumazet /* 3333ab5aee7SEric Dumazet * if the nulls value we got at the end of this lookup is 3343ab5aee7SEric Dumazet * not the expected one, we must restart lookup. 3353ab5aee7SEric Dumazet * We probably met an item that was moved to another chain. 3363ab5aee7SEric Dumazet */ 3373ab5aee7SEric Dumazet if (get_nulls_value(node) != slot) 3383ab5aee7SEric Dumazet goto begin; 33977a5ba55SPavel Emelyanov out: 34005dbc7b5SEric Dumazet sk = NULL; 34105dbc7b5SEric Dumazet found: 3423ab5aee7SEric Dumazet rcu_read_unlock(); 34377a5ba55SPavel Emelyanov return sk; 34477a5ba55SPavel Emelyanov } 34577a5ba55SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_lookup_established); 34677a5ba55SPavel Emelyanov 347a7f5e7f1SArnaldo Carvalho de Melo /* called with local bh disabled */ 348a7f5e7f1SArnaldo Carvalho de Melo static int __inet_check_established(struct inet_timewait_death_row *death_row, 349a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk, __u16 lport, 350a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock **twp) 351a7f5e7f1SArnaldo Carvalho de Melo { 352a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 353a7f5e7f1SArnaldo Carvalho de Melo struct inet_sock *inet = inet_sk(sk); 354c720c7e8SEric Dumazet __be32 daddr = inet->inet_rcv_saddr; 355c720c7e8SEric Dumazet __be32 saddr = inet->inet_daddr; 356a7f5e7f1SArnaldo Carvalho de Melo int dif = sk->sk_bound_dev_if; 357c7228317SJoe Perches INET_ADDR_COOKIE(acookie, saddr, daddr); 358c720c7e8SEric Dumazet const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); 3599f26b3adSPavel Emelyanov struct net *net = sock_net(sk); 360c720c7e8SEric Dumazet unsigned int hash = inet_ehashfn(net, daddr, lport, 361c720c7e8SEric Dumazet saddr, inet->inet_dport); 362a7f5e7f1SArnaldo Carvalho de Melo struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); 3639db66bdcSEric Dumazet spinlock_t *lock = inet_ehash_lockp(hinfo, hash); 364a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk2; 3653ab5aee7SEric Dumazet const struct hlist_nulls_node *node; 36605dbc7b5SEric Dumazet struct inet_timewait_sock *tw = NULL; 367a7f5e7f1SArnaldo Carvalho de Melo 3689db66bdcSEric Dumazet spin_lock(lock); 369a7f5e7f1SArnaldo Carvalho de Melo 3703ab5aee7SEric Dumazet sk_nulls_for_each(sk2, node, &head->chain) { 371ce43b03eSEric Dumazet if (sk2->sk_hash != hash) 372ce43b03eSEric Dumazet continue; 37305dbc7b5SEric Dumazet 374ce43b03eSEric Dumazet if (likely(INET_MATCH(sk2, net, acookie, 37505dbc7b5SEric Dumazet saddr, daddr, ports, dif))) { 37605dbc7b5SEric Dumazet if (sk2->sk_state == TCP_TIME_WAIT) { 37705dbc7b5SEric Dumazet tw = inet_twsk(sk2); 37805dbc7b5SEric Dumazet if (twsk_unique(sk, sk2, twp)) 37905dbc7b5SEric Dumazet break; 38005dbc7b5SEric Dumazet } 381a7f5e7f1SArnaldo Carvalho de Melo goto not_unique; 382a7f5e7f1SArnaldo Carvalho de Melo } 38305dbc7b5SEric Dumazet } 384a7f5e7f1SArnaldo Carvalho de Melo 385a7f5e7f1SArnaldo Carvalho de Melo /* Must record num and sport now. Otherwise we will see 38605dbc7b5SEric Dumazet * in hash table socket with a funny identity. 38705dbc7b5SEric Dumazet */ 388c720c7e8SEric Dumazet inet->inet_num = lport; 389c720c7e8SEric Dumazet inet->inet_sport = htons(lport); 390a7f5e7f1SArnaldo Carvalho de Melo sk->sk_hash = hash; 391547b792cSIlpo Järvinen WARN_ON(!sk_unhashed(sk)); 3923ab5aee7SEric Dumazet __sk_nulls_add_node_rcu(sk, &head->chain); 39313475a30SEric Dumazet if (tw) { 394fc01538fSEric Dumazet sk_nulls_del_node_init_rcu((struct sock *)tw); 39513475a30SEric Dumazet NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED); 39613475a30SEric Dumazet } 3979db66bdcSEric Dumazet spin_unlock(lock); 398c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 399a7f5e7f1SArnaldo Carvalho de Melo 400a7f5e7f1SArnaldo Carvalho de Melo if (twp) { 401a7f5e7f1SArnaldo Carvalho de Melo *twp = tw; 402a7f5e7f1SArnaldo Carvalho de Melo } else if (tw) { 403a7f5e7f1SArnaldo Carvalho de Melo /* Silly. Should hash-dance instead... */ 404dbe7faa4SEric Dumazet inet_twsk_deschedule_put(tw); 405a7f5e7f1SArnaldo Carvalho de Melo } 406a7f5e7f1SArnaldo Carvalho de Melo return 0; 407a7f5e7f1SArnaldo Carvalho de Melo 408a7f5e7f1SArnaldo Carvalho de Melo not_unique: 4099db66bdcSEric Dumazet spin_unlock(lock); 410a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 411a7f5e7f1SArnaldo Carvalho de Melo } 412a7f5e7f1SArnaldo Carvalho de Melo 413e2baad9eSEric Dumazet static u32 inet_sk_port_offset(const struct sock *sk) 414a7f5e7f1SArnaldo Carvalho de Melo { 415a7f5e7f1SArnaldo Carvalho de Melo const struct inet_sock *inet = inet_sk(sk); 416e2baad9eSEric Dumazet 417c720c7e8SEric Dumazet return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr, 418c720c7e8SEric Dumazet inet->inet_daddr, 419c720c7e8SEric Dumazet inet->inet_dport); 420a7f5e7f1SArnaldo Carvalho de Melo } 421a7f5e7f1SArnaldo Carvalho de Melo 422079096f1SEric Dumazet /* insert a socket into ehash, and eventually remove another one 423079096f1SEric Dumazet * (The another one can be a SYN_RECV or TIMEWAIT 424079096f1SEric Dumazet */ 4255e0724d0SEric Dumazet bool inet_ehash_insert(struct sock *sk, struct sock *osk) 426152da81dSPavel Emelyanov { 42739d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 4283ab5aee7SEric Dumazet struct hlist_nulls_head *list; 429152da81dSPavel Emelyanov struct inet_ehash_bucket *head; 4305b441f76SEric Dumazet spinlock_t *lock; 4315e0724d0SEric Dumazet bool ret = true; 432152da81dSPavel Emelyanov 433079096f1SEric Dumazet WARN_ON_ONCE(!sk_unhashed(sk)); 434152da81dSPavel Emelyanov 4355b441f76SEric Dumazet sk->sk_hash = sk_ehashfn(sk); 436152da81dSPavel Emelyanov head = inet_ehash_bucket(hashinfo, sk->sk_hash); 437152da81dSPavel Emelyanov list = &head->chain; 438152da81dSPavel Emelyanov lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 439152da81dSPavel Emelyanov 4409db66bdcSEric Dumazet spin_lock(lock); 441fc01538fSEric Dumazet if (osk) { 4425e0724d0SEric Dumazet WARN_ON_ONCE(sk->sk_hash != osk->sk_hash); 4435e0724d0SEric Dumazet ret = sk_nulls_del_node_init_rcu(osk); 4449327f705SEric Dumazet } 4455e0724d0SEric Dumazet if (ret) 4465e0724d0SEric Dumazet __sk_nulls_add_node_rcu(sk, list); 4479db66bdcSEric Dumazet spin_unlock(lock); 448079096f1SEric Dumazet return ret; 449079096f1SEric Dumazet } 450079096f1SEric Dumazet 4515e0724d0SEric Dumazet bool inet_ehash_nolisten(struct sock *sk, struct sock *osk) 452079096f1SEric Dumazet { 4535e0724d0SEric Dumazet bool ok = inet_ehash_insert(sk, osk); 4545e0724d0SEric Dumazet 4555e0724d0SEric Dumazet if (ok) { 456c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 4575e0724d0SEric Dumazet } else { 4585e0724d0SEric Dumazet percpu_counter_inc(sk->sk_prot->orphan_count); 4595e0724d0SEric Dumazet sk->sk_state = TCP_CLOSE; 4605e0724d0SEric Dumazet sock_set_flag(sk, SOCK_DEAD); 4615e0724d0SEric Dumazet inet_csk_destroy_sock(sk); 462152da81dSPavel Emelyanov } 4635e0724d0SEric Dumazet return ok; 4645e0724d0SEric Dumazet } 4655e0724d0SEric Dumazet EXPORT_SYMBOL_GPL(inet_ehash_nolisten); 466152da81dSPavel Emelyanov 467c125e80bSCraig Gallek static int inet_reuseport_add_sock(struct sock *sk, 468c125e80bSCraig Gallek struct inet_listen_hashbucket *ilb, 469c125e80bSCraig Gallek int (*saddr_same)(const struct sock *sk1, 470c125e80bSCraig Gallek const struct sock *sk2, 471c125e80bSCraig Gallek bool match_wildcard)) 472c125e80bSCraig Gallek { 473c125e80bSCraig Gallek struct sock *sk2; 474c125e80bSCraig Gallek struct hlist_nulls_node *node; 475c125e80bSCraig Gallek kuid_t uid = sock_i_uid(sk); 476c125e80bSCraig Gallek 477c125e80bSCraig Gallek sk_nulls_for_each_rcu(sk2, node, &ilb->head) { 478c125e80bSCraig Gallek if (sk2 != sk && 479c125e80bSCraig Gallek sk2->sk_family == sk->sk_family && 480c125e80bSCraig Gallek ipv6_only_sock(sk2) == ipv6_only_sock(sk) && 481c125e80bSCraig Gallek sk2->sk_bound_dev_if == sk->sk_bound_dev_if && 482c125e80bSCraig Gallek sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && 483c125e80bSCraig Gallek saddr_same(sk, sk2, false)) 484c125e80bSCraig Gallek return reuseport_add_sock(sk, sk2); 485c125e80bSCraig Gallek } 486c125e80bSCraig Gallek 487c125e80bSCraig Gallek /* Initial allocation may have already happened via setsockopt */ 488c125e80bSCraig Gallek if (!rcu_access_pointer(sk->sk_reuseport_cb)) 489c125e80bSCraig Gallek return reuseport_alloc(sk); 490c125e80bSCraig Gallek return 0; 491c125e80bSCraig Gallek } 492c125e80bSCraig Gallek 493c125e80bSCraig Gallek int __inet_hash(struct sock *sk, struct sock *osk, 494c125e80bSCraig Gallek int (*saddr_same)(const struct sock *sk1, 495c125e80bSCraig Gallek const struct sock *sk2, 496c125e80bSCraig Gallek bool match_wildcard)) 497152da81dSPavel Emelyanov { 49839d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 4995caea4eaSEric Dumazet struct inet_listen_hashbucket *ilb; 500c125e80bSCraig Gallek int err = 0; 501152da81dSPavel Emelyanov 5025e0724d0SEric Dumazet if (sk->sk_state != TCP_LISTEN) { 5035e0724d0SEric Dumazet inet_ehash_nolisten(sk, osk); 504c125e80bSCraig Gallek return 0; 5055e0724d0SEric Dumazet } 506547b792cSIlpo Järvinen WARN_ON(!sk_unhashed(sk)); 5075caea4eaSEric Dumazet ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)]; 508152da81dSPavel Emelyanov 5095caea4eaSEric Dumazet spin_lock(&ilb->lock); 510c125e80bSCraig Gallek if (sk->sk_reuseport) { 511c125e80bSCraig Gallek err = inet_reuseport_add_sock(sk, ilb, saddr_same); 512c125e80bSCraig Gallek if (err) 513c125e80bSCraig Gallek goto unlock; 514c125e80bSCraig Gallek } 515c25eb3bfSEric Dumazet __sk_nulls_add_node_rcu(sk, &ilb->head); 516c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 517c125e80bSCraig Gallek unlock: 5185caea4eaSEric Dumazet spin_unlock(&ilb->lock); 519c125e80bSCraig Gallek 520c125e80bSCraig Gallek return err; 521152da81dSPavel Emelyanov } 52277a6a471SEric Dumazet EXPORT_SYMBOL(__inet_hash); 523ab1e0a13SArnaldo Carvalho de Melo 524086c653fSCraig Gallek int inet_hash(struct sock *sk) 525ab1e0a13SArnaldo Carvalho de Melo { 526c125e80bSCraig Gallek int err = 0; 527c125e80bSCraig Gallek 528ab1e0a13SArnaldo Carvalho de Melo if (sk->sk_state != TCP_CLOSE) { 529ab1e0a13SArnaldo Carvalho de Melo local_bh_disable(); 530c125e80bSCraig Gallek err = __inet_hash(sk, NULL, ipv4_rcv_saddr_equal); 531ab1e0a13SArnaldo Carvalho de Melo local_bh_enable(); 532ab1e0a13SArnaldo Carvalho de Melo } 533086c653fSCraig Gallek 534c125e80bSCraig Gallek return err; 535ab1e0a13SArnaldo Carvalho de Melo } 536ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash); 537ab1e0a13SArnaldo Carvalho de Melo 538ab1e0a13SArnaldo Carvalho de Melo void inet_unhash(struct sock *sk) 539ab1e0a13SArnaldo Carvalho de Melo { 54039d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 541c25eb3bfSEric Dumazet spinlock_t *lock; 542c25eb3bfSEric Dumazet int done; 543ab1e0a13SArnaldo Carvalho de Melo 544ab1e0a13SArnaldo Carvalho de Melo if (sk_unhashed(sk)) 5455caea4eaSEric Dumazet return; 546ab1e0a13SArnaldo Carvalho de Melo 547c25eb3bfSEric Dumazet if (sk->sk_state == TCP_LISTEN) 548c25eb3bfSEric Dumazet lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock; 549c25eb3bfSEric Dumazet else 550c25eb3bfSEric Dumazet lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 5515caea4eaSEric Dumazet 5529db66bdcSEric Dumazet spin_lock_bh(lock); 553c125e80bSCraig Gallek if (rcu_access_pointer(sk->sk_reuseport_cb)) 554c125e80bSCraig Gallek reuseport_detach_sock(sk); 555c25eb3bfSEric Dumazet done = __sk_nulls_del_node_init_rcu(sk); 556c25eb3bfSEric Dumazet if (done) 557c25eb3bfSEric Dumazet sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 558920de804SEric Dumazet spin_unlock_bh(lock); 559ab1e0a13SArnaldo Carvalho de Melo } 560ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_unhash); 561152da81dSPavel Emelyanov 5625ee31fc1SPavel Emelyanov int __inet_hash_connect(struct inet_timewait_death_row *death_row, 5635d8c0aa9SPavel Emelyanov struct sock *sk, u32 port_offset, 5645ee31fc1SPavel Emelyanov int (*check_established)(struct inet_timewait_death_row *, 565b4d6444eSEric Dumazet struct sock *, __u16, struct inet_timewait_sock **)) 566a7f5e7f1SArnaldo Carvalho de Melo { 567a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 568a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock *tw = NULL; 569*1580ab63SEric Dumazet struct inet_bind_hashbucket *head; 570*1580ab63SEric Dumazet int port = inet_sk(sk)->inet_num; 571*1580ab63SEric Dumazet struct net *net = sock_net(sk); 572*1580ab63SEric Dumazet struct inet_bind_bucket *tb; 573*1580ab63SEric Dumazet u32 remaining, offset; 574*1580ab63SEric Dumazet int ret, i, low, high; 575*1580ab63SEric Dumazet static u32 hint; 576*1580ab63SEric Dumazet 577*1580ab63SEric Dumazet if (port) { 578*1580ab63SEric Dumazet head = &hinfo->bhash[inet_bhashfn(net, port, 579*1580ab63SEric Dumazet hinfo->bhash_size)]; 580*1580ab63SEric Dumazet tb = inet_csk(sk)->icsk_bind_hash; 581*1580ab63SEric Dumazet spin_lock_bh(&head->lock); 582*1580ab63SEric Dumazet if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) { 583*1580ab63SEric Dumazet inet_ehash_nolisten(sk, NULL); 584*1580ab63SEric Dumazet spin_unlock_bh(&head->lock); 585*1580ab63SEric Dumazet return 0; 586*1580ab63SEric Dumazet } 587*1580ab63SEric Dumazet spin_unlock(&head->lock); 588*1580ab63SEric Dumazet /* No definite answer... Walk to established hash table */ 589*1580ab63SEric Dumazet ret = check_established(death_row, sk, port, NULL); 590*1580ab63SEric Dumazet local_bh_enable(); 591*1580ab63SEric Dumazet return ret; 592*1580ab63SEric Dumazet } 593a7f5e7f1SArnaldo Carvalho de Melo 5940bbf87d8SEric W. Biederman inet_get_local_port_range(net, &low, &high); 595*1580ab63SEric Dumazet high++; /* [32768, 60999] -> [32768, 61000[ */ 596*1580ab63SEric Dumazet remaining = high - low; 597*1580ab63SEric Dumazet if (likely(remaining > 1)) 598*1580ab63SEric Dumazet remaining &= ~1U; 599227b60f5SStephen Hemminger 600*1580ab63SEric Dumazet offset = (hint + port_offset) % remaining; 601*1580ab63SEric Dumazet /* In first pass we try ports of @low parity. 602*1580ab63SEric Dumazet * inet_csk_get_port() does the opposite choice. 60307f4c900SEric Dumazet */ 604*1580ab63SEric Dumazet offset &= ~1U; 605*1580ab63SEric Dumazet other_parity_scan: 606*1580ab63SEric Dumazet port = low + offset; 607*1580ab63SEric Dumazet for (i = 0; i < remaining; i += 2, port += 2) { 608*1580ab63SEric Dumazet if (unlikely(port >= high)) 609*1580ab63SEric Dumazet port -= remaining; 610122ff243SWANG Cong if (inet_is_local_reserved_port(net, port)) 611e3826f1eSAmerigo Wang continue; 6127f635ab7SPavel Emelyanov head = &hinfo->bhash[inet_bhashfn(net, port, 6137f635ab7SPavel Emelyanov hinfo->bhash_size)]; 614*1580ab63SEric Dumazet spin_lock_bh(&head->lock); 615a7f5e7f1SArnaldo Carvalho de Melo 616*1580ab63SEric Dumazet /* Does not bother with rcv_saddr checks, because 617*1580ab63SEric Dumazet * the established check is already unique enough. 618a7f5e7f1SArnaldo Carvalho de Melo */ 619b67bfe0dSSasha Levin inet_bind_bucket_for_each(tb, &head->chain) { 620*1580ab63SEric Dumazet if (net_eq(ib_net(tb), net) && tb->port == port) { 621da5e3630STom Herbert if (tb->fastreuse >= 0 || 622da5e3630STom Herbert tb->fastreuseport >= 0) 623a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 624a9d8f911SEvgeniy Polyakov WARN_ON(hlist_empty(&tb->owners)); 6255ee31fc1SPavel Emelyanov if (!check_established(death_row, sk, 6265ee31fc1SPavel Emelyanov port, &tw)) 627a7f5e7f1SArnaldo Carvalho de Melo goto ok; 628a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 629a7f5e7f1SArnaldo Carvalho de Melo } 630a7f5e7f1SArnaldo Carvalho de Melo } 631a7f5e7f1SArnaldo Carvalho de Melo 632941b1d22SPavel Emelyanov tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, 633941b1d22SPavel Emelyanov net, head, port); 634a7f5e7f1SArnaldo Carvalho de Melo if (!tb) { 635*1580ab63SEric Dumazet spin_unlock_bh(&head->lock); 636*1580ab63SEric Dumazet return -ENOMEM; 637a7f5e7f1SArnaldo Carvalho de Melo } 638a7f5e7f1SArnaldo Carvalho de Melo tb->fastreuse = -1; 639da5e3630STom Herbert tb->fastreuseport = -1; 640a7f5e7f1SArnaldo Carvalho de Melo goto ok; 641a7f5e7f1SArnaldo Carvalho de Melo next_port: 642*1580ab63SEric Dumazet spin_unlock_bh(&head->lock); 643*1580ab63SEric Dumazet cond_resched(); 644a7f5e7f1SArnaldo Carvalho de Melo } 645*1580ab63SEric Dumazet 646*1580ab63SEric Dumazet offset++; 647*1580ab63SEric Dumazet if ((offset & 1) && remaining > 1) 648*1580ab63SEric Dumazet goto other_parity_scan; 649a7f5e7f1SArnaldo Carvalho de Melo 650a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 651a7f5e7f1SArnaldo Carvalho de Melo 652a7f5e7f1SArnaldo Carvalho de Melo ok: 653*1580ab63SEric Dumazet hint += i + 2; 654a7f5e7f1SArnaldo Carvalho de Melo 655a7f5e7f1SArnaldo Carvalho de Melo /* Head lock still held and bh's disabled */ 656a7f5e7f1SArnaldo Carvalho de Melo inet_bind_hash(sk, tb, port); 657a7f5e7f1SArnaldo Carvalho de Melo if (sk_unhashed(sk)) { 658c720c7e8SEric Dumazet inet_sk(sk)->inet_sport = htons(port); 6595e0724d0SEric Dumazet inet_ehash_nolisten(sk, (struct sock *)tw); 660a7f5e7f1SArnaldo Carvalho de Melo } 6613cdaedaeSEric Dumazet if (tw) 662fc01538fSEric Dumazet inet_twsk_bind_unhash(tw, hinfo); 663a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 664dbe7faa4SEric Dumazet if (tw) 665dbe7faa4SEric Dumazet inet_twsk_deschedule_put(tw); 666a7f5e7f1SArnaldo Carvalho de Melo local_bh_enable(); 667*1580ab63SEric Dumazet return 0; 668a7f5e7f1SArnaldo Carvalho de Melo } 6695ee31fc1SPavel Emelyanov 6705ee31fc1SPavel Emelyanov /* 6715ee31fc1SPavel Emelyanov * Bind a port for a connect operation and hash it. 6725ee31fc1SPavel Emelyanov */ 6735ee31fc1SPavel Emelyanov int inet_hash_connect(struct inet_timewait_death_row *death_row, 6745ee31fc1SPavel Emelyanov struct sock *sk) 6755ee31fc1SPavel Emelyanov { 676e2baad9eSEric Dumazet u32 port_offset = 0; 677e2baad9eSEric Dumazet 678e2baad9eSEric Dumazet if (!inet_sk(sk)->inet_num) 679e2baad9eSEric Dumazet port_offset = inet_sk_port_offset(sk); 680e2baad9eSEric Dumazet return __inet_hash_connect(death_row, sk, port_offset, 681b4d6444eSEric Dumazet __inet_check_established); 6825ee31fc1SPavel Emelyanov } 683a7f5e7f1SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash_connect); 6845caea4eaSEric Dumazet 6855caea4eaSEric Dumazet void inet_hashinfo_init(struct inet_hashinfo *h) 6865caea4eaSEric Dumazet { 6875caea4eaSEric Dumazet int i; 6885caea4eaSEric Dumazet 689c25eb3bfSEric Dumazet for (i = 0; i < INET_LHTABLE_SIZE; i++) { 6905caea4eaSEric Dumazet spin_lock_init(&h->listening_hash[i].lock); 691c25eb3bfSEric Dumazet INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head, 692c25eb3bfSEric Dumazet i + LISTENING_NULLS_BASE); 693c25eb3bfSEric Dumazet } 6945caea4eaSEric Dumazet } 6955caea4eaSEric Dumazet EXPORT_SYMBOL_GPL(inet_hashinfo_init); 696095dc8e0SEric Dumazet 697095dc8e0SEric Dumazet int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo) 698095dc8e0SEric Dumazet { 69989e478a2SEric Dumazet unsigned int locksz = sizeof(spinlock_t); 700095dc8e0SEric Dumazet unsigned int i, nblocks = 1; 701095dc8e0SEric Dumazet 70289e478a2SEric Dumazet if (locksz != 0) { 703095dc8e0SEric Dumazet /* allocate 2 cache lines or at least one spinlock per cpu */ 70489e478a2SEric Dumazet nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U); 705095dc8e0SEric Dumazet nblocks = roundup_pow_of_two(nblocks * num_possible_cpus()); 706095dc8e0SEric Dumazet 707095dc8e0SEric Dumazet /* no more locks than number of hash buckets */ 708095dc8e0SEric Dumazet nblocks = min(nblocks, hashinfo->ehash_mask + 1); 709095dc8e0SEric Dumazet 71089e478a2SEric Dumazet hashinfo->ehash_locks = kmalloc_array(nblocks, locksz, 711095dc8e0SEric Dumazet GFP_KERNEL | __GFP_NOWARN); 712095dc8e0SEric Dumazet if (!hashinfo->ehash_locks) 71389e478a2SEric Dumazet hashinfo->ehash_locks = vmalloc(nblocks * locksz); 714095dc8e0SEric Dumazet 715095dc8e0SEric Dumazet if (!hashinfo->ehash_locks) 716095dc8e0SEric Dumazet return -ENOMEM; 717095dc8e0SEric Dumazet 718095dc8e0SEric Dumazet for (i = 0; i < nblocks; i++) 719095dc8e0SEric Dumazet spin_lock_init(&hashinfo->ehash_locks[i]); 720095dc8e0SEric Dumazet } 721095dc8e0SEric Dumazet hashinfo->ehash_locks_mask = nblocks - 1; 722095dc8e0SEric Dumazet return 0; 723095dc8e0SEric Dumazet } 724095dc8e0SEric Dumazet EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc); 725