177d8bf9cSArnaldo Carvalho de Melo /* 277d8bf9cSArnaldo Carvalho de Melo * INET An implementation of the TCP/IP protocol suite for the LINUX 377d8bf9cSArnaldo Carvalho de Melo * operating system. INET is implemented using the BSD Socket 477d8bf9cSArnaldo Carvalho de Melo * interface as the means of communication with the user level. 577d8bf9cSArnaldo Carvalho de Melo * 677d8bf9cSArnaldo Carvalho de Melo * Generic INET transport hashtables 777d8bf9cSArnaldo Carvalho de Melo * 877d8bf9cSArnaldo Carvalho de Melo * Authors: Lotsa people, from code originally in tcp 977d8bf9cSArnaldo Carvalho de Melo * 1077d8bf9cSArnaldo Carvalho de Melo * This program is free software; you can redistribute it and/or 1177d8bf9cSArnaldo Carvalho de Melo * modify it under the terms of the GNU General Public License 1277d8bf9cSArnaldo Carvalho de Melo * as published by the Free Software Foundation; either version 1377d8bf9cSArnaldo Carvalho de Melo * 2 of the License, or (at your option) any later version. 1477d8bf9cSArnaldo Carvalho de Melo */ 1577d8bf9cSArnaldo Carvalho de Melo 162d8c4ce5SArnaldo Carvalho de Melo #include <linux/module.h> 17a7f5e7f1SArnaldo Carvalho de Melo #include <linux/random.h> 18f3f05f70SArnaldo Carvalho de Melo #include <linux/sched.h> 1977d8bf9cSArnaldo Carvalho de Melo #include <linux/slab.h> 20f3f05f70SArnaldo Carvalho de Melo #include <linux/wait.h> 2177d8bf9cSArnaldo Carvalho de Melo 22463c84b9SArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 2377d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 24a7f5e7f1SArnaldo Carvalho de Melo #include <net/ip.h> 2577d8bf9cSArnaldo Carvalho de Melo 2677d8bf9cSArnaldo Carvalho de Melo /* 2777d8bf9cSArnaldo Carvalho de Melo * Allocate and initialize a new local port bind bucket. 2877d8bf9cSArnaldo Carvalho de Melo * The bindhash mutex for snum's hash chain must be held here. 2977d8bf9cSArnaldo Carvalho de Melo */ 30e18b890bSChristoph Lameter struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, 31941b1d22SPavel Emelyanov struct net *net, 3277d8bf9cSArnaldo Carvalho de Melo struct inet_bind_hashbucket *head, 3377d8bf9cSArnaldo Carvalho de Melo const unsigned short snum) 3477d8bf9cSArnaldo Carvalho de Melo { 3554e6ecb2SChristoph Lameter struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); 3677d8bf9cSArnaldo Carvalho de Melo 3777d8bf9cSArnaldo Carvalho de Melo if (tb != NULL) { 388c5da49aSDenis V. Lunev tb->ib_net = hold_net(net); 3977d8bf9cSArnaldo Carvalho de Melo tb->port = snum; 4077d8bf9cSArnaldo Carvalho de Melo tb->fastreuse = 0; 4177d8bf9cSArnaldo Carvalho de Melo INIT_HLIST_HEAD(&tb->owners); 4277d8bf9cSArnaldo Carvalho de Melo hlist_add_head(&tb->node, &head->chain); 4377d8bf9cSArnaldo Carvalho de Melo } 4477d8bf9cSArnaldo Carvalho de Melo return tb; 4577d8bf9cSArnaldo Carvalho de Melo } 4677d8bf9cSArnaldo Carvalho de Melo 4777d8bf9cSArnaldo Carvalho de Melo /* 4877d8bf9cSArnaldo Carvalho de Melo * Caller must hold hashbucket lock for this tb with local BH disabled 4977d8bf9cSArnaldo Carvalho de Melo */ 50e18b890bSChristoph Lameter void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb) 5177d8bf9cSArnaldo Carvalho de Melo { 5277d8bf9cSArnaldo Carvalho de Melo if (hlist_empty(&tb->owners)) { 5377d8bf9cSArnaldo Carvalho de Melo __hlist_del(&tb->node); 548c5da49aSDenis V. Lunev release_net(tb->ib_net); 5577d8bf9cSArnaldo Carvalho de Melo kmem_cache_free(cachep, tb); 5677d8bf9cSArnaldo Carvalho de Melo } 5777d8bf9cSArnaldo Carvalho de Melo } 582d8c4ce5SArnaldo Carvalho de Melo 592d8c4ce5SArnaldo Carvalho de Melo void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, 602d8c4ce5SArnaldo Carvalho de Melo const unsigned short snum) 612d8c4ce5SArnaldo Carvalho de Melo { 62463c84b9SArnaldo Carvalho de Melo inet_sk(sk)->num = snum; 632d8c4ce5SArnaldo Carvalho de Melo sk_add_bind_node(sk, &tb->owners); 64463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = tb; 652d8c4ce5SArnaldo Carvalho de Melo } 662d8c4ce5SArnaldo Carvalho de Melo 672d8c4ce5SArnaldo Carvalho de Melo /* 682d8c4ce5SArnaldo Carvalho de Melo * Get rid of any references to a local port held by the given sock. 692d8c4ce5SArnaldo Carvalho de Melo */ 70ab1e0a13SArnaldo Carvalho de Melo static void __inet_put_port(struct sock *sk) 712d8c4ce5SArnaldo Carvalho de Melo { 7239d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 73*7f635ab7SPavel Emelyanov const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->num, 74*7f635ab7SPavel Emelyanov hashinfo->bhash_size); 752d8c4ce5SArnaldo Carvalho de Melo struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; 762d8c4ce5SArnaldo Carvalho de Melo struct inet_bind_bucket *tb; 772d8c4ce5SArnaldo Carvalho de Melo 782d8c4ce5SArnaldo Carvalho de Melo spin_lock(&head->lock); 79463c84b9SArnaldo Carvalho de Melo tb = inet_csk(sk)->icsk_bind_hash; 802d8c4ce5SArnaldo Carvalho de Melo __sk_del_bind_node(sk); 81463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = NULL; 82463c84b9SArnaldo Carvalho de Melo inet_sk(sk)->num = 0; 832d8c4ce5SArnaldo Carvalho de Melo inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 842d8c4ce5SArnaldo Carvalho de Melo spin_unlock(&head->lock); 852d8c4ce5SArnaldo Carvalho de Melo } 862d8c4ce5SArnaldo Carvalho de Melo 87ab1e0a13SArnaldo Carvalho de Melo void inet_put_port(struct sock *sk) 882d8c4ce5SArnaldo Carvalho de Melo { 892d8c4ce5SArnaldo Carvalho de Melo local_bh_disable(); 90ab1e0a13SArnaldo Carvalho de Melo __inet_put_port(sk); 912d8c4ce5SArnaldo Carvalho de Melo local_bh_enable(); 922d8c4ce5SArnaldo Carvalho de Melo } 932d8c4ce5SArnaldo Carvalho de Melo 942d8c4ce5SArnaldo Carvalho de Melo EXPORT_SYMBOL(inet_put_port); 95f3f05f70SArnaldo Carvalho de Melo 9653083773SPavel Emelyanov void __inet_inherit_port(struct sock *sk, struct sock *child) 9753083773SPavel Emelyanov { 9853083773SPavel Emelyanov struct inet_hashinfo *table = sk->sk_prot->h.hashinfo; 99*7f635ab7SPavel Emelyanov const int bhash = inet_bhashfn(sock_net(sk), inet_sk(child)->num, 100*7f635ab7SPavel Emelyanov table->bhash_size); 10153083773SPavel Emelyanov struct inet_bind_hashbucket *head = &table->bhash[bhash]; 10253083773SPavel Emelyanov struct inet_bind_bucket *tb; 10353083773SPavel Emelyanov 10453083773SPavel Emelyanov spin_lock(&head->lock); 10553083773SPavel Emelyanov tb = inet_csk(sk)->icsk_bind_hash; 10653083773SPavel Emelyanov sk_add_bind_node(child, &tb->owners); 10753083773SPavel Emelyanov inet_csk(child)->icsk_bind_hash = tb; 10853083773SPavel Emelyanov spin_unlock(&head->lock); 10953083773SPavel Emelyanov } 11053083773SPavel Emelyanov 11153083773SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_inherit_port); 11253083773SPavel Emelyanov 113f3f05f70SArnaldo Carvalho de Melo /* 114f3f05f70SArnaldo Carvalho de Melo * This lock without WQ_FLAG_EXCLUSIVE is good on UP and it can be very bad on SMP. 115f3f05f70SArnaldo Carvalho de Melo * Look, when several writers sleep and reader wakes them up, all but one 116f3f05f70SArnaldo Carvalho de Melo * immediately hit write lock and grab all the cpus. Exclusive sleep solves 117f3f05f70SArnaldo Carvalho de Melo * this, _but_ remember, it adds useless work on UP machines (wake up each 118f3f05f70SArnaldo Carvalho de Melo * exclusive lock release). It should be ifdefed really. 119f3f05f70SArnaldo Carvalho de Melo */ 120f3f05f70SArnaldo Carvalho de Melo void inet_listen_wlock(struct inet_hashinfo *hashinfo) 1219a429c49SEric Dumazet __acquires(hashinfo->lhash_lock) 122f3f05f70SArnaldo Carvalho de Melo { 123f3f05f70SArnaldo Carvalho de Melo write_lock(&hashinfo->lhash_lock); 124f3f05f70SArnaldo Carvalho de Melo 125f3f05f70SArnaldo Carvalho de Melo if (atomic_read(&hashinfo->lhash_users)) { 126f3f05f70SArnaldo Carvalho de Melo DEFINE_WAIT(wait); 127f3f05f70SArnaldo Carvalho de Melo 128f3f05f70SArnaldo Carvalho de Melo for (;;) { 129f3f05f70SArnaldo Carvalho de Melo prepare_to_wait_exclusive(&hashinfo->lhash_wait, 130f3f05f70SArnaldo Carvalho de Melo &wait, TASK_UNINTERRUPTIBLE); 131f3f05f70SArnaldo Carvalho de Melo if (!atomic_read(&hashinfo->lhash_users)) 132f3f05f70SArnaldo Carvalho de Melo break; 133f3f05f70SArnaldo Carvalho de Melo write_unlock_bh(&hashinfo->lhash_lock); 134f3f05f70SArnaldo Carvalho de Melo schedule(); 135f3f05f70SArnaldo Carvalho de Melo write_lock_bh(&hashinfo->lhash_lock); 136f3f05f70SArnaldo Carvalho de Melo } 137f3f05f70SArnaldo Carvalho de Melo 138f3f05f70SArnaldo Carvalho de Melo finish_wait(&hashinfo->lhash_wait, &wait); 139f3f05f70SArnaldo Carvalho de Melo } 140f3f05f70SArnaldo Carvalho de Melo } 141f3f05f70SArnaldo Carvalho de Melo 14233b62231SArnaldo Carvalho de Melo /* 14333b62231SArnaldo Carvalho de Melo * Don't inline this cruft. Here are some nice properties to exploit here. The 14433b62231SArnaldo Carvalho de Melo * BSD API does not allow a listening sock to specify the remote port nor the 14533b62231SArnaldo Carvalho de Melo * remote address for the connection. So always assume those are both 14633b62231SArnaldo Carvalho de Melo * wildcarded during the search since they can never be otherwise. 14733b62231SArnaldo Carvalho de Melo */ 148c67499c0SPavel Emelyanov static struct sock *inet_lookup_listener_slow(struct net *net, 149c67499c0SPavel Emelyanov const struct hlist_head *head, 150fb99c848SAl Viro const __be32 daddr, 15199a92ff5SHerbert Xu const unsigned short hnum, 15299a92ff5SHerbert Xu const int dif) 15333b62231SArnaldo Carvalho de Melo { 15433b62231SArnaldo Carvalho de Melo struct sock *result = NULL, *sk; 15533b62231SArnaldo Carvalho de Melo const struct hlist_node *node; 15633b62231SArnaldo Carvalho de Melo int hiscore = -1; 15733b62231SArnaldo Carvalho de Melo 15833b62231SArnaldo Carvalho de Melo sk_for_each(sk, node, head) { 15933b62231SArnaldo Carvalho de Melo const struct inet_sock *inet = inet_sk(sk); 16033b62231SArnaldo Carvalho de Melo 161878628fbSYOSHIFUJI Hideaki if (net_eq(sock_net(sk), net) && inet->num == hnum && 162c67499c0SPavel Emelyanov !ipv6_only_sock(sk)) { 163fb99c848SAl Viro const __be32 rcv_saddr = inet->rcv_saddr; 16433b62231SArnaldo Carvalho de Melo int score = sk->sk_family == PF_INET ? 1 : 0; 16533b62231SArnaldo Carvalho de Melo 16633b62231SArnaldo Carvalho de Melo if (rcv_saddr) { 16733b62231SArnaldo Carvalho de Melo if (rcv_saddr != daddr) 16833b62231SArnaldo Carvalho de Melo continue; 16933b62231SArnaldo Carvalho de Melo score += 2; 17033b62231SArnaldo Carvalho de Melo } 17133b62231SArnaldo Carvalho de Melo if (sk->sk_bound_dev_if) { 17233b62231SArnaldo Carvalho de Melo if (sk->sk_bound_dev_if != dif) 17333b62231SArnaldo Carvalho de Melo continue; 17433b62231SArnaldo Carvalho de Melo score += 2; 17533b62231SArnaldo Carvalho de Melo } 17633b62231SArnaldo Carvalho de Melo if (score == 5) 17733b62231SArnaldo Carvalho de Melo return sk; 17833b62231SArnaldo Carvalho de Melo if (score > hiscore) { 17933b62231SArnaldo Carvalho de Melo hiscore = score; 18033b62231SArnaldo Carvalho de Melo result = sk; 18133b62231SArnaldo Carvalho de Melo } 18233b62231SArnaldo Carvalho de Melo } 18333b62231SArnaldo Carvalho de Melo } 18433b62231SArnaldo Carvalho de Melo return result; 18533b62231SArnaldo Carvalho de Melo } 186e48c414eSArnaldo Carvalho de Melo 18799a92ff5SHerbert Xu /* Optimize the common listener case. */ 188c67499c0SPavel Emelyanov struct sock *__inet_lookup_listener(struct net *net, 189c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 190fb99c848SAl Viro const __be32 daddr, const unsigned short hnum, 19199a92ff5SHerbert Xu const int dif) 19299a92ff5SHerbert Xu { 19399a92ff5SHerbert Xu struct sock *sk = NULL; 19499a92ff5SHerbert Xu const struct hlist_head *head; 19599a92ff5SHerbert Xu 19699a92ff5SHerbert Xu read_lock(&hashinfo->lhash_lock); 19799a92ff5SHerbert Xu head = &hashinfo->listening_hash[inet_lhashfn(hnum)]; 19899a92ff5SHerbert Xu if (!hlist_empty(head)) { 19999a92ff5SHerbert Xu const struct inet_sock *inet = inet_sk((sk = __sk_head(head))); 20099a92ff5SHerbert Xu 20199a92ff5SHerbert Xu if (inet->num == hnum && !sk->sk_node.next && 20299a92ff5SHerbert Xu (!inet->rcv_saddr || inet->rcv_saddr == daddr) && 20399a92ff5SHerbert Xu (sk->sk_family == PF_INET || !ipv6_only_sock(sk)) && 204878628fbSYOSHIFUJI Hideaki !sk->sk_bound_dev_if && net_eq(sock_net(sk), net)) 20599a92ff5SHerbert Xu goto sherry_cache; 206c67499c0SPavel Emelyanov sk = inet_lookup_listener_slow(net, head, daddr, hnum, dif); 20799a92ff5SHerbert Xu } 20899a92ff5SHerbert Xu if (sk) { 20999a92ff5SHerbert Xu sherry_cache: 21099a92ff5SHerbert Xu sock_hold(sk); 21199a92ff5SHerbert Xu } 21299a92ff5SHerbert Xu read_unlock(&hashinfo->lhash_lock); 21399a92ff5SHerbert Xu return sk; 21499a92ff5SHerbert Xu } 2158f491069SHerbert Xu EXPORT_SYMBOL_GPL(__inet_lookup_listener); 216a7f5e7f1SArnaldo Carvalho de Melo 217c67499c0SPavel Emelyanov struct sock * __inet_lookup_established(struct net *net, 218c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 21977a5ba55SPavel Emelyanov const __be32 saddr, const __be16 sport, 22077a5ba55SPavel Emelyanov const __be32 daddr, const u16 hnum, 22177a5ba55SPavel Emelyanov const int dif) 22277a5ba55SPavel Emelyanov { 22377a5ba55SPavel Emelyanov INET_ADDR_COOKIE(acookie, saddr, daddr) 22477a5ba55SPavel Emelyanov const __portpair ports = INET_COMBINED_PORTS(sport, hnum); 22577a5ba55SPavel Emelyanov struct sock *sk; 22677a5ba55SPavel Emelyanov const struct hlist_node *node; 22777a5ba55SPavel Emelyanov /* Optimize here for direct hit, only listening connections can 22877a5ba55SPavel Emelyanov * have wildcards anyways. 22977a5ba55SPavel Emelyanov */ 23077a5ba55SPavel Emelyanov unsigned int hash = inet_ehashfn(daddr, hnum, saddr, sport); 23177a5ba55SPavel Emelyanov struct inet_ehash_bucket *head = inet_ehash_bucket(hashinfo, hash); 23277a5ba55SPavel Emelyanov rwlock_t *lock = inet_ehash_lockp(hashinfo, hash); 23377a5ba55SPavel Emelyanov 23477a5ba55SPavel Emelyanov prefetch(head->chain.first); 23577a5ba55SPavel Emelyanov read_lock(lock); 23677a5ba55SPavel Emelyanov sk_for_each(sk, node, &head->chain) { 237c67499c0SPavel Emelyanov if (INET_MATCH(sk, net, hash, acookie, 238c67499c0SPavel Emelyanov saddr, daddr, ports, dif)) 23977a5ba55SPavel Emelyanov goto hit; /* You sunk my battleship! */ 24077a5ba55SPavel Emelyanov } 24177a5ba55SPavel Emelyanov 24277a5ba55SPavel Emelyanov /* Must check for a TIME_WAIT'er before going to listener hash. */ 24377a5ba55SPavel Emelyanov sk_for_each(sk, node, &head->twchain) { 244c67499c0SPavel Emelyanov if (INET_TW_MATCH(sk, net, hash, acookie, 245c67499c0SPavel Emelyanov saddr, daddr, ports, dif)) 24677a5ba55SPavel Emelyanov goto hit; 24777a5ba55SPavel Emelyanov } 24877a5ba55SPavel Emelyanov sk = NULL; 24977a5ba55SPavel Emelyanov out: 25077a5ba55SPavel Emelyanov read_unlock(lock); 25177a5ba55SPavel Emelyanov return sk; 25277a5ba55SPavel Emelyanov hit: 25377a5ba55SPavel Emelyanov sock_hold(sk); 25477a5ba55SPavel Emelyanov goto out; 25577a5ba55SPavel Emelyanov } 25677a5ba55SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_lookup_established); 25777a5ba55SPavel Emelyanov 258a7f5e7f1SArnaldo Carvalho de Melo /* called with local bh disabled */ 259a7f5e7f1SArnaldo Carvalho de Melo static int __inet_check_established(struct inet_timewait_death_row *death_row, 260a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk, __u16 lport, 261a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock **twp) 262a7f5e7f1SArnaldo Carvalho de Melo { 263a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 264a7f5e7f1SArnaldo Carvalho de Melo struct inet_sock *inet = inet_sk(sk); 265fb99c848SAl Viro __be32 daddr = inet->rcv_saddr; 266fb99c848SAl Viro __be32 saddr = inet->daddr; 267a7f5e7f1SArnaldo Carvalho de Melo int dif = sk->sk_bound_dev_if; 268a7f5e7f1SArnaldo Carvalho de Melo INET_ADDR_COOKIE(acookie, saddr, daddr) 2694f765d84SAl Viro const __portpair ports = INET_COMBINED_PORTS(inet->dport, lport); 270a7f5e7f1SArnaldo Carvalho de Melo unsigned int hash = inet_ehashfn(daddr, lport, saddr, inet->dport); 271a7f5e7f1SArnaldo Carvalho de Melo struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); 272230140cfSEric Dumazet rwlock_t *lock = inet_ehash_lockp(hinfo, hash); 273a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk2; 274a7f5e7f1SArnaldo Carvalho de Melo const struct hlist_node *node; 275a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock *tw; 2763b1e0a65SYOSHIFUJI Hideaki struct net *net = sock_net(sk); 277a7f5e7f1SArnaldo Carvalho de Melo 278a7f5e7f1SArnaldo Carvalho de Melo prefetch(head->chain.first); 279230140cfSEric Dumazet write_lock(lock); 280a7f5e7f1SArnaldo Carvalho de Melo 281a7f5e7f1SArnaldo Carvalho de Melo /* Check TIME-WAIT sockets first. */ 282dbca9b27SEric Dumazet sk_for_each(sk2, node, &head->twchain) { 283a7f5e7f1SArnaldo Carvalho de Melo tw = inet_twsk(sk2); 284a7f5e7f1SArnaldo Carvalho de Melo 285c67499c0SPavel Emelyanov if (INET_TW_MATCH(sk2, net, hash, acookie, 286c67499c0SPavel Emelyanov saddr, daddr, ports, dif)) { 287a7f5e7f1SArnaldo Carvalho de Melo if (twsk_unique(sk, sk2, twp)) 288a7f5e7f1SArnaldo Carvalho de Melo goto unique; 289a7f5e7f1SArnaldo Carvalho de Melo else 290a7f5e7f1SArnaldo Carvalho de Melo goto not_unique; 291a7f5e7f1SArnaldo Carvalho de Melo } 292a7f5e7f1SArnaldo Carvalho de Melo } 293a7f5e7f1SArnaldo Carvalho de Melo tw = NULL; 294a7f5e7f1SArnaldo Carvalho de Melo 295a7f5e7f1SArnaldo Carvalho de Melo /* And established part... */ 296a7f5e7f1SArnaldo Carvalho de Melo sk_for_each(sk2, node, &head->chain) { 297c67499c0SPavel Emelyanov if (INET_MATCH(sk2, net, hash, acookie, 298c67499c0SPavel Emelyanov saddr, daddr, ports, dif)) 299a7f5e7f1SArnaldo Carvalho de Melo goto not_unique; 300a7f5e7f1SArnaldo Carvalho de Melo } 301a7f5e7f1SArnaldo Carvalho de Melo 302a7f5e7f1SArnaldo Carvalho de Melo unique: 303a7f5e7f1SArnaldo Carvalho de Melo /* Must record num and sport now. Otherwise we will see 304a7f5e7f1SArnaldo Carvalho de Melo * in hash table socket with a funny identity. */ 305a7f5e7f1SArnaldo Carvalho de Melo inet->num = lport; 306a7f5e7f1SArnaldo Carvalho de Melo inet->sport = htons(lport); 307a7f5e7f1SArnaldo Carvalho de Melo sk->sk_hash = hash; 308a7f5e7f1SArnaldo Carvalho de Melo BUG_TRAP(sk_unhashed(sk)); 309a7f5e7f1SArnaldo Carvalho de Melo __sk_add_node(sk, &head->chain); 310c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 311230140cfSEric Dumazet write_unlock(lock); 312a7f5e7f1SArnaldo Carvalho de Melo 313a7f5e7f1SArnaldo Carvalho de Melo if (twp) { 314a7f5e7f1SArnaldo Carvalho de Melo *twp = tw; 315a7f5e7f1SArnaldo Carvalho de Melo NET_INC_STATS_BH(LINUX_MIB_TIMEWAITRECYCLED); 316a7f5e7f1SArnaldo Carvalho de Melo } else if (tw) { 317a7f5e7f1SArnaldo Carvalho de Melo /* Silly. Should hash-dance instead... */ 318a7f5e7f1SArnaldo Carvalho de Melo inet_twsk_deschedule(tw, death_row); 319a7f5e7f1SArnaldo Carvalho de Melo NET_INC_STATS_BH(LINUX_MIB_TIMEWAITRECYCLED); 320a7f5e7f1SArnaldo Carvalho de Melo 321a7f5e7f1SArnaldo Carvalho de Melo inet_twsk_put(tw); 322a7f5e7f1SArnaldo Carvalho de Melo } 323a7f5e7f1SArnaldo Carvalho de Melo 324a7f5e7f1SArnaldo Carvalho de Melo return 0; 325a7f5e7f1SArnaldo Carvalho de Melo 326a7f5e7f1SArnaldo Carvalho de Melo not_unique: 327230140cfSEric Dumazet write_unlock(lock); 328a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 329a7f5e7f1SArnaldo Carvalho de Melo } 330a7f5e7f1SArnaldo Carvalho de Melo 331a7f5e7f1SArnaldo Carvalho de Melo static inline u32 inet_sk_port_offset(const struct sock *sk) 332a7f5e7f1SArnaldo Carvalho de Melo { 333a7f5e7f1SArnaldo Carvalho de Melo const struct inet_sock *inet = inet_sk(sk); 334a7f5e7f1SArnaldo Carvalho de Melo return secure_ipv4_port_ephemeral(inet->rcv_saddr, inet->daddr, 335a7f5e7f1SArnaldo Carvalho de Melo inet->dport); 336a7f5e7f1SArnaldo Carvalho de Melo } 337a7f5e7f1SArnaldo Carvalho de Melo 338ab1e0a13SArnaldo Carvalho de Melo void __inet_hash_nolisten(struct sock *sk) 339152da81dSPavel Emelyanov { 34039d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 341152da81dSPavel Emelyanov struct hlist_head *list; 342152da81dSPavel Emelyanov rwlock_t *lock; 343152da81dSPavel Emelyanov struct inet_ehash_bucket *head; 344152da81dSPavel Emelyanov 345152da81dSPavel Emelyanov BUG_TRAP(sk_unhashed(sk)); 346152da81dSPavel Emelyanov 347152da81dSPavel Emelyanov sk->sk_hash = inet_sk_ehashfn(sk); 348152da81dSPavel Emelyanov head = inet_ehash_bucket(hashinfo, sk->sk_hash); 349152da81dSPavel Emelyanov list = &head->chain; 350152da81dSPavel Emelyanov lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 351152da81dSPavel Emelyanov 352152da81dSPavel Emelyanov write_lock(lock); 353152da81dSPavel Emelyanov __sk_add_node(sk, list); 354c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 355152da81dSPavel Emelyanov write_unlock(lock); 356152da81dSPavel Emelyanov } 357152da81dSPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_hash_nolisten); 358152da81dSPavel Emelyanov 359ab1e0a13SArnaldo Carvalho de Melo static void __inet_hash(struct sock *sk) 360152da81dSPavel Emelyanov { 36139d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 362152da81dSPavel Emelyanov struct hlist_head *list; 363152da81dSPavel Emelyanov rwlock_t *lock; 364152da81dSPavel Emelyanov 365152da81dSPavel Emelyanov if (sk->sk_state != TCP_LISTEN) { 366ab1e0a13SArnaldo Carvalho de Melo __inet_hash_nolisten(sk); 367152da81dSPavel Emelyanov return; 368152da81dSPavel Emelyanov } 369152da81dSPavel Emelyanov 370152da81dSPavel Emelyanov BUG_TRAP(sk_unhashed(sk)); 371152da81dSPavel Emelyanov list = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)]; 372152da81dSPavel Emelyanov lock = &hashinfo->lhash_lock; 373152da81dSPavel Emelyanov 374152da81dSPavel Emelyanov inet_listen_wlock(hashinfo); 375152da81dSPavel Emelyanov __sk_add_node(sk, list); 376c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 377152da81dSPavel Emelyanov write_unlock(lock); 378152da81dSPavel Emelyanov wake_up(&hashinfo->lhash_wait); 379152da81dSPavel Emelyanov } 380ab1e0a13SArnaldo Carvalho de Melo 381ab1e0a13SArnaldo Carvalho de Melo void inet_hash(struct sock *sk) 382ab1e0a13SArnaldo Carvalho de Melo { 383ab1e0a13SArnaldo Carvalho de Melo if (sk->sk_state != TCP_CLOSE) { 384ab1e0a13SArnaldo Carvalho de Melo local_bh_disable(); 385ab1e0a13SArnaldo Carvalho de Melo __inet_hash(sk); 386ab1e0a13SArnaldo Carvalho de Melo local_bh_enable(); 387ab1e0a13SArnaldo Carvalho de Melo } 388ab1e0a13SArnaldo Carvalho de Melo } 389ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash); 390ab1e0a13SArnaldo Carvalho de Melo 391ab1e0a13SArnaldo Carvalho de Melo void inet_unhash(struct sock *sk) 392ab1e0a13SArnaldo Carvalho de Melo { 393ab1e0a13SArnaldo Carvalho de Melo rwlock_t *lock; 39439d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 395ab1e0a13SArnaldo Carvalho de Melo 396ab1e0a13SArnaldo Carvalho de Melo if (sk_unhashed(sk)) 397ab1e0a13SArnaldo Carvalho de Melo goto out; 398ab1e0a13SArnaldo Carvalho de Melo 399ab1e0a13SArnaldo Carvalho de Melo if (sk->sk_state == TCP_LISTEN) { 400ab1e0a13SArnaldo Carvalho de Melo local_bh_disable(); 401ab1e0a13SArnaldo Carvalho de Melo inet_listen_wlock(hashinfo); 402ab1e0a13SArnaldo Carvalho de Melo lock = &hashinfo->lhash_lock; 403ab1e0a13SArnaldo Carvalho de Melo } else { 404ab1e0a13SArnaldo Carvalho de Melo lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 405ab1e0a13SArnaldo Carvalho de Melo write_lock_bh(lock); 406ab1e0a13SArnaldo Carvalho de Melo } 407ab1e0a13SArnaldo Carvalho de Melo 408ab1e0a13SArnaldo Carvalho de Melo if (__sk_del_node_init(sk)) 409c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 410ab1e0a13SArnaldo Carvalho de Melo write_unlock_bh(lock); 411ab1e0a13SArnaldo Carvalho de Melo out: 412ab1e0a13SArnaldo Carvalho de Melo if (sk->sk_state == TCP_LISTEN) 413ab1e0a13SArnaldo Carvalho de Melo wake_up(&hashinfo->lhash_wait); 414ab1e0a13SArnaldo Carvalho de Melo } 415ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_unhash); 416152da81dSPavel Emelyanov 4175ee31fc1SPavel Emelyanov int __inet_hash_connect(struct inet_timewait_death_row *death_row, 4185d8c0aa9SPavel Emelyanov struct sock *sk, u32 port_offset, 4195ee31fc1SPavel Emelyanov int (*check_established)(struct inet_timewait_death_row *, 4205ee31fc1SPavel Emelyanov struct sock *, __u16, struct inet_timewait_sock **), 421ab1e0a13SArnaldo Carvalho de Melo void (*hash)(struct sock *sk)) 422a7f5e7f1SArnaldo Carvalho de Melo { 423a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 424a7f5e7f1SArnaldo Carvalho de Melo const unsigned short snum = inet_sk(sk)->num; 425a7f5e7f1SArnaldo Carvalho de Melo struct inet_bind_hashbucket *head; 426a7f5e7f1SArnaldo Carvalho de Melo struct inet_bind_bucket *tb; 427a7f5e7f1SArnaldo Carvalho de Melo int ret; 4283b1e0a65SYOSHIFUJI Hideaki struct net *net = sock_net(sk); 429a7f5e7f1SArnaldo Carvalho de Melo 430a7f5e7f1SArnaldo Carvalho de Melo if (!snum) { 431227b60f5SStephen Hemminger int i, remaining, low, high, port; 432a7f5e7f1SArnaldo Carvalho de Melo static u32 hint; 4335d8c0aa9SPavel Emelyanov u32 offset = hint + port_offset; 434a7f5e7f1SArnaldo Carvalho de Melo struct hlist_node *node; 435a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock *tw = NULL; 436a7f5e7f1SArnaldo Carvalho de Melo 437227b60f5SStephen Hemminger inet_get_local_port_range(&low, &high); 438a25de534SAnton Arapov remaining = (high - low) + 1; 439227b60f5SStephen Hemminger 440a7f5e7f1SArnaldo Carvalho de Melo local_bh_disable(); 441227b60f5SStephen Hemminger for (i = 1; i <= remaining; i++) { 442227b60f5SStephen Hemminger port = low + (i + offset) % remaining; 443*7f635ab7SPavel Emelyanov head = &hinfo->bhash[inet_bhashfn(net, port, 444*7f635ab7SPavel Emelyanov hinfo->bhash_size)]; 445a7f5e7f1SArnaldo Carvalho de Melo spin_lock(&head->lock); 446a7f5e7f1SArnaldo Carvalho de Melo 447a7f5e7f1SArnaldo Carvalho de Melo /* Does not bother with rcv_saddr checks, 448a7f5e7f1SArnaldo Carvalho de Melo * because the established check is already 449a7f5e7f1SArnaldo Carvalho de Melo * unique enough. 450a7f5e7f1SArnaldo Carvalho de Melo */ 451a7f5e7f1SArnaldo Carvalho de Melo inet_bind_bucket_for_each(tb, node, &head->chain) { 452941b1d22SPavel Emelyanov if (tb->ib_net == net && tb->port == port) { 453a7f5e7f1SArnaldo Carvalho de Melo BUG_TRAP(!hlist_empty(&tb->owners)); 454a7f5e7f1SArnaldo Carvalho de Melo if (tb->fastreuse >= 0) 455a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 4565ee31fc1SPavel Emelyanov if (!check_established(death_row, sk, 4575ee31fc1SPavel Emelyanov port, &tw)) 458a7f5e7f1SArnaldo Carvalho de Melo goto ok; 459a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 460a7f5e7f1SArnaldo Carvalho de Melo } 461a7f5e7f1SArnaldo Carvalho de Melo } 462a7f5e7f1SArnaldo Carvalho de Melo 463941b1d22SPavel Emelyanov tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, 464941b1d22SPavel Emelyanov net, head, port); 465a7f5e7f1SArnaldo Carvalho de Melo if (!tb) { 466a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 467a7f5e7f1SArnaldo Carvalho de Melo break; 468a7f5e7f1SArnaldo Carvalho de Melo } 469a7f5e7f1SArnaldo Carvalho de Melo tb->fastreuse = -1; 470a7f5e7f1SArnaldo Carvalho de Melo goto ok; 471a7f5e7f1SArnaldo Carvalho de Melo 472a7f5e7f1SArnaldo Carvalho de Melo next_port: 473a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 474a7f5e7f1SArnaldo Carvalho de Melo } 475a7f5e7f1SArnaldo Carvalho de Melo local_bh_enable(); 476a7f5e7f1SArnaldo Carvalho de Melo 477a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 478a7f5e7f1SArnaldo Carvalho de Melo 479a7f5e7f1SArnaldo Carvalho de Melo ok: 480a7f5e7f1SArnaldo Carvalho de Melo hint += i; 481a7f5e7f1SArnaldo Carvalho de Melo 482a7f5e7f1SArnaldo Carvalho de Melo /* Head lock still held and bh's disabled */ 483a7f5e7f1SArnaldo Carvalho de Melo inet_bind_hash(sk, tb, port); 484a7f5e7f1SArnaldo Carvalho de Melo if (sk_unhashed(sk)) { 485a7f5e7f1SArnaldo Carvalho de Melo inet_sk(sk)->sport = htons(port); 486ab1e0a13SArnaldo Carvalho de Melo hash(sk); 487a7f5e7f1SArnaldo Carvalho de Melo } 488a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 489a7f5e7f1SArnaldo Carvalho de Melo 490a7f5e7f1SArnaldo Carvalho de Melo if (tw) { 49153b3531bSAlexey Dobriyan inet_twsk_deschedule(tw, death_row); 492a7f5e7f1SArnaldo Carvalho de Melo inet_twsk_put(tw); 493a7f5e7f1SArnaldo Carvalho de Melo } 494a7f5e7f1SArnaldo Carvalho de Melo 495a7f5e7f1SArnaldo Carvalho de Melo ret = 0; 496a7f5e7f1SArnaldo Carvalho de Melo goto out; 497a7f5e7f1SArnaldo Carvalho de Melo } 498a7f5e7f1SArnaldo Carvalho de Melo 499*7f635ab7SPavel Emelyanov head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)]; 500a7f5e7f1SArnaldo Carvalho de Melo tb = inet_csk(sk)->icsk_bind_hash; 501a7f5e7f1SArnaldo Carvalho de Melo spin_lock_bh(&head->lock); 502a7f5e7f1SArnaldo Carvalho de Melo if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) { 503ab1e0a13SArnaldo Carvalho de Melo hash(sk); 504a7f5e7f1SArnaldo Carvalho de Melo spin_unlock_bh(&head->lock); 505a7f5e7f1SArnaldo Carvalho de Melo return 0; 506a7f5e7f1SArnaldo Carvalho de Melo } else { 507a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 508a7f5e7f1SArnaldo Carvalho de Melo /* No definite answer... Walk to established hash table */ 5095ee31fc1SPavel Emelyanov ret = check_established(death_row, sk, snum, NULL); 510a7f5e7f1SArnaldo Carvalho de Melo out: 511a7f5e7f1SArnaldo Carvalho de Melo local_bh_enable(); 512a7f5e7f1SArnaldo Carvalho de Melo return ret; 513a7f5e7f1SArnaldo Carvalho de Melo } 514a7f5e7f1SArnaldo Carvalho de Melo } 5155ee31fc1SPavel Emelyanov 5165ee31fc1SPavel Emelyanov /* 5175ee31fc1SPavel Emelyanov * Bind a port for a connect operation and hash it. 5185ee31fc1SPavel Emelyanov */ 5195ee31fc1SPavel Emelyanov int inet_hash_connect(struct inet_timewait_death_row *death_row, 5205ee31fc1SPavel Emelyanov struct sock *sk) 5215ee31fc1SPavel Emelyanov { 5225d8c0aa9SPavel Emelyanov return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk), 5235ee31fc1SPavel Emelyanov __inet_check_established, __inet_hash_nolisten); 5245ee31fc1SPavel Emelyanov } 525a7f5e7f1SArnaldo Carvalho de Melo 526a7f5e7f1SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash_connect); 527