12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 277d8bf9cSArnaldo Carvalho de Melo /* 377d8bf9cSArnaldo Carvalho de Melo * INET An implementation of the TCP/IP protocol suite for the LINUX 477d8bf9cSArnaldo Carvalho de Melo * operating system. INET is implemented using the BSD Socket 577d8bf9cSArnaldo Carvalho de Melo * interface as the means of communication with the user level. 677d8bf9cSArnaldo Carvalho de Melo * 777d8bf9cSArnaldo Carvalho de Melo * Generic INET transport hashtables 877d8bf9cSArnaldo Carvalho de Melo * 977d8bf9cSArnaldo Carvalho de Melo * Authors: Lotsa people, from code originally in tcp 1077d8bf9cSArnaldo Carvalho de Melo */ 1177d8bf9cSArnaldo Carvalho de Melo 122d8c4ce5SArnaldo Carvalho de Melo #include <linux/module.h> 13a7f5e7f1SArnaldo Carvalho de Melo #include <linux/random.h> 14f3f05f70SArnaldo Carvalho de Melo #include <linux/sched.h> 1577d8bf9cSArnaldo Carvalho de Melo #include <linux/slab.h> 16f3f05f70SArnaldo Carvalho de Melo #include <linux/wait.h> 17095dc8e0SEric Dumazet #include <linux/vmalloc.h> 1857c8a661SMike Rapoport #include <linux/memblock.h> 1977d8bf9cSArnaldo Carvalho de Melo 20c125e80bSCraig Gallek #include <net/addrconf.h> 21463c84b9SArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 2277d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 2301770a16SRicardo Dias #if IS_ENABLED(CONFIG_IPV6) 2401770a16SRicardo Dias #include <net/inet6_hashtables.h> 2501770a16SRicardo Dias #endif 266e5714eaSDavid S. Miller #include <net/secure_seq.h> 27a7f5e7f1SArnaldo Carvalho de Melo #include <net/ip.h> 28a04a480dSDavid Ahern #include <net/tcp.h> 29c125e80bSCraig Gallek #include <net/sock_reuseport.h> 3077d8bf9cSArnaldo Carvalho de Melo 316eada011SEric Dumazet static u32 inet_ehashfn(const struct net *net, const __be32 laddr, 3265cd8033SHannes Frederic Sowa const __u16 lport, const __be32 faddr, 3365cd8033SHannes Frederic Sowa const __be16 fport) 3465cd8033SHannes Frederic Sowa { 351bbdceefSHannes Frederic Sowa static u32 inet_ehash_secret __read_mostly; 361bbdceefSHannes Frederic Sowa 371bbdceefSHannes Frederic Sowa net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret)); 381bbdceefSHannes Frederic Sowa 3965cd8033SHannes Frederic Sowa return __inet_ehashfn(laddr, lport, faddr, fport, 4065cd8033SHannes Frederic Sowa inet_ehash_secret + net_hash_mix(net)); 4165cd8033SHannes Frederic Sowa } 4265cd8033SHannes Frederic Sowa 43d1e559d0SEric Dumazet /* This function handles inet_sock, but also timewait and request sockets 44d1e559d0SEric Dumazet * for IPv4/IPv6. 45d1e559d0SEric Dumazet */ 46784c372aSEric Dumazet static u32 sk_ehashfn(const struct sock *sk) 4765cd8033SHannes Frederic Sowa { 48d1e559d0SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 49d1e559d0SEric Dumazet if (sk->sk_family == AF_INET6 && 50d1e559d0SEric Dumazet !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 51d1e559d0SEric Dumazet return inet6_ehashfn(sock_net(sk), 52d1e559d0SEric Dumazet &sk->sk_v6_rcv_saddr, sk->sk_num, 53d1e559d0SEric Dumazet &sk->sk_v6_daddr, sk->sk_dport); 54d1e559d0SEric Dumazet #endif 555b441f76SEric Dumazet return inet_ehashfn(sock_net(sk), 565b441f76SEric Dumazet sk->sk_rcv_saddr, sk->sk_num, 575b441f76SEric Dumazet sk->sk_daddr, sk->sk_dport); 5865cd8033SHannes Frederic Sowa } 5965cd8033SHannes Frederic Sowa 6077d8bf9cSArnaldo Carvalho de Melo /* 6177d8bf9cSArnaldo Carvalho de Melo * Allocate and initialize a new local port bind bucket. 6277d8bf9cSArnaldo Carvalho de Melo * The bindhash mutex for snum's hash chain must be held here. 6377d8bf9cSArnaldo Carvalho de Melo */ 64e18b890bSChristoph Lameter struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, 65941b1d22SPavel Emelyanov struct net *net, 6677d8bf9cSArnaldo Carvalho de Melo struct inet_bind_hashbucket *head, 673c82a21fSRobert Shearman const unsigned short snum, 683c82a21fSRobert Shearman int l3mdev) 6977d8bf9cSArnaldo Carvalho de Melo { 7054e6ecb2SChristoph Lameter struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); 7177d8bf9cSArnaldo Carvalho de Melo 7200db4124SIan Morris if (tb) { 73efd7ef1cSEric W. Biederman write_pnet(&tb->ib_net, net); 743c82a21fSRobert Shearman tb->l3mdev = l3mdev; 7577d8bf9cSArnaldo Carvalho de Melo tb->port = snum; 7677d8bf9cSArnaldo Carvalho de Melo tb->fastreuse = 0; 77da5e3630STom Herbert tb->fastreuseport = 0; 7877d8bf9cSArnaldo Carvalho de Melo INIT_HLIST_HEAD(&tb->owners); 7977d8bf9cSArnaldo Carvalho de Melo hlist_add_head(&tb->node, &head->chain); 8077d8bf9cSArnaldo Carvalho de Melo } 8177d8bf9cSArnaldo Carvalho de Melo return tb; 8277d8bf9cSArnaldo Carvalho de Melo } 8377d8bf9cSArnaldo Carvalho de Melo 8477d8bf9cSArnaldo Carvalho de Melo /* 8577d8bf9cSArnaldo Carvalho de Melo * Caller must hold hashbucket lock for this tb with local BH disabled 8677d8bf9cSArnaldo Carvalho de Melo */ 87e18b890bSChristoph Lameter void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb) 8877d8bf9cSArnaldo Carvalho de Melo { 8977d8bf9cSArnaldo Carvalho de Melo if (hlist_empty(&tb->owners)) { 9077d8bf9cSArnaldo Carvalho de Melo __hlist_del(&tb->node); 9177d8bf9cSArnaldo Carvalho de Melo kmem_cache_free(cachep, tb); 9277d8bf9cSArnaldo Carvalho de Melo } 9377d8bf9cSArnaldo Carvalho de Melo } 942d8c4ce5SArnaldo Carvalho de Melo 9528044fc1SJoanne Koong bool inet_bind_bucket_match(const struct inet_bind_bucket *tb, const struct net *net, 9628044fc1SJoanne Koong unsigned short port, int l3mdev) 972d8c4ce5SArnaldo Carvalho de Melo { 9828044fc1SJoanne Koong return net_eq(ib_net(tb), net) && tb->port == port && 9928044fc1SJoanne Koong tb->l3mdev == l3mdev; 10028044fc1SJoanne Koong } 10128044fc1SJoanne Koong 10228044fc1SJoanne Koong static void inet_bind2_bucket_init(struct inet_bind2_bucket *tb, 10328044fc1SJoanne Koong struct net *net, 10428044fc1SJoanne Koong struct inet_bind_hashbucket *head, 10528044fc1SJoanne Koong unsigned short port, int l3mdev, 10628044fc1SJoanne Koong const struct sock *sk) 10728044fc1SJoanne Koong { 10828044fc1SJoanne Koong write_pnet(&tb->ib_net, net); 10928044fc1SJoanne Koong tb->l3mdev = l3mdev; 11028044fc1SJoanne Koong tb->port = port; 11128044fc1SJoanne Koong #if IS_ENABLED(CONFIG_IPV6) 11228044fc1SJoanne Koong if (sk->sk_family == AF_INET6) 11328044fc1SJoanne Koong tb->v6_rcv_saddr = sk->sk_v6_rcv_saddr; 11428044fc1SJoanne Koong else 11528044fc1SJoanne Koong #endif 11628044fc1SJoanne Koong tb->rcv_saddr = sk->sk_rcv_saddr; 11728044fc1SJoanne Koong INIT_HLIST_HEAD(&tb->owners); 11828044fc1SJoanne Koong hlist_add_head(&tb->node, &head->chain); 11928044fc1SJoanne Koong } 12028044fc1SJoanne Koong 12128044fc1SJoanne Koong struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep, 12228044fc1SJoanne Koong struct net *net, 12328044fc1SJoanne Koong struct inet_bind_hashbucket *head, 12428044fc1SJoanne Koong unsigned short port, 12528044fc1SJoanne Koong int l3mdev, 12628044fc1SJoanne Koong const struct sock *sk) 12728044fc1SJoanne Koong { 12828044fc1SJoanne Koong struct inet_bind2_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); 12928044fc1SJoanne Koong 13028044fc1SJoanne Koong if (tb) 13128044fc1SJoanne Koong inet_bind2_bucket_init(tb, net, head, port, l3mdev, sk); 13228044fc1SJoanne Koong 13328044fc1SJoanne Koong return tb; 13428044fc1SJoanne Koong } 13528044fc1SJoanne Koong 13628044fc1SJoanne Koong /* Caller must hold hashbucket lock for this tb with local BH disabled */ 13728044fc1SJoanne Koong void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb) 13828044fc1SJoanne Koong { 13928044fc1SJoanne Koong if (hlist_empty(&tb->owners)) { 14028044fc1SJoanne Koong __hlist_del(&tb->node); 14128044fc1SJoanne Koong kmem_cache_free(cachep, tb); 14228044fc1SJoanne Koong } 14328044fc1SJoanne Koong } 14428044fc1SJoanne Koong 14528044fc1SJoanne Koong static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2, 14628044fc1SJoanne Koong const struct sock *sk) 14728044fc1SJoanne Koong { 14828044fc1SJoanne Koong #if IS_ENABLED(CONFIG_IPV6) 14928044fc1SJoanne Koong if (sk->sk_family == AF_INET6) 15028044fc1SJoanne Koong return ipv6_addr_equal(&tb2->v6_rcv_saddr, 15128044fc1SJoanne Koong &sk->sk_v6_rcv_saddr); 15228044fc1SJoanne Koong #endif 15328044fc1SJoanne Koong return tb2->rcv_saddr == sk->sk_rcv_saddr; 15428044fc1SJoanne Koong } 15528044fc1SJoanne Koong 15628044fc1SJoanne Koong void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, 15728044fc1SJoanne Koong struct inet_bind2_bucket *tb2, unsigned short port) 15828044fc1SJoanne Koong { 15928044fc1SJoanne Koong inet_sk(sk)->inet_num = port; 1602d8c4ce5SArnaldo Carvalho de Melo sk_add_bind_node(sk, &tb->owners); 161463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = tb; 16228044fc1SJoanne Koong sk_add_bind2_node(sk, &tb2->owners); 16328044fc1SJoanne Koong inet_csk(sk)->icsk_bind2_hash = tb2; 1642d8c4ce5SArnaldo Carvalho de Melo } 1652d8c4ce5SArnaldo Carvalho de Melo 1662d8c4ce5SArnaldo Carvalho de Melo /* 1672d8c4ce5SArnaldo Carvalho de Melo * Get rid of any references to a local port held by the given sock. 1682d8c4ce5SArnaldo Carvalho de Melo */ 169ab1e0a13SArnaldo Carvalho de Melo static void __inet_put_port(struct sock *sk) 1702d8c4ce5SArnaldo Carvalho de Melo { 17139d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 172*08eaef90SKuniyuki Iwashima struct inet_bind_hashbucket *head, *head2; 173*08eaef90SKuniyuki Iwashima struct net *net = sock_net(sk); 1742d8c4ce5SArnaldo Carvalho de Melo struct inet_bind_bucket *tb; 175*08eaef90SKuniyuki Iwashima int bhash; 176*08eaef90SKuniyuki Iwashima 177*08eaef90SKuniyuki Iwashima bhash = inet_bhashfn(net, inet_sk(sk)->inet_num, hashinfo->bhash_size); 178*08eaef90SKuniyuki Iwashima head = &hashinfo->bhash[bhash]; 179*08eaef90SKuniyuki Iwashima head2 = inet_bhashfn_portaddr(hashinfo, sk, net, inet_sk(sk)->inet_num); 1802d8c4ce5SArnaldo Carvalho de Melo 1812d8c4ce5SArnaldo Carvalho de Melo spin_lock(&head->lock); 182463c84b9SArnaldo Carvalho de Melo tb = inet_csk(sk)->icsk_bind_hash; 1832d8c4ce5SArnaldo Carvalho de Melo __sk_del_bind_node(sk); 184463c84b9SArnaldo Carvalho de Melo inet_csk(sk)->icsk_bind_hash = NULL; 185c720c7e8SEric Dumazet inet_sk(sk)->inet_num = 0; 1862d8c4ce5SArnaldo Carvalho de Melo inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); 18728044fc1SJoanne Koong 18828044fc1SJoanne Koong spin_lock(&head2->lock); 18928044fc1SJoanne Koong if (inet_csk(sk)->icsk_bind2_hash) { 19028044fc1SJoanne Koong struct inet_bind2_bucket *tb2 = inet_csk(sk)->icsk_bind2_hash; 19128044fc1SJoanne Koong 19228044fc1SJoanne Koong __sk_del_bind2_node(sk); 19328044fc1SJoanne Koong inet_csk(sk)->icsk_bind2_hash = NULL; 19428044fc1SJoanne Koong inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2); 19528044fc1SJoanne Koong } 19628044fc1SJoanne Koong spin_unlock(&head2->lock); 19728044fc1SJoanne Koong 1982d8c4ce5SArnaldo Carvalho de Melo spin_unlock(&head->lock); 1992d8c4ce5SArnaldo Carvalho de Melo } 2002d8c4ce5SArnaldo Carvalho de Melo 201ab1e0a13SArnaldo Carvalho de Melo void inet_put_port(struct sock *sk) 2022d8c4ce5SArnaldo Carvalho de Melo { 2032d8c4ce5SArnaldo Carvalho de Melo local_bh_disable(); 204ab1e0a13SArnaldo Carvalho de Melo __inet_put_port(sk); 2052d8c4ce5SArnaldo Carvalho de Melo local_bh_enable(); 2062d8c4ce5SArnaldo Carvalho de Melo } 2072d8c4ce5SArnaldo Carvalho de Melo EXPORT_SYMBOL(inet_put_port); 208f3f05f70SArnaldo Carvalho de Melo 2091ce31c9eSEric Dumazet int __inet_inherit_port(const struct sock *sk, struct sock *child) 21053083773SPavel Emelyanov { 21153083773SPavel Emelyanov struct inet_hashinfo *table = sk->sk_prot->h.hashinfo; 212093d2823SBalazs Scheidler unsigned short port = inet_sk(child)->inet_num; 213*08eaef90SKuniyuki Iwashima struct inet_bind_hashbucket *head, *head2; 21428044fc1SJoanne Koong bool created_inet_bind_bucket = false; 21528044fc1SJoanne Koong struct net *net = sock_net(sk); 216*08eaef90SKuniyuki Iwashima bool update_fastreuse = false; 21728044fc1SJoanne Koong struct inet_bind2_bucket *tb2; 21853083773SPavel Emelyanov struct inet_bind_bucket *tb; 219*08eaef90SKuniyuki Iwashima int bhash, l3mdev; 220*08eaef90SKuniyuki Iwashima 221*08eaef90SKuniyuki Iwashima bhash = inet_bhashfn(net, port, table->bhash_size); 222*08eaef90SKuniyuki Iwashima head = &table->bhash[bhash]; 223*08eaef90SKuniyuki Iwashima head2 = inet_bhashfn_portaddr(table, child, net, port); 22453083773SPavel Emelyanov 22553083773SPavel Emelyanov spin_lock(&head->lock); 22628044fc1SJoanne Koong spin_lock(&head2->lock); 22753083773SPavel Emelyanov tb = inet_csk(sk)->icsk_bind_hash; 22828044fc1SJoanne Koong tb2 = inet_csk(sk)->icsk_bind2_hash; 22928044fc1SJoanne Koong if (unlikely(!tb || !tb2)) { 23028044fc1SJoanne Koong spin_unlock(&head2->lock); 231c2f34a65SEric Dumazet spin_unlock(&head->lock); 232c2f34a65SEric Dumazet return -ENOENT; 233c2f34a65SEric Dumazet } 234093d2823SBalazs Scheidler if (tb->port != port) { 2353c82a21fSRobert Shearman l3mdev = inet_sk_bound_l3mdev(sk); 2363c82a21fSRobert Shearman 237093d2823SBalazs Scheidler /* NOTE: using tproxy and redirecting skbs to a proxy 238093d2823SBalazs Scheidler * on a different listener port breaks the assumption 239093d2823SBalazs Scheidler * that the listener socket's icsk_bind_hash is the same 240093d2823SBalazs Scheidler * as that of the child socket. We have to look up or 241093d2823SBalazs Scheidler * create a new bind bucket for the child here. */ 242b67bfe0dSSasha Levin inet_bind_bucket_for_each(tb, &head->chain) { 24328044fc1SJoanne Koong if (inet_bind_bucket_match(tb, net, port, l3mdev)) 244093d2823SBalazs Scheidler break; 245093d2823SBalazs Scheidler } 246b67bfe0dSSasha Levin if (!tb) { 247093d2823SBalazs Scheidler tb = inet_bind_bucket_create(table->bind_bucket_cachep, 24828044fc1SJoanne Koong net, head, port, l3mdev); 249093d2823SBalazs Scheidler if (!tb) { 25028044fc1SJoanne Koong spin_unlock(&head2->lock); 251093d2823SBalazs Scheidler spin_unlock(&head->lock); 252093d2823SBalazs Scheidler return -ENOMEM; 253093d2823SBalazs Scheidler } 25428044fc1SJoanne Koong created_inet_bind_bucket = true; 255093d2823SBalazs Scheidler } 25628044fc1SJoanne Koong update_fastreuse = true; 25728044fc1SJoanne Koong 25828044fc1SJoanne Koong goto bhash2_find; 25928044fc1SJoanne Koong } else if (!inet_bind2_bucket_addr_match(tb2, child)) { 26028044fc1SJoanne Koong l3mdev = inet_sk_bound_l3mdev(sk); 26128044fc1SJoanne Koong 26228044fc1SJoanne Koong bhash2_find: 26328044fc1SJoanne Koong tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, child); 26428044fc1SJoanne Koong if (!tb2) { 26528044fc1SJoanne Koong tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep, 26628044fc1SJoanne Koong net, head2, port, 26728044fc1SJoanne Koong l3mdev, child); 26828044fc1SJoanne Koong if (!tb2) 26928044fc1SJoanne Koong goto error; 27028044fc1SJoanne Koong } 27128044fc1SJoanne Koong } 27228044fc1SJoanne Koong if (update_fastreuse) 273d76f3351STim Froidcoeur inet_csk_update_fastreuse(tb, child); 27428044fc1SJoanne Koong inet_bind_hash(child, tb, tb2, port); 27528044fc1SJoanne Koong spin_unlock(&head2->lock); 27653083773SPavel Emelyanov spin_unlock(&head->lock); 277093d2823SBalazs Scheidler 278093d2823SBalazs Scheidler return 0; 27928044fc1SJoanne Koong 28028044fc1SJoanne Koong error: 28128044fc1SJoanne Koong if (created_inet_bind_bucket) 28228044fc1SJoanne Koong inet_bind_bucket_destroy(table->bind_bucket_cachep, tb); 28328044fc1SJoanne Koong spin_unlock(&head2->lock); 28428044fc1SJoanne Koong spin_unlock(&head->lock); 28528044fc1SJoanne Koong return -ENOMEM; 28653083773SPavel Emelyanov } 28753083773SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_inherit_port); 28853083773SPavel Emelyanov 28961b7c691SMartin KaFai Lau static struct inet_listen_hashbucket * 29061b7c691SMartin KaFai Lau inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk) 29161b7c691SMartin KaFai Lau { 29261b7c691SMartin KaFai Lau u32 hash; 29361b7c691SMartin KaFai Lau 29461b7c691SMartin KaFai Lau #if IS_ENABLED(CONFIG_IPV6) 29561b7c691SMartin KaFai Lau if (sk->sk_family == AF_INET6) 29661b7c691SMartin KaFai Lau hash = ipv6_portaddr_hash(sock_net(sk), 29761b7c691SMartin KaFai Lau &sk->sk_v6_rcv_saddr, 29861b7c691SMartin KaFai Lau inet_sk(sk)->inet_num); 29961b7c691SMartin KaFai Lau else 30061b7c691SMartin KaFai Lau #endif 30161b7c691SMartin KaFai Lau hash = ipv4_portaddr_hash(sock_net(sk), 30261b7c691SMartin KaFai Lau inet_sk(sk)->inet_rcv_saddr, 30361b7c691SMartin KaFai Lau inet_sk(sk)->inet_num); 30461b7c691SMartin KaFai Lau return inet_lhash2_bucket(h, hash); 30561b7c691SMartin KaFai Lau } 30661b7c691SMartin KaFai Lau 307c25eb3bfSEric Dumazet static inline int compute_score(struct sock *sk, struct net *net, 308c25eb3bfSEric Dumazet const unsigned short hnum, const __be32 daddr, 30934e1ec31SMiaohe Lin const int dif, const int sdif) 310c25eb3bfSEric Dumazet { 311c25eb3bfSEric Dumazet int score = -1; 312c25eb3bfSEric Dumazet 313d9fbc7f6SPeter Oskolkov if (net_eq(sock_net(sk), net) && sk->sk_num == hnum && 314c25eb3bfSEric Dumazet !ipv6_only_sock(sk)) { 315d9fbc7f6SPeter Oskolkov if (sk->sk_rcv_saddr != daddr) 316c25eb3bfSEric Dumazet return -1; 317e7819058SMike Manning 318d9fbc7f6SPeter Oskolkov if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) 319d9fbc7f6SPeter Oskolkov return -1; 3208d6c414cSMike Manning score = sk->sk_bound_dev_if ? 2 : 1; 321d9fbc7f6SPeter Oskolkov 3228d6c414cSMike Manning if (sk->sk_family == PF_INET) 3238d6c414cSMike Manning score++; 3247170a977SEric Dumazet if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) 32570da268bSEric Dumazet score++; 326c25eb3bfSEric Dumazet } 327c25eb3bfSEric Dumazet return score; 328c25eb3bfSEric Dumazet } 329c25eb3bfSEric Dumazet 33080b373f7SJakub Sitnicki static inline struct sock *lookup_reuseport(struct net *net, struct sock *sk, 33180b373f7SJakub Sitnicki struct sk_buff *skb, int doff, 33280b373f7SJakub Sitnicki __be32 saddr, __be16 sport, 33380b373f7SJakub Sitnicki __be32 daddr, unsigned short hnum) 33480b373f7SJakub Sitnicki { 33580b373f7SJakub Sitnicki struct sock *reuse_sk = NULL; 33680b373f7SJakub Sitnicki u32 phash; 33780b373f7SJakub Sitnicki 33880b373f7SJakub Sitnicki if (sk->sk_reuseport) { 33980b373f7SJakub Sitnicki phash = inet_ehashfn(net, daddr, hnum, saddr, sport); 34080b373f7SJakub Sitnicki reuse_sk = reuseport_select_sock(sk, phash, skb, doff); 34180b373f7SJakub Sitnicki } 34280b373f7SJakub Sitnicki return reuse_sk; 34380b373f7SJakub Sitnicki } 34480b373f7SJakub Sitnicki 345f3f05f70SArnaldo Carvalho de Melo /* 3463b24d854SEric Dumazet * Here are some nice properties to exploit here. The BSD API 3473b24d854SEric Dumazet * does not allow a listening sock to specify the remote port nor the 34833b62231SArnaldo Carvalho de Melo * remote address for the connection. So always assume those are both 34933b62231SArnaldo Carvalho de Melo * wildcarded during the search since they can never be otherwise. 35033b62231SArnaldo Carvalho de Melo */ 35133b62231SArnaldo Carvalho de Melo 3523b24d854SEric Dumazet /* called with rcu_read_lock() : No refcount taken on the socket */ 35361b7c691SMartin KaFai Lau static struct sock *inet_lhash2_lookup(struct net *net, 35461b7c691SMartin KaFai Lau struct inet_listen_hashbucket *ilb2, 35561b7c691SMartin KaFai Lau struct sk_buff *skb, int doff, 35661b7c691SMartin KaFai Lau const __be32 saddr, __be16 sport, 35761b7c691SMartin KaFai Lau const __be32 daddr, const unsigned short hnum, 35861b7c691SMartin KaFai Lau const int dif, const int sdif) 35961b7c691SMartin KaFai Lau { 36061b7c691SMartin KaFai Lau struct sock *sk, *result = NULL; 361cae3873cSMartin KaFai Lau struct hlist_nulls_node *node; 36261b7c691SMartin KaFai Lau int score, hiscore = 0; 36361b7c691SMartin KaFai Lau 364cae3873cSMartin KaFai Lau sk_nulls_for_each_rcu(sk, node, &ilb2->nulls_head) { 36534e1ec31SMiaohe Lin score = compute_score(sk, net, hnum, daddr, dif, sdif); 36661b7c691SMartin KaFai Lau if (score > hiscore) { 36780b373f7SJakub Sitnicki result = lookup_reuseport(net, sk, skb, doff, 36880b373f7SJakub Sitnicki saddr, sport, daddr, hnum); 36961b7c691SMartin KaFai Lau if (result) 37061b7c691SMartin KaFai Lau return result; 37180b373f7SJakub Sitnicki 37261b7c691SMartin KaFai Lau result = sk; 37361b7c691SMartin KaFai Lau hiscore = score; 37461b7c691SMartin KaFai Lau } 37561b7c691SMartin KaFai Lau } 37661b7c691SMartin KaFai Lau 37761b7c691SMartin KaFai Lau return result; 37861b7c691SMartin KaFai Lau } 37961b7c691SMartin KaFai Lau 3801559b4aaSJakub Sitnicki static inline struct sock *inet_lookup_run_bpf(struct net *net, 3811559b4aaSJakub Sitnicki struct inet_hashinfo *hashinfo, 3821559b4aaSJakub Sitnicki struct sk_buff *skb, int doff, 3831559b4aaSJakub Sitnicki __be32 saddr, __be16 sport, 384f8931565SMark Pashmfouroush __be32 daddr, u16 hnum, const int dif) 3851559b4aaSJakub Sitnicki { 3861559b4aaSJakub Sitnicki struct sock *sk, *reuse_sk; 3871559b4aaSJakub Sitnicki bool no_reuseport; 3881559b4aaSJakub Sitnicki 3891559b4aaSJakub Sitnicki if (hashinfo != &tcp_hashinfo) 3901559b4aaSJakub Sitnicki return NULL; /* only TCP is supported */ 3911559b4aaSJakub Sitnicki 392f8931565SMark Pashmfouroush no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, saddr, sport, 393f8931565SMark Pashmfouroush daddr, hnum, dif, &sk); 3941559b4aaSJakub Sitnicki if (no_reuseport || IS_ERR_OR_NULL(sk)) 3951559b4aaSJakub Sitnicki return sk; 3961559b4aaSJakub Sitnicki 3971559b4aaSJakub Sitnicki reuse_sk = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum); 3981559b4aaSJakub Sitnicki if (reuse_sk) 3991559b4aaSJakub Sitnicki sk = reuse_sk; 4001559b4aaSJakub Sitnicki return sk; 4011559b4aaSJakub Sitnicki } 4021559b4aaSJakub Sitnicki 403c67499c0SPavel Emelyanov struct sock *__inet_lookup_listener(struct net *net, 404c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 405a583636aSCraig Gallek struct sk_buff *skb, int doff, 406da5e3630STom Herbert const __be32 saddr, __be16 sport, 407fb99c848SAl Viro const __be32 daddr, const unsigned short hnum, 4083fa6f616SDavid Ahern const int dif, const int sdif) 40999a92ff5SHerbert Xu { 41061b7c691SMartin KaFai Lau struct inet_listen_hashbucket *ilb2; 411d9fbc7f6SPeter Oskolkov struct sock *result = NULL; 41261b7c691SMartin KaFai Lau unsigned int hash2; 41361b7c691SMartin KaFai Lau 4141559b4aaSJakub Sitnicki /* Lookup redirect from BPF */ 4151559b4aaSJakub Sitnicki if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { 4161559b4aaSJakub Sitnicki result = inet_lookup_run_bpf(net, hashinfo, skb, doff, 417f8931565SMark Pashmfouroush saddr, sport, daddr, hnum, dif); 4181559b4aaSJakub Sitnicki if (result) 4191559b4aaSJakub Sitnicki goto done; 4201559b4aaSJakub Sitnicki } 4211559b4aaSJakub Sitnicki 42261b7c691SMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, daddr, hnum); 42361b7c691SMartin KaFai Lau ilb2 = inet_lhash2_bucket(hashinfo, hash2); 42461b7c691SMartin KaFai Lau 42561b7c691SMartin KaFai Lau result = inet_lhash2_lookup(net, ilb2, skb, doff, 42661b7c691SMartin KaFai Lau saddr, sport, daddr, hnum, 42761b7c691SMartin KaFai Lau dif, sdif); 42861b7c691SMartin KaFai Lau if (result) 4298217ca65SMartin KaFai Lau goto done; 43061b7c691SMartin KaFai Lau 43161b7c691SMartin KaFai Lau /* Lookup lhash2 with INADDR_ANY */ 43261b7c691SMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum); 43361b7c691SMartin KaFai Lau ilb2 = inet_lhash2_bucket(hashinfo, hash2); 43461b7c691SMartin KaFai Lau 4358217ca65SMartin KaFai Lau result = inet_lhash2_lookup(net, ilb2, skb, doff, 436d9fbc7f6SPeter Oskolkov saddr, sport, htonl(INADDR_ANY), hnum, 43761b7c691SMartin KaFai Lau dif, sdif); 4388217ca65SMartin KaFai Lau done: 43988e235b8SEnrico Weigelt if (IS_ERR(result)) 4408217ca65SMartin KaFai Lau return NULL; 441c25eb3bfSEric Dumazet return result; 44299a92ff5SHerbert Xu } 4438f491069SHerbert Xu EXPORT_SYMBOL_GPL(__inet_lookup_listener); 444a7f5e7f1SArnaldo Carvalho de Melo 44505dbc7b5SEric Dumazet /* All sockets share common refcount, but have different destructors */ 44605dbc7b5SEric Dumazet void sock_gen_put(struct sock *sk) 44705dbc7b5SEric Dumazet { 44841c6d650SReshetova, Elena if (!refcount_dec_and_test(&sk->sk_refcnt)) 44905dbc7b5SEric Dumazet return; 45005dbc7b5SEric Dumazet 45105dbc7b5SEric Dumazet if (sk->sk_state == TCP_TIME_WAIT) 45205dbc7b5SEric Dumazet inet_twsk_free(inet_twsk(sk)); 45341b822c5SEric Dumazet else if (sk->sk_state == TCP_NEW_SYN_RECV) 45441b822c5SEric Dumazet reqsk_free(inet_reqsk(sk)); 45505dbc7b5SEric Dumazet else 45605dbc7b5SEric Dumazet sk_free(sk); 45705dbc7b5SEric Dumazet } 45805dbc7b5SEric Dumazet EXPORT_SYMBOL_GPL(sock_gen_put); 45905dbc7b5SEric Dumazet 4602c13270bSEric Dumazet void sock_edemux(struct sk_buff *skb) 4612c13270bSEric Dumazet { 4622c13270bSEric Dumazet sock_gen_put(skb->sk); 4632c13270bSEric Dumazet } 4642c13270bSEric Dumazet EXPORT_SYMBOL(sock_edemux); 4652c13270bSEric Dumazet 466c67499c0SPavel Emelyanov struct sock *__inet_lookup_established(struct net *net, 467c67499c0SPavel Emelyanov struct inet_hashinfo *hashinfo, 46877a5ba55SPavel Emelyanov const __be32 saddr, const __be16 sport, 46977a5ba55SPavel Emelyanov const __be32 daddr, const u16 hnum, 4703fa6f616SDavid Ahern const int dif, const int sdif) 47177a5ba55SPavel Emelyanov { 472c7228317SJoe Perches INET_ADDR_COOKIE(acookie, saddr, daddr); 47377a5ba55SPavel Emelyanov const __portpair ports = INET_COMBINED_PORTS(sport, hnum); 47477a5ba55SPavel Emelyanov struct sock *sk; 4753ab5aee7SEric Dumazet const struct hlist_nulls_node *node; 47677a5ba55SPavel Emelyanov /* Optimize here for direct hit, only listening connections can 47777a5ba55SPavel Emelyanov * have wildcards anyways. 47877a5ba55SPavel Emelyanov */ 4799f26b3adSPavel Emelyanov unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport); 480f373b53bSEric Dumazet unsigned int slot = hash & hashinfo->ehash_mask; 4813ab5aee7SEric Dumazet struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; 48277a5ba55SPavel Emelyanov 4833ab5aee7SEric Dumazet begin: 4843ab5aee7SEric Dumazet sk_nulls_for_each_rcu(sk, node, &head->chain) { 485ce43b03eSEric Dumazet if (sk->sk_hash != hash) 486ce43b03eSEric Dumazet continue; 487eda090c3SEric Dumazet if (likely(inet_match(net, sk, acookie, ports, dif, sdif))) { 48841c6d650SReshetova, Elena if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) 48905dbc7b5SEric Dumazet goto out; 490eda090c3SEric Dumazet if (unlikely(!inet_match(net, sk, acookie, 4914915d50eSEric Dumazet ports, dif, sdif))) { 49205dbc7b5SEric Dumazet sock_gen_put(sk); 4933ab5aee7SEric Dumazet goto begin; 49477a5ba55SPavel Emelyanov } 49505dbc7b5SEric Dumazet goto found; 4963ab5aee7SEric Dumazet } 4973ab5aee7SEric Dumazet } 4983ab5aee7SEric Dumazet /* 4993ab5aee7SEric Dumazet * if the nulls value we got at the end of this lookup is 5003ab5aee7SEric Dumazet * not the expected one, we must restart lookup. 5013ab5aee7SEric Dumazet * We probably met an item that was moved to another chain. 5023ab5aee7SEric Dumazet */ 5033ab5aee7SEric Dumazet if (get_nulls_value(node) != slot) 5043ab5aee7SEric Dumazet goto begin; 50577a5ba55SPavel Emelyanov out: 50605dbc7b5SEric Dumazet sk = NULL; 50705dbc7b5SEric Dumazet found: 50877a5ba55SPavel Emelyanov return sk; 50977a5ba55SPavel Emelyanov } 51077a5ba55SPavel Emelyanov EXPORT_SYMBOL_GPL(__inet_lookup_established); 51177a5ba55SPavel Emelyanov 512a7f5e7f1SArnaldo Carvalho de Melo /* called with local bh disabled */ 513a7f5e7f1SArnaldo Carvalho de Melo static int __inet_check_established(struct inet_timewait_death_row *death_row, 514a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk, __u16 lport, 515a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock **twp) 516a7f5e7f1SArnaldo Carvalho de Melo { 517a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 518a7f5e7f1SArnaldo Carvalho de Melo struct inet_sock *inet = inet_sk(sk); 519c720c7e8SEric Dumazet __be32 daddr = inet->inet_rcv_saddr; 520c720c7e8SEric Dumazet __be32 saddr = inet->inet_daddr; 521a7f5e7f1SArnaldo Carvalho de Melo int dif = sk->sk_bound_dev_if; 5223fa6f616SDavid Ahern struct net *net = sock_net(sk); 5233fa6f616SDavid Ahern int sdif = l3mdev_master_ifindex_by_index(net, dif); 524c7228317SJoe Perches INET_ADDR_COOKIE(acookie, saddr, daddr); 525c720c7e8SEric Dumazet const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); 526c720c7e8SEric Dumazet unsigned int hash = inet_ehashfn(net, daddr, lport, 527c720c7e8SEric Dumazet saddr, inet->inet_dport); 528a7f5e7f1SArnaldo Carvalho de Melo struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); 5299db66bdcSEric Dumazet spinlock_t *lock = inet_ehash_lockp(hinfo, hash); 530a7f5e7f1SArnaldo Carvalho de Melo struct sock *sk2; 5313ab5aee7SEric Dumazet const struct hlist_nulls_node *node; 53205dbc7b5SEric Dumazet struct inet_timewait_sock *tw = NULL; 533a7f5e7f1SArnaldo Carvalho de Melo 5349db66bdcSEric Dumazet spin_lock(lock); 535a7f5e7f1SArnaldo Carvalho de Melo 5363ab5aee7SEric Dumazet sk_nulls_for_each(sk2, node, &head->chain) { 537ce43b03eSEric Dumazet if (sk2->sk_hash != hash) 538ce43b03eSEric Dumazet continue; 53905dbc7b5SEric Dumazet 540eda090c3SEric Dumazet if (likely(inet_match(net, sk2, acookie, ports, dif, sdif))) { 54105dbc7b5SEric Dumazet if (sk2->sk_state == TCP_TIME_WAIT) { 54205dbc7b5SEric Dumazet tw = inet_twsk(sk2); 54305dbc7b5SEric Dumazet if (twsk_unique(sk, sk2, twp)) 54405dbc7b5SEric Dumazet break; 54505dbc7b5SEric Dumazet } 546a7f5e7f1SArnaldo Carvalho de Melo goto not_unique; 547a7f5e7f1SArnaldo Carvalho de Melo } 54805dbc7b5SEric Dumazet } 549a7f5e7f1SArnaldo Carvalho de Melo 550a7f5e7f1SArnaldo Carvalho de Melo /* Must record num and sport now. Otherwise we will see 55105dbc7b5SEric Dumazet * in hash table socket with a funny identity. 55205dbc7b5SEric Dumazet */ 553c720c7e8SEric Dumazet inet->inet_num = lport; 554c720c7e8SEric Dumazet inet->inet_sport = htons(lport); 555a7f5e7f1SArnaldo Carvalho de Melo sk->sk_hash = hash; 556547b792cSIlpo Järvinen WARN_ON(!sk_unhashed(sk)); 5573ab5aee7SEric Dumazet __sk_nulls_add_node_rcu(sk, &head->chain); 55813475a30SEric Dumazet if (tw) { 559fc01538fSEric Dumazet sk_nulls_del_node_init_rcu((struct sock *)tw); 56002a1d6e7SEric Dumazet __NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED); 56113475a30SEric Dumazet } 5629db66bdcSEric Dumazet spin_unlock(lock); 563c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 564a7f5e7f1SArnaldo Carvalho de Melo 565a7f5e7f1SArnaldo Carvalho de Melo if (twp) { 566a7f5e7f1SArnaldo Carvalho de Melo *twp = tw; 567a7f5e7f1SArnaldo Carvalho de Melo } else if (tw) { 568a7f5e7f1SArnaldo Carvalho de Melo /* Silly. Should hash-dance instead... */ 569dbe7faa4SEric Dumazet inet_twsk_deschedule_put(tw); 570a7f5e7f1SArnaldo Carvalho de Melo } 571a7f5e7f1SArnaldo Carvalho de Melo return 0; 572a7f5e7f1SArnaldo Carvalho de Melo 573a7f5e7f1SArnaldo Carvalho de Melo not_unique: 5749db66bdcSEric Dumazet spin_unlock(lock); 575a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 576a7f5e7f1SArnaldo Carvalho de Melo } 577a7f5e7f1SArnaldo Carvalho de Melo 578b2d05756SWilly Tarreau static u64 inet_sk_port_offset(const struct sock *sk) 579a7f5e7f1SArnaldo Carvalho de Melo { 580a7f5e7f1SArnaldo Carvalho de Melo const struct inet_sock *inet = inet_sk(sk); 581e2baad9eSEric Dumazet 582c720c7e8SEric Dumazet return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr, 583c720c7e8SEric Dumazet inet->inet_daddr, 584c720c7e8SEric Dumazet inet->inet_dport); 585a7f5e7f1SArnaldo Carvalho de Melo } 586a7f5e7f1SArnaldo Carvalho de Melo 58701770a16SRicardo Dias /* Searches for an exsiting socket in the ehash bucket list. 58801770a16SRicardo Dias * Returns true if found, false otherwise. 589079096f1SEric Dumazet */ 59001770a16SRicardo Dias static bool inet_ehash_lookup_by_sk(struct sock *sk, 59101770a16SRicardo Dias struct hlist_nulls_head *list) 59201770a16SRicardo Dias { 59301770a16SRicardo Dias const __portpair ports = INET_COMBINED_PORTS(sk->sk_dport, sk->sk_num); 59401770a16SRicardo Dias const int sdif = sk->sk_bound_dev_if; 59501770a16SRicardo Dias const int dif = sk->sk_bound_dev_if; 59601770a16SRicardo Dias const struct hlist_nulls_node *node; 59701770a16SRicardo Dias struct net *net = sock_net(sk); 59801770a16SRicardo Dias struct sock *esk; 59901770a16SRicardo Dias 60001770a16SRicardo Dias INET_ADDR_COOKIE(acookie, sk->sk_daddr, sk->sk_rcv_saddr); 60101770a16SRicardo Dias 60201770a16SRicardo Dias sk_nulls_for_each_rcu(esk, node, list) { 60301770a16SRicardo Dias if (esk->sk_hash != sk->sk_hash) 60401770a16SRicardo Dias continue; 60501770a16SRicardo Dias if (sk->sk_family == AF_INET) { 606eda090c3SEric Dumazet if (unlikely(inet_match(net, esk, acookie, 60701770a16SRicardo Dias ports, dif, sdif))) { 60801770a16SRicardo Dias return true; 60901770a16SRicardo Dias } 61001770a16SRicardo Dias } 61101770a16SRicardo Dias #if IS_ENABLED(CONFIG_IPV6) 61201770a16SRicardo Dias else if (sk->sk_family == AF_INET6) { 6135d368f03SEric Dumazet if (unlikely(inet6_match(net, esk, 61401770a16SRicardo Dias &sk->sk_v6_daddr, 61501770a16SRicardo Dias &sk->sk_v6_rcv_saddr, 61601770a16SRicardo Dias ports, dif, sdif))) { 61701770a16SRicardo Dias return true; 61801770a16SRicardo Dias } 61901770a16SRicardo Dias } 62001770a16SRicardo Dias #endif 62101770a16SRicardo Dias } 62201770a16SRicardo Dias return false; 62301770a16SRicardo Dias } 62401770a16SRicardo Dias 62501770a16SRicardo Dias /* Insert a socket into ehash, and eventually remove another one 62601770a16SRicardo Dias * (The another one can be a SYN_RECV or TIMEWAIT) 62701770a16SRicardo Dias * If an existing socket already exists, socket sk is not inserted, 62801770a16SRicardo Dias * and sets found_dup_sk parameter to true. 62901770a16SRicardo Dias */ 63001770a16SRicardo Dias bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk) 631152da81dSPavel Emelyanov { 63239d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 633152da81dSPavel Emelyanov struct inet_ehash_bucket *head; 634*08eaef90SKuniyuki Iwashima struct hlist_nulls_head *list; 6355b441f76SEric Dumazet spinlock_t *lock; 6365e0724d0SEric Dumazet bool ret = true; 637152da81dSPavel Emelyanov 638079096f1SEric Dumazet WARN_ON_ONCE(!sk_unhashed(sk)); 639152da81dSPavel Emelyanov 6405b441f76SEric Dumazet sk->sk_hash = sk_ehashfn(sk); 641152da81dSPavel Emelyanov head = inet_ehash_bucket(hashinfo, sk->sk_hash); 642152da81dSPavel Emelyanov list = &head->chain; 643152da81dSPavel Emelyanov lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 644152da81dSPavel Emelyanov 6459db66bdcSEric Dumazet spin_lock(lock); 646fc01538fSEric Dumazet if (osk) { 6475e0724d0SEric Dumazet WARN_ON_ONCE(sk->sk_hash != osk->sk_hash); 6485e0724d0SEric Dumazet ret = sk_nulls_del_node_init_rcu(osk); 64901770a16SRicardo Dias } else if (found_dup_sk) { 65001770a16SRicardo Dias *found_dup_sk = inet_ehash_lookup_by_sk(sk, list); 65101770a16SRicardo Dias if (*found_dup_sk) 65201770a16SRicardo Dias ret = false; 6539327f705SEric Dumazet } 65401770a16SRicardo Dias 6555e0724d0SEric Dumazet if (ret) 6565e0724d0SEric Dumazet __sk_nulls_add_node_rcu(sk, list); 65701770a16SRicardo Dias 6589db66bdcSEric Dumazet spin_unlock(lock); 65901770a16SRicardo Dias 660079096f1SEric Dumazet return ret; 661079096f1SEric Dumazet } 662079096f1SEric Dumazet 66301770a16SRicardo Dias bool inet_ehash_nolisten(struct sock *sk, struct sock *osk, bool *found_dup_sk) 664079096f1SEric Dumazet { 66501770a16SRicardo Dias bool ok = inet_ehash_insert(sk, osk, found_dup_sk); 6665e0724d0SEric Dumazet 6675e0724d0SEric Dumazet if (ok) { 668c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 6695e0724d0SEric Dumazet } else { 67019757cebSEric Dumazet this_cpu_inc(*sk->sk_prot->orphan_count); 671563e0bb0SYafang Shao inet_sk_set_state(sk, TCP_CLOSE); 6725e0724d0SEric Dumazet sock_set_flag(sk, SOCK_DEAD); 6735e0724d0SEric Dumazet inet_csk_destroy_sock(sk); 674152da81dSPavel Emelyanov } 6755e0724d0SEric Dumazet return ok; 6765e0724d0SEric Dumazet } 6775e0724d0SEric Dumazet EXPORT_SYMBOL_GPL(inet_ehash_nolisten); 678152da81dSPavel Emelyanov 679c125e80bSCraig Gallek static int inet_reuseport_add_sock(struct sock *sk, 680fe38d2a1SJosef Bacik struct inet_listen_hashbucket *ilb) 681c125e80bSCraig Gallek { 68290e5d0dbSCraig Gallek struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash; 6838dbd76e7SEric Dumazet const struct hlist_nulls_node *node; 684c125e80bSCraig Gallek struct sock *sk2; 685c125e80bSCraig Gallek kuid_t uid = sock_i_uid(sk); 686c125e80bSCraig Gallek 6878dbd76e7SEric Dumazet sk_nulls_for_each_rcu(sk2, node, &ilb->nulls_head) { 688c125e80bSCraig Gallek if (sk2 != sk && 689c125e80bSCraig Gallek sk2->sk_family == sk->sk_family && 690c125e80bSCraig Gallek ipv6_only_sock(sk2) == ipv6_only_sock(sk) && 691c125e80bSCraig Gallek sk2->sk_bound_dev_if == sk->sk_bound_dev_if && 69290e5d0dbSCraig Gallek inet_csk(sk2)->icsk_bind_hash == tb && 693c125e80bSCraig Gallek sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && 694fe38d2a1SJosef Bacik inet_rcv_saddr_equal(sk, sk2, false)) 6952dbb9b9eSMartin KaFai Lau return reuseport_add_sock(sk, sk2, 6962dbb9b9eSMartin KaFai Lau inet_rcv_saddr_any(sk)); 697c125e80bSCraig Gallek } 698c125e80bSCraig Gallek 6992dbb9b9eSMartin KaFai Lau return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); 700c125e80bSCraig Gallek } 701c125e80bSCraig Gallek 702fe38d2a1SJosef Bacik int __inet_hash(struct sock *sk, struct sock *osk) 703152da81dSPavel Emelyanov { 70439d8cda7SPavel Emelyanov struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 705e8d00590SMartin KaFai Lau struct inet_listen_hashbucket *ilb2; 706c125e80bSCraig Gallek int err = 0; 707152da81dSPavel Emelyanov 7085e0724d0SEric Dumazet if (sk->sk_state != TCP_LISTEN) { 7094f9bf2a2SSebastian Andrzej Siewior local_bh_disable(); 71001770a16SRicardo Dias inet_ehash_nolisten(sk, osk, NULL); 7114f9bf2a2SSebastian Andrzej Siewior local_bh_enable(); 712c125e80bSCraig Gallek return 0; 7135e0724d0SEric Dumazet } 714547b792cSIlpo Järvinen WARN_ON(!sk_unhashed(sk)); 715e8d00590SMartin KaFai Lau ilb2 = inet_lhash2_bucket_sk(hashinfo, sk); 716152da81dSPavel Emelyanov 717e8d00590SMartin KaFai Lau spin_lock(&ilb2->lock); 718c125e80bSCraig Gallek if (sk->sk_reuseport) { 719cae3873cSMartin KaFai Lau err = inet_reuseport_add_sock(sk, ilb2); 720c125e80bSCraig Gallek if (err) 721c125e80bSCraig Gallek goto unlock; 722c125e80bSCraig Gallek } 723d296ba60SCraig Gallek if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && 724cae3873cSMartin KaFai Lau sk->sk_family == AF_INET6) 725cae3873cSMartin KaFai Lau __sk_nulls_add_node_tail_rcu(sk, &ilb2->nulls_head); 726cae3873cSMartin KaFai Lau else 727cae3873cSMartin KaFai Lau __sk_nulls_add_node_rcu(sk, &ilb2->nulls_head); 7283b24d854SEric Dumazet sock_set_flag(sk, SOCK_RCU_FREE); 729c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 730c125e80bSCraig Gallek unlock: 731e8d00590SMartin KaFai Lau spin_unlock(&ilb2->lock); 732c125e80bSCraig Gallek 733c125e80bSCraig Gallek return err; 734152da81dSPavel Emelyanov } 73577a6a471SEric Dumazet EXPORT_SYMBOL(__inet_hash); 736ab1e0a13SArnaldo Carvalho de Melo 737086c653fSCraig Gallek int inet_hash(struct sock *sk) 738ab1e0a13SArnaldo Carvalho de Melo { 739c125e80bSCraig Gallek int err = 0; 740c125e80bSCraig Gallek 7414f9bf2a2SSebastian Andrzej Siewior if (sk->sk_state != TCP_CLOSE) 742fe38d2a1SJosef Bacik err = __inet_hash(sk, NULL); 743086c653fSCraig Gallek 744c125e80bSCraig Gallek return err; 745ab1e0a13SArnaldo Carvalho de Melo } 746ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash); 747ab1e0a13SArnaldo Carvalho de Melo 7484f9bf2a2SSebastian Andrzej Siewior void inet_unhash(struct sock *sk) 7494f9bf2a2SSebastian Andrzej Siewior { 7504f9bf2a2SSebastian Andrzej Siewior struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; 7514f9bf2a2SSebastian Andrzej Siewior 7524f9bf2a2SSebastian Andrzej Siewior if (sk_unhashed(sk)) 7534f9bf2a2SSebastian Andrzej Siewior return; 7544f9bf2a2SSebastian Andrzej Siewior 7554f9bf2a2SSebastian Andrzej Siewior if (sk->sk_state == TCP_LISTEN) { 756e8d00590SMartin KaFai Lau struct inet_listen_hashbucket *ilb2; 7574f9bf2a2SSebastian Andrzej Siewior 758e8d00590SMartin KaFai Lau ilb2 = inet_lhash2_bucket_sk(hashinfo, sk); 7594f9bf2a2SSebastian Andrzej Siewior /* Don't disable bottom halves while acquiring the lock to 7604f9bf2a2SSebastian Andrzej Siewior * avoid circular locking dependency on PREEMPT_RT. 7614f9bf2a2SSebastian Andrzej Siewior */ 762e8d00590SMartin KaFai Lau spin_lock(&ilb2->lock); 763e8d00590SMartin KaFai Lau if (sk_unhashed(sk)) { 764e8d00590SMartin KaFai Lau spin_unlock(&ilb2->lock); 765e8d00590SMartin KaFai Lau return; 766e8d00590SMartin KaFai Lau } 767e8d00590SMartin KaFai Lau 768e8d00590SMartin KaFai Lau if (rcu_access_pointer(sk->sk_reuseport_cb)) 769e8d00590SMartin KaFai Lau reuseport_stop_listen_sock(sk); 770e8d00590SMartin KaFai Lau 771e8d00590SMartin KaFai Lau __sk_nulls_del_node_init_rcu(sk); 772e8d00590SMartin KaFai Lau sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 773e8d00590SMartin KaFai Lau spin_unlock(&ilb2->lock); 7744f9bf2a2SSebastian Andrzej Siewior } else { 7754f9bf2a2SSebastian Andrzej Siewior spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash); 7764f9bf2a2SSebastian Andrzej Siewior 7774f9bf2a2SSebastian Andrzej Siewior spin_lock_bh(lock); 778e8d00590SMartin KaFai Lau if (sk_unhashed(sk)) { 779e8d00590SMartin KaFai Lau spin_unlock_bh(lock); 780e8d00590SMartin KaFai Lau return; 781e8d00590SMartin KaFai Lau } 782e8d00590SMartin KaFai Lau __sk_nulls_del_node_init_rcu(sk); 783e8d00590SMartin KaFai Lau sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 784920de804SEric Dumazet spin_unlock_bh(lock); 785ab1e0a13SArnaldo Carvalho de Melo } 7864f9bf2a2SSebastian Andrzej Siewior } 787ab1e0a13SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_unhash); 788152da81dSPavel Emelyanov 78928044fc1SJoanne Koong static bool inet_bind2_bucket_match(const struct inet_bind2_bucket *tb, 79028044fc1SJoanne Koong const struct net *net, unsigned short port, 79128044fc1SJoanne Koong int l3mdev, const struct sock *sk) 79228044fc1SJoanne Koong { 79328044fc1SJoanne Koong #if IS_ENABLED(CONFIG_IPV6) 79428044fc1SJoanne Koong if (sk->sk_family == AF_INET6) 79528044fc1SJoanne Koong return net_eq(ib2_net(tb), net) && tb->port == port && 79628044fc1SJoanne Koong tb->l3mdev == l3mdev && 79728044fc1SJoanne Koong ipv6_addr_equal(&tb->v6_rcv_saddr, &sk->sk_v6_rcv_saddr); 79828044fc1SJoanne Koong else 79928044fc1SJoanne Koong #endif 80028044fc1SJoanne Koong return net_eq(ib2_net(tb), net) && tb->port == port && 80128044fc1SJoanne Koong tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr; 80228044fc1SJoanne Koong } 80328044fc1SJoanne Koong 80428044fc1SJoanne Koong bool inet_bind2_bucket_match_addr_any(const struct inet_bind2_bucket *tb, const struct net *net, 80528044fc1SJoanne Koong unsigned short port, int l3mdev, const struct sock *sk) 80628044fc1SJoanne Koong { 80728044fc1SJoanne Koong #if IS_ENABLED(CONFIG_IPV6) 80828044fc1SJoanne Koong struct in6_addr addr_any = {}; 80928044fc1SJoanne Koong 81028044fc1SJoanne Koong if (sk->sk_family == AF_INET6) 81128044fc1SJoanne Koong return net_eq(ib2_net(tb), net) && tb->port == port && 81228044fc1SJoanne Koong tb->l3mdev == l3mdev && 81328044fc1SJoanne Koong ipv6_addr_equal(&tb->v6_rcv_saddr, &addr_any); 81428044fc1SJoanne Koong else 81528044fc1SJoanne Koong #endif 81628044fc1SJoanne Koong return net_eq(ib2_net(tb), net) && tb->port == port && 81728044fc1SJoanne Koong tb->l3mdev == l3mdev && tb->rcv_saddr == 0; 81828044fc1SJoanne Koong } 81928044fc1SJoanne Koong 82028044fc1SJoanne Koong /* The socket's bhash2 hashbucket spinlock must be held when this is called */ 82128044fc1SJoanne Koong struct inet_bind2_bucket * 82228044fc1SJoanne Koong inet_bind2_bucket_find(const struct inet_bind_hashbucket *head, const struct net *net, 82328044fc1SJoanne Koong unsigned short port, int l3mdev, const struct sock *sk) 82428044fc1SJoanne Koong { 82528044fc1SJoanne Koong struct inet_bind2_bucket *bhash2 = NULL; 82628044fc1SJoanne Koong 82728044fc1SJoanne Koong inet_bind_bucket_for_each(bhash2, &head->chain) 82828044fc1SJoanne Koong if (inet_bind2_bucket_match(bhash2, net, port, l3mdev, sk)) 82928044fc1SJoanne Koong break; 83028044fc1SJoanne Koong 83128044fc1SJoanne Koong return bhash2; 83228044fc1SJoanne Koong } 83328044fc1SJoanne Koong 83428044fc1SJoanne Koong struct inet_bind_hashbucket * 83528044fc1SJoanne Koong inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port) 83628044fc1SJoanne Koong { 83728044fc1SJoanne Koong struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; 83828044fc1SJoanne Koong u32 hash; 83928044fc1SJoanne Koong #if IS_ENABLED(CONFIG_IPV6) 84028044fc1SJoanne Koong struct in6_addr addr_any = {}; 84128044fc1SJoanne Koong 84228044fc1SJoanne Koong if (sk->sk_family == AF_INET6) 84328044fc1SJoanne Koong hash = ipv6_portaddr_hash(net, &addr_any, port); 84428044fc1SJoanne Koong else 84528044fc1SJoanne Koong #endif 84628044fc1SJoanne Koong hash = ipv4_portaddr_hash(net, 0, port); 84728044fc1SJoanne Koong 84828044fc1SJoanne Koong return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)]; 84928044fc1SJoanne Koong } 85028044fc1SJoanne Koong 85128044fc1SJoanne Koong int inet_bhash2_update_saddr(struct inet_bind_hashbucket *prev_saddr, struct sock *sk) 85228044fc1SJoanne Koong { 85328044fc1SJoanne Koong struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; 85428044fc1SJoanne Koong struct inet_bind2_bucket *tb2, *new_tb2; 85528044fc1SJoanne Koong int l3mdev = inet_sk_bound_l3mdev(sk); 85628044fc1SJoanne Koong struct inet_bind_hashbucket *head2; 85728044fc1SJoanne Koong int port = inet_sk(sk)->inet_num; 85828044fc1SJoanne Koong struct net *net = sock_net(sk); 85928044fc1SJoanne Koong 86028044fc1SJoanne Koong /* Allocate a bind2 bucket ahead of time to avoid permanently putting 86128044fc1SJoanne Koong * the bhash2 table in an inconsistent state if a new tb2 bucket 86228044fc1SJoanne Koong * allocation fails. 86328044fc1SJoanne Koong */ 86428044fc1SJoanne Koong new_tb2 = kmem_cache_alloc(hinfo->bind2_bucket_cachep, GFP_ATOMIC); 86528044fc1SJoanne Koong if (!new_tb2) 86628044fc1SJoanne Koong return -ENOMEM; 86728044fc1SJoanne Koong 86828044fc1SJoanne Koong head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); 86928044fc1SJoanne Koong 87028044fc1SJoanne Koong if (prev_saddr) { 87128044fc1SJoanne Koong spin_lock_bh(&prev_saddr->lock); 87228044fc1SJoanne Koong __sk_del_bind2_node(sk); 87328044fc1SJoanne Koong inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, 87428044fc1SJoanne Koong inet_csk(sk)->icsk_bind2_hash); 87528044fc1SJoanne Koong spin_unlock_bh(&prev_saddr->lock); 87628044fc1SJoanne Koong } 87728044fc1SJoanne Koong 87828044fc1SJoanne Koong spin_lock_bh(&head2->lock); 87928044fc1SJoanne Koong tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 88028044fc1SJoanne Koong if (!tb2) { 88128044fc1SJoanne Koong tb2 = new_tb2; 88228044fc1SJoanne Koong inet_bind2_bucket_init(tb2, net, head2, port, l3mdev, sk); 88328044fc1SJoanne Koong } 88428044fc1SJoanne Koong sk_add_bind2_node(sk, &tb2->owners); 88528044fc1SJoanne Koong inet_csk(sk)->icsk_bind2_hash = tb2; 88628044fc1SJoanne Koong spin_unlock_bh(&head2->lock); 88728044fc1SJoanne Koong 88828044fc1SJoanne Koong if (tb2 != new_tb2) 88928044fc1SJoanne Koong kmem_cache_free(hinfo->bind2_bucket_cachep, new_tb2); 89028044fc1SJoanne Koong 89128044fc1SJoanne Koong return 0; 89228044fc1SJoanne Koong } 89328044fc1SJoanne Koong EXPORT_SYMBOL_GPL(inet_bhash2_update_saddr); 89428044fc1SJoanne Koong 895190cc824SEric Dumazet /* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm 896190cc824SEric Dumazet * Note that we use 32bit integers (vs RFC 'short integers') 897190cc824SEric Dumazet * because 2^16 is not a multiple of num_ephemeral and this 898190cc824SEric Dumazet * property might be used by clever attacker. 8994c2c8f03SWilly Tarreau * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, though 9004c2c8f03SWilly Tarreau * attacks were since demonstrated, thus we use 65536 instead to really 9014c2c8f03SWilly Tarreau * give more isolation and privacy, at the expense of 256kB of kernel 9024c2c8f03SWilly Tarreau * memory. 903190cc824SEric Dumazet */ 9044c2c8f03SWilly Tarreau #define INET_TABLE_PERTURB_SHIFT 16 905e9261476SWilly Tarreau #define INET_TABLE_PERTURB_SIZE (1 << INET_TABLE_PERTURB_SHIFT) 906e9261476SWilly Tarreau static u32 *table_perturb; 907190cc824SEric Dumazet 9085ee31fc1SPavel Emelyanov int __inet_hash_connect(struct inet_timewait_death_row *death_row, 909b2d05756SWilly Tarreau struct sock *sk, u64 port_offset, 9105ee31fc1SPavel Emelyanov int (*check_established)(struct inet_timewait_death_row *, 911b4d6444eSEric Dumazet struct sock *, __u16, struct inet_timewait_sock **)) 912a7f5e7f1SArnaldo Carvalho de Melo { 913a7f5e7f1SArnaldo Carvalho de Melo struct inet_hashinfo *hinfo = death_row->hashinfo; 91428044fc1SJoanne Koong struct inet_bind_hashbucket *head, *head2; 915a7f5e7f1SArnaldo Carvalho de Melo struct inet_timewait_sock *tw = NULL; 9161580ab63SEric Dumazet int port = inet_sk(sk)->inet_num; 9171580ab63SEric Dumazet struct net *net = sock_net(sk); 91828044fc1SJoanne Koong struct inet_bind2_bucket *tb2; 9191580ab63SEric Dumazet struct inet_bind_bucket *tb; 92028044fc1SJoanne Koong bool tb_created = false; 9211580ab63SEric Dumazet u32 remaining, offset; 9221580ab63SEric Dumazet int ret, i, low, high; 9233c82a21fSRobert Shearman int l3mdev; 924190cc824SEric Dumazet u32 index; 9251580ab63SEric Dumazet 9261580ab63SEric Dumazet if (port) { 9271580ab63SEric Dumazet head = &hinfo->bhash[inet_bhashfn(net, port, 9281580ab63SEric Dumazet hinfo->bhash_size)]; 9291580ab63SEric Dumazet tb = inet_csk(sk)->icsk_bind_hash; 9301580ab63SEric Dumazet spin_lock_bh(&head->lock); 9311580ab63SEric Dumazet if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) { 93201770a16SRicardo Dias inet_ehash_nolisten(sk, NULL, NULL); 9331580ab63SEric Dumazet spin_unlock_bh(&head->lock); 9341580ab63SEric Dumazet return 0; 9351580ab63SEric Dumazet } 9361580ab63SEric Dumazet spin_unlock(&head->lock); 9371580ab63SEric Dumazet /* No definite answer... Walk to established hash table */ 9381580ab63SEric Dumazet ret = check_established(death_row, sk, port, NULL); 9391580ab63SEric Dumazet local_bh_enable(); 9401580ab63SEric Dumazet return ret; 9411580ab63SEric Dumazet } 942a7f5e7f1SArnaldo Carvalho de Melo 9433c82a21fSRobert Shearman l3mdev = inet_sk_bound_l3mdev(sk); 9443c82a21fSRobert Shearman 9450bbf87d8SEric W. Biederman inet_get_local_port_range(net, &low, &high); 9461580ab63SEric Dumazet high++; /* [32768, 60999] -> [32768, 61000[ */ 9471580ab63SEric Dumazet remaining = high - low; 9481580ab63SEric Dumazet if (likely(remaining > 1)) 9491580ab63SEric Dumazet remaining &= ~1U; 950227b60f5SStephen Hemminger 951e9261476SWilly Tarreau net_get_random_once(table_perturb, 952e9261476SWilly Tarreau INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb)); 953e8161345SWilly Tarreau index = port_offset & (INET_TABLE_PERTURB_SIZE - 1); 954190cc824SEric Dumazet 9559e9b70aeSWilly Tarreau offset = READ_ONCE(table_perturb[index]) + (port_offset >> 32); 956b2d05756SWilly Tarreau offset %= remaining; 957b2d05756SWilly Tarreau 9581580ab63SEric Dumazet /* In first pass we try ports of @low parity. 9591580ab63SEric Dumazet * inet_csk_get_port() does the opposite choice. 96007f4c900SEric Dumazet */ 9611580ab63SEric Dumazet offset &= ~1U; 9621580ab63SEric Dumazet other_parity_scan: 9631580ab63SEric Dumazet port = low + offset; 9641580ab63SEric Dumazet for (i = 0; i < remaining; i += 2, port += 2) { 9651580ab63SEric Dumazet if (unlikely(port >= high)) 9661580ab63SEric Dumazet port -= remaining; 967122ff243SWANG Cong if (inet_is_local_reserved_port(net, port)) 968e3826f1eSAmerigo Wang continue; 9697f635ab7SPavel Emelyanov head = &hinfo->bhash[inet_bhashfn(net, port, 9707f635ab7SPavel Emelyanov hinfo->bhash_size)]; 9711580ab63SEric Dumazet spin_lock_bh(&head->lock); 972a7f5e7f1SArnaldo Carvalho de Melo 9731580ab63SEric Dumazet /* Does not bother with rcv_saddr checks, because 9741580ab63SEric Dumazet * the established check is already unique enough. 975a7f5e7f1SArnaldo Carvalho de Melo */ 976b67bfe0dSSasha Levin inet_bind_bucket_for_each(tb, &head->chain) { 97728044fc1SJoanne Koong if (inet_bind_bucket_match(tb, net, port, l3mdev)) { 978da5e3630STom Herbert if (tb->fastreuse >= 0 || 979da5e3630STom Herbert tb->fastreuseport >= 0) 980a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 981a9d8f911SEvgeniy Polyakov WARN_ON(hlist_empty(&tb->owners)); 9825ee31fc1SPavel Emelyanov if (!check_established(death_row, sk, 9835ee31fc1SPavel Emelyanov port, &tw)) 984a7f5e7f1SArnaldo Carvalho de Melo goto ok; 985a7f5e7f1SArnaldo Carvalho de Melo goto next_port; 986a7f5e7f1SArnaldo Carvalho de Melo } 987a7f5e7f1SArnaldo Carvalho de Melo } 988a7f5e7f1SArnaldo Carvalho de Melo 989941b1d22SPavel Emelyanov tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, 9903c82a21fSRobert Shearman net, head, port, l3mdev); 991a7f5e7f1SArnaldo Carvalho de Melo if (!tb) { 9921580ab63SEric Dumazet spin_unlock_bh(&head->lock); 9931580ab63SEric Dumazet return -ENOMEM; 994a7f5e7f1SArnaldo Carvalho de Melo } 99528044fc1SJoanne Koong tb_created = true; 996a7f5e7f1SArnaldo Carvalho de Melo tb->fastreuse = -1; 997da5e3630STom Herbert tb->fastreuseport = -1; 998a7f5e7f1SArnaldo Carvalho de Melo goto ok; 999a7f5e7f1SArnaldo Carvalho de Melo next_port: 10001580ab63SEric Dumazet spin_unlock_bh(&head->lock); 10011580ab63SEric Dumazet cond_resched(); 1002a7f5e7f1SArnaldo Carvalho de Melo } 10031580ab63SEric Dumazet 10041580ab63SEric Dumazet offset++; 10051580ab63SEric Dumazet if ((offset & 1) && remaining > 1) 10061580ab63SEric Dumazet goto other_parity_scan; 1007a7f5e7f1SArnaldo Carvalho de Melo 1008a7f5e7f1SArnaldo Carvalho de Melo return -EADDRNOTAVAIL; 1009a7f5e7f1SArnaldo Carvalho de Melo 1010a7f5e7f1SArnaldo Carvalho de Melo ok: 101128044fc1SJoanne Koong /* Find the corresponding tb2 bucket since we need to 101228044fc1SJoanne Koong * add the socket to the bhash2 table as well 101328044fc1SJoanne Koong */ 101428044fc1SJoanne Koong head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); 101528044fc1SJoanne Koong spin_lock(&head2->lock); 101628044fc1SJoanne Koong 101728044fc1SJoanne Koong tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 101828044fc1SJoanne Koong if (!tb2) { 101928044fc1SJoanne Koong tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, net, 102028044fc1SJoanne Koong head2, port, l3mdev, sk); 102128044fc1SJoanne Koong if (!tb2) 102228044fc1SJoanne Koong goto error; 102328044fc1SJoanne Koong } 102428044fc1SJoanne Koong 1025ca7af040SWilly Tarreau /* Here we want to add a little bit of randomness to the next source 1026ca7af040SWilly Tarreau * port that will be chosen. We use a max() with a random here so that 1027ca7af040SWilly Tarreau * on low contention the randomness is maximal and on high contention 1028ca7af040SWilly Tarreau * it may be inexistent. 1029c579bd1bSEric Dumazet */ 1030ca7af040SWilly Tarreau i = max_t(int, i, (prandom_u32() & 7) * 2); 1031190cc824SEric Dumazet WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2); 1032a7f5e7f1SArnaldo Carvalho de Melo 1033a7f5e7f1SArnaldo Carvalho de Melo /* Head lock still held and bh's disabled */ 103428044fc1SJoanne Koong inet_bind_hash(sk, tb, tb2, port); 103528044fc1SJoanne Koong 103628044fc1SJoanne Koong spin_unlock(&head2->lock); 103728044fc1SJoanne Koong 1038a7f5e7f1SArnaldo Carvalho de Melo if (sk_unhashed(sk)) { 1039c720c7e8SEric Dumazet inet_sk(sk)->inet_sport = htons(port); 104001770a16SRicardo Dias inet_ehash_nolisten(sk, (struct sock *)tw, NULL); 1041a7f5e7f1SArnaldo Carvalho de Melo } 10423cdaedaeSEric Dumazet if (tw) 1043fc01538fSEric Dumazet inet_twsk_bind_unhash(tw, hinfo); 1044a7f5e7f1SArnaldo Carvalho de Melo spin_unlock(&head->lock); 1045dbe7faa4SEric Dumazet if (tw) 1046dbe7faa4SEric Dumazet inet_twsk_deschedule_put(tw); 1047a7f5e7f1SArnaldo Carvalho de Melo local_bh_enable(); 10481580ab63SEric Dumazet return 0; 104928044fc1SJoanne Koong 105028044fc1SJoanne Koong error: 105128044fc1SJoanne Koong spin_unlock(&head2->lock); 105228044fc1SJoanne Koong if (tb_created) 105328044fc1SJoanne Koong inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb); 105428044fc1SJoanne Koong spin_unlock_bh(&head->lock); 105528044fc1SJoanne Koong return -ENOMEM; 1056a7f5e7f1SArnaldo Carvalho de Melo } 10575ee31fc1SPavel Emelyanov 10585ee31fc1SPavel Emelyanov /* 10595ee31fc1SPavel Emelyanov * Bind a port for a connect operation and hash it. 10605ee31fc1SPavel Emelyanov */ 10615ee31fc1SPavel Emelyanov int inet_hash_connect(struct inet_timewait_death_row *death_row, 10625ee31fc1SPavel Emelyanov struct sock *sk) 10635ee31fc1SPavel Emelyanov { 1064b2d05756SWilly Tarreau u64 port_offset = 0; 1065e2baad9eSEric Dumazet 1066e2baad9eSEric Dumazet if (!inet_sk(sk)->inet_num) 1067e2baad9eSEric Dumazet port_offset = inet_sk_port_offset(sk); 1068e2baad9eSEric Dumazet return __inet_hash_connect(death_row, sk, port_offset, 1069b4d6444eSEric Dumazet __inet_check_established); 10705ee31fc1SPavel Emelyanov } 1071a7f5e7f1SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_hash_connect); 10725caea4eaSEric Dumazet 1073c92c81dfSPeter Oskolkov static void init_hashinfo_lhash2(struct inet_hashinfo *h) 1074c92c81dfSPeter Oskolkov { 1075c92c81dfSPeter Oskolkov int i; 1076c92c81dfSPeter Oskolkov 1077c92c81dfSPeter Oskolkov for (i = 0; i <= h->lhash2_mask; i++) { 1078c92c81dfSPeter Oskolkov spin_lock_init(&h->lhash2[i].lock); 1079cae3873cSMartin KaFai Lau INIT_HLIST_NULLS_HEAD(&h->lhash2[i].nulls_head, 1080cae3873cSMartin KaFai Lau i + LISTENING_NULLS_BASE); 1081c92c81dfSPeter Oskolkov } 1082c92c81dfSPeter Oskolkov } 1083c92c81dfSPeter Oskolkov 108461b7c691SMartin KaFai Lau void __init inet_hashinfo2_init(struct inet_hashinfo *h, const char *name, 108561b7c691SMartin KaFai Lau unsigned long numentries, int scale, 108661b7c691SMartin KaFai Lau unsigned long low_limit, 108761b7c691SMartin KaFai Lau unsigned long high_limit) 108861b7c691SMartin KaFai Lau { 108961b7c691SMartin KaFai Lau h->lhash2 = alloc_large_system_hash(name, 109061b7c691SMartin KaFai Lau sizeof(*h->lhash2), 109161b7c691SMartin KaFai Lau numentries, 109261b7c691SMartin KaFai Lau scale, 109361b7c691SMartin KaFai Lau 0, 109461b7c691SMartin KaFai Lau NULL, 109561b7c691SMartin KaFai Lau &h->lhash2_mask, 109661b7c691SMartin KaFai Lau low_limit, 109761b7c691SMartin KaFai Lau high_limit); 1098c92c81dfSPeter Oskolkov init_hashinfo_lhash2(h); 1099e9261476SWilly Tarreau 1100e9261476SWilly Tarreau /* this one is used for source ports of outgoing connections */ 1101e67b72b9SMuchun Song table_perturb = alloc_large_system_hash("Table-perturb", 1102e67b72b9SMuchun Song sizeof(*table_perturb), 1103e67b72b9SMuchun Song INET_TABLE_PERTURB_SIZE, 1104e67b72b9SMuchun Song 0, 0, NULL, NULL, 1105e67b72b9SMuchun Song INET_TABLE_PERTURB_SIZE, 1106e67b72b9SMuchun Song INET_TABLE_PERTURB_SIZE); 1107c92c81dfSPeter Oskolkov } 110861b7c691SMartin KaFai Lau 1109c92c81dfSPeter Oskolkov int inet_hashinfo2_init_mod(struct inet_hashinfo *h) 1110c92c81dfSPeter Oskolkov { 1111c92c81dfSPeter Oskolkov h->lhash2 = kmalloc_array(INET_LHTABLE_SIZE, sizeof(*h->lhash2), GFP_KERNEL); 1112c92c81dfSPeter Oskolkov if (!h->lhash2) 1113c92c81dfSPeter Oskolkov return -ENOMEM; 1114c92c81dfSPeter Oskolkov 1115c92c81dfSPeter Oskolkov h->lhash2_mask = INET_LHTABLE_SIZE - 1; 1116c92c81dfSPeter Oskolkov /* INET_LHTABLE_SIZE must be a power of 2 */ 1117c92c81dfSPeter Oskolkov BUG_ON(INET_LHTABLE_SIZE & h->lhash2_mask); 1118c92c81dfSPeter Oskolkov 1119c92c81dfSPeter Oskolkov init_hashinfo_lhash2(h); 1120c92c81dfSPeter Oskolkov return 0; 112161b7c691SMartin KaFai Lau } 1122c92c81dfSPeter Oskolkov EXPORT_SYMBOL_GPL(inet_hashinfo2_init_mod); 112361b7c691SMartin KaFai Lau 1124095dc8e0SEric Dumazet int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo) 1125095dc8e0SEric Dumazet { 112689e478a2SEric Dumazet unsigned int locksz = sizeof(spinlock_t); 1127095dc8e0SEric Dumazet unsigned int i, nblocks = 1; 1128095dc8e0SEric Dumazet 112989e478a2SEric Dumazet if (locksz != 0) { 1130095dc8e0SEric Dumazet /* allocate 2 cache lines or at least one spinlock per cpu */ 113189e478a2SEric Dumazet nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U); 1132095dc8e0SEric Dumazet nblocks = roundup_pow_of_two(nblocks * num_possible_cpus()); 1133095dc8e0SEric Dumazet 1134095dc8e0SEric Dumazet /* no more locks than number of hash buckets */ 1135095dc8e0SEric Dumazet nblocks = min(nblocks, hashinfo->ehash_mask + 1); 1136095dc8e0SEric Dumazet 1137752ade68SMichal Hocko hashinfo->ehash_locks = kvmalloc_array(nblocks, locksz, GFP_KERNEL); 1138095dc8e0SEric Dumazet if (!hashinfo->ehash_locks) 1139095dc8e0SEric Dumazet return -ENOMEM; 1140095dc8e0SEric Dumazet 1141095dc8e0SEric Dumazet for (i = 0; i < nblocks; i++) 1142095dc8e0SEric Dumazet spin_lock_init(&hashinfo->ehash_locks[i]); 1143095dc8e0SEric Dumazet } 1144095dc8e0SEric Dumazet hashinfo->ehash_locks_mask = nblocks - 1; 1145095dc8e0SEric Dumazet return 0; 1146095dc8e0SEric Dumazet } 1147095dc8e0SEric Dumazet EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc); 1148