1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2e48c414eSArnaldo Carvalho de Melo /*
3e48c414eSArnaldo Carvalho de Melo * INET An implementation of the TCP/IP protocol suite for the LINUX
4e48c414eSArnaldo Carvalho de Melo * operating system. INET is implemented using the BSD Socket
5e48c414eSArnaldo Carvalho de Melo * interface as the means of communication with the user level.
6e48c414eSArnaldo Carvalho de Melo *
7e48c414eSArnaldo Carvalho de Melo * Generic TIME_WAIT sockets functions
8e48c414eSArnaldo Carvalho de Melo *
9e48c414eSArnaldo Carvalho de Melo * From code orinally in TCP
10e48c414eSArnaldo Carvalho de Melo */
11e48c414eSArnaldo Carvalho de Melo
12172589ccSIlpo Järvinen #include <linux/kernel.h>
135a0e3ad6STejun Heo #include <linux/slab.h>
143a9a231dSPaul Gortmaker #include <linux/module.h>
15e48c414eSArnaldo Carvalho de Melo #include <net/inet_hashtables.h>
16e48c414eSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h>
17696ab2d3SArnaldo Carvalho de Melo #include <net/ip.h>
18e48c414eSArnaldo Carvalho de Melo
1913475a30SEric Dumazet
202a8875e7SEric Dumazet /**
212a8875e7SEric Dumazet * inet_twsk_bind_unhash - unhash a timewait socket from bind hash
222a8875e7SEric Dumazet * @tw: timewait socket
232a8875e7SEric Dumazet * @hashinfo: hashinfo pointer
242a8875e7SEric Dumazet *
252a8875e7SEric Dumazet * unhash a timewait socket from bind hash, if hashed.
262a8875e7SEric Dumazet * bind hash lock must be held by caller.
272a8875e7SEric Dumazet * Returns 1 if caller should call inet_twsk_put() after lock release.
283cdaedaeSEric Dumazet */
inet_twsk_bind_unhash(struct inet_timewait_sock * tw,struct inet_hashinfo * hashinfo)29fc01538fSEric Dumazet void inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
303cdaedaeSEric Dumazet struct inet_hashinfo *hashinfo)
313cdaedaeSEric Dumazet {
32936a192fSKuniyuki Iwashima struct inet_bind2_bucket *tb2 = tw->tw_tb2;
333cdaedaeSEric Dumazet struct inet_bind_bucket *tb = tw->tw_tb;
343cdaedaeSEric Dumazet
353cdaedaeSEric Dumazet if (!tb)
36fc01538fSEric Dumazet return;
373cdaedaeSEric Dumazet
38770041d3SKuniyuki Iwashima __sk_del_bind_node((struct sock *)tw);
393cdaedaeSEric Dumazet tw->tw_tb = NULL;
40936a192fSKuniyuki Iwashima tw->tw_tb2 = NULL;
41936a192fSKuniyuki Iwashima inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2);
428002d44fSKuniyuki Iwashima inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
43936a192fSKuniyuki Iwashima
44fc01538fSEric Dumazet __sock_put((struct sock *)tw);
453cdaedaeSEric Dumazet }
463cdaedaeSEric Dumazet
47e48c414eSArnaldo Carvalho de Melo /* Must be called with locally disabled BHs. */
inet_twsk_kill(struct inet_timewait_sock * tw)48789f558cSEric Dumazet static void inet_twsk_kill(struct inet_timewait_sock *tw)
49e48c414eSArnaldo Carvalho de Melo {
50789f558cSEric Dumazet struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
519db66bdcSEric Dumazet spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
52936a192fSKuniyuki Iwashima struct inet_bind_hashbucket *bhead, *bhead2;
53e48c414eSArnaldo Carvalho de Melo
549db66bdcSEric Dumazet spin_lock(lock);
55fc01538fSEric Dumazet sk_nulls_del_node_init_rcu((struct sock *)tw);
569db66bdcSEric Dumazet spin_unlock(lock);
57e48c414eSArnaldo Carvalho de Melo
58e48c414eSArnaldo Carvalho de Melo /* Disassociate with bind bucket. */
5904c494e6SEric Dumazet bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
6004c494e6SEric Dumazet hashinfo->bhash_size)];
61936a192fSKuniyuki Iwashima bhead2 = inet_bhashfn_portaddr(hashinfo, (struct sock *)tw,
62936a192fSKuniyuki Iwashima twsk_net(tw), tw->tw_num);
633cdaedaeSEric Dumazet
64e48c414eSArnaldo Carvalho de Melo spin_lock(&bhead->lock);
65936a192fSKuniyuki Iwashima spin_lock(&bhead2->lock);
66fc01538fSEric Dumazet inet_twsk_bind_unhash(tw, hashinfo);
67936a192fSKuniyuki Iwashima spin_unlock(&bhead2->lock);
68e48c414eSArnaldo Carvalho de Melo spin_unlock(&bhead->lock);
693cdaedaeSEric Dumazet
70e9bd0ccaSKuniyuki Iwashima refcount_dec(&tw->tw_dr->tw_refcount);
71789f558cSEric Dumazet inet_twsk_put(tw);
72e48c414eSArnaldo Carvalho de Melo }
73e48c414eSArnaldo Carvalho de Melo
inet_twsk_free(struct inet_timewait_sock * tw)7405dbc7b5SEric Dumazet void inet_twsk_free(struct inet_timewait_sock *tw)
757054fb93SPavel Emelyanov {
767054fb93SPavel Emelyanov struct module *owner = tw->tw_prot->owner;
777054fb93SPavel Emelyanov twsk_destructor((struct sock *)tw);
787054fb93SPavel Emelyanov kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
797054fb93SPavel Emelyanov module_put(owner);
807054fb93SPavel Emelyanov }
814dbc8ef7SArnaldo Carvalho de Melo
inet_twsk_put(struct inet_timewait_sock * tw)824dbc8ef7SArnaldo Carvalho de Melo void inet_twsk_put(struct inet_timewait_sock *tw)
834dbc8ef7SArnaldo Carvalho de Melo {
8441c6d650SReshetova, Elena if (refcount_dec_and_test(&tw->tw_refcnt))
854dbc8ef7SArnaldo Carvalho de Melo inet_twsk_free(tw);
867054fb93SPavel Emelyanov }
877054fb93SPavel Emelyanov EXPORT_SYMBOL_GPL(inet_twsk_put);
887054fb93SPavel Emelyanov
inet_twsk_add_node_rcu(struct inet_timewait_sock * tw,struct hlist_nulls_head * list)8981b3ade5SKuniyuki Iwashima static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
9005dbc7b5SEric Dumazet struct hlist_nulls_head *list)
9105dbc7b5SEric Dumazet {
9281b3ade5SKuniyuki Iwashima hlist_nulls_add_head_rcu(&tw->tw_node, list);
9305dbc7b5SEric Dumazet }
9405dbc7b5SEric Dumazet
inet_twsk_schedule(struct inet_timewait_sock * tw,int timeo)95*f81d0dd2SFlorian Westphal static void inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo)
96*f81d0dd2SFlorian Westphal {
97*f81d0dd2SFlorian Westphal __inet_twsk_schedule(tw, timeo, false);
98*f81d0dd2SFlorian Westphal }
99*f81d0dd2SFlorian Westphal
100e48c414eSArnaldo Carvalho de Melo /*
101c75ad7c7SFlorian Westphal * Enter the time wait state.
102e48c414eSArnaldo Carvalho de Melo * Essentially we whip up a timewait bucket, copy the relevant info into it
103e48c414eSArnaldo Carvalho de Melo * from the SK, and mess with hash chains and list linkage.
104b334b924SValentin Schneider *
105b334b924SValentin Schneider * The caller must not access @tw anymore after this function returns.
106e48c414eSArnaldo Carvalho de Melo */
inet_twsk_hashdance_schedule(struct inet_timewait_sock * tw,struct sock * sk,struct inet_hashinfo * hashinfo,int timeo)107b334b924SValentin Schneider void inet_twsk_hashdance_schedule(struct inet_timewait_sock *tw,
108b334b924SValentin Schneider struct sock *sk,
109b334b924SValentin Schneider struct inet_hashinfo *hashinfo,
110b334b924SValentin Schneider int timeo)
111e48c414eSArnaldo Carvalho de Melo {
112e48c414eSArnaldo Carvalho de Melo const struct inet_sock *inet = inet_sk(sk);
113463c84b9SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk);
11481c3d547SEric Dumazet struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
1159db66bdcSEric Dumazet spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
116936a192fSKuniyuki Iwashima struct inet_bind_hashbucket *bhead, *bhead2;
117936a192fSKuniyuki Iwashima
118e48c414eSArnaldo Carvalho de Melo /* Step 1: Put TW into bind hash. Original socket stays there too.
119e48c414eSArnaldo Carvalho de Melo Note, that any socket with inet->num != 0 MUST be bound in
120e48c414eSArnaldo Carvalho de Melo binding cache, even if it is closed.
121e48c414eSArnaldo Carvalho de Melo */
12204c494e6SEric Dumazet bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
12304c494e6SEric Dumazet hashinfo->bhash_size)];
124936a192fSKuniyuki Iwashima bhead2 = inet_bhashfn_portaddr(hashinfo, sk, twsk_net(tw), inet->inet_num);
125936a192fSKuniyuki Iwashima
126c75ad7c7SFlorian Westphal local_bh_disable();
127e599ea14SEric Dumazet spin_lock(&bhead->lock);
128936a192fSKuniyuki Iwashima spin_lock(&bhead2->lock);
129936a192fSKuniyuki Iwashima
130463c84b9SArnaldo Carvalho de Melo tw->tw_tb = icsk->icsk_bind_hash;
131547b792cSIlpo Järvinen WARN_ON(!icsk->icsk_bind_hash);
132936a192fSKuniyuki Iwashima
133936a192fSKuniyuki Iwashima tw->tw_tb2 = icsk->icsk_bind2_hash;
134936a192fSKuniyuki Iwashima WARN_ON(!icsk->icsk_bind2_hash);
135770041d3SKuniyuki Iwashima sk_add_bind_node((struct sock *)tw, &tw->tw_tb2->owners);
136936a192fSKuniyuki Iwashima
137936a192fSKuniyuki Iwashima spin_unlock(&bhead2->lock);
138e48c414eSArnaldo Carvalho de Melo spin_unlock(&bhead->lock);
139e48c414eSArnaldo Carvalho de Melo
1409db66bdcSEric Dumazet spin_lock(lock);
141e48c414eSArnaldo Carvalho de Melo
142b334b924SValentin Schneider /* Step 2: Hash TW into tcp ehash chain */
14381b3ade5SKuniyuki Iwashima inet_twsk_add_node_rcu(tw, &ehead->chain);
1443ab5aee7SEric Dumazet
14505dbc7b5SEric Dumazet /* Step 3: Remove SK from hash chain */
1463ab5aee7SEric Dumazet if (__sk_nulls_del_node_init_rcu(sk))
1473ab5aee7SEric Dumazet sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
148e48c414eSArnaldo Carvalho de Melo
149ec94c269SEric Dumazet
150b334b924SValentin Schneider /* Ensure above writes are committed into memory before updating the
151b334b924SValentin Schneider * refcount.
152b334b924SValentin Schneider * Provides ordering vs later refcount_inc().
153b334b924SValentin Schneider */
154b334b924SValentin Schneider smp_wmb();
155ec94c269SEric Dumazet /* tw_refcnt is set to 3 because we have :
156ec94c269SEric Dumazet * - one reference for bhash chain.
157ec94c269SEric Dumazet * - one reference for ehash chain.
158ec94c269SEric Dumazet * - one reference for timer.
159ec94c269SEric Dumazet * Also note that after this point, we lost our implicit reference
160ec94c269SEric Dumazet * so we are not allowed to use tw anymore.
161ec94c269SEric Dumazet */
162ec94c269SEric Dumazet refcount_set(&tw->tw_refcnt, 3);
163b334b924SValentin Schneider
164b334b924SValentin Schneider inet_twsk_schedule(tw, timeo);
165b334b924SValentin Schneider
166b334b924SValentin Schneider spin_unlock(lock);
167c75ad7c7SFlorian Westphal local_bh_enable();
168e48c414eSArnaldo Carvalho de Melo }
169b334b924SValentin Schneider EXPORT_SYMBOL_GPL(inet_twsk_hashdance_schedule);
170696ab2d3SArnaldo Carvalho de Melo
tw_timer_handler(struct timer_list * t)1711ab791dcSKees Cook static void tw_timer_handler(struct timer_list *t)
172c676270bSArnaldo Carvalho de Melo {
1731ab791dcSKees Cook struct inet_timewait_sock *tw = from_timer(tw, t, tw_timer);
174789f558cSEric Dumazet
175789f558cSEric Dumazet inet_twsk_kill(tw);
176789f558cSEric Dumazet }
177789f558cSEric Dumazet
inet_twsk_alloc(const struct sock * sk,struct inet_timewait_death_row * dr,const int state)178789f558cSEric Dumazet struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
179789f558cSEric Dumazet struct inet_timewait_death_row *dr,
180789f558cSEric Dumazet const int state)
181789f558cSEric Dumazet {
182789f558cSEric Dumazet struct inet_timewait_sock *tw;
183789f558cSEric Dumazet
1846f605b57SKuniyuki Iwashima if (refcount_read(&dr->tw_refcount) - 1 >=
1856f605b57SKuniyuki Iwashima READ_ONCE(dr->sysctl_max_tw_buckets))
186789f558cSEric Dumazet return NULL;
187789f558cSEric Dumazet
188789f558cSEric Dumazet tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
18954e6ecb2SChristoph Lameter GFP_ATOMIC);
19000db4124SIan Morris if (tw) {
191c676270bSArnaldo Carvalho de Melo const struct inet_sock *inet = inet_sk(sk);
192c676270bSArnaldo Carvalho de Melo
193789f558cSEric Dumazet tw->tw_dr = dr;
194c676270bSArnaldo Carvalho de Melo /* Give us an identity. */
195c720c7e8SEric Dumazet tw->tw_daddr = inet->inet_daddr;
196c720c7e8SEric Dumazet tw->tw_rcv_saddr = inet->inet_rcv_saddr;
197c676270bSArnaldo Carvalho de Melo tw->tw_bound_dev_if = sk->sk_bound_dev_if;
19866b13d99SEric Dumazet tw->tw_tos = inet->tos;
199c720c7e8SEric Dumazet tw->tw_num = inet->inet_num;
200c676270bSArnaldo Carvalho de Melo tw->tw_state = TCP_TIME_WAIT;
201c676270bSArnaldo Carvalho de Melo tw->tw_substate = state;
202c720c7e8SEric Dumazet tw->tw_sport = inet->inet_sport;
203c720c7e8SEric Dumazet tw->tw_dport = inet->inet_dport;
204c676270bSArnaldo Carvalho de Melo tw->tw_family = sk->sk_family;
205c676270bSArnaldo Carvalho de Melo tw->tw_reuse = sk->sk_reuse;
2063099a529SEric Dumazet tw->tw_reuseport = sk->sk_reuseport;
20781c3d547SEric Dumazet tw->tw_hash = sk->sk_hash;
208c676270bSArnaldo Carvalho de Melo tw->tw_ipv6only = 0;
2094bd0623fSEric Dumazet tw->tw_transparent = inet_test_bit(TRANSPARENT, sk);
210c676270bSArnaldo Carvalho de Melo tw->tw_prot = sk->sk_prot_creator;
21133cf7c90SEric Dumazet atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
212efd7ef1cSEric W. Biederman twsk_net_set(tw, sock_net(sk));
213c75ad7c7SFlorian Westphal timer_setup(&tw->tw_timer, tw_timer_handler, 0);
21447e1c323SEric Dumazet /*
21547e1c323SEric Dumazet * Because we use RCU lookups, we should not set tw_refcnt
21647e1c323SEric Dumazet * to a non null value before everything is setup for this
21747e1c323SEric Dumazet * timewait socket.
21847e1c323SEric Dumazet */
21941c6d650SReshetova, Elena refcount_set(&tw->tw_refcnt, 0);
220789f558cSEric Dumazet
221eeb2b856SArnaldo Carvalho de Melo __module_get(tw->tw_prot->owner);
222c676270bSArnaldo Carvalho de Melo }
223c676270bSArnaldo Carvalho de Melo
224c676270bSArnaldo Carvalho de Melo return tw;
225c676270bSArnaldo Carvalho de Melo }
226696ab2d3SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(inet_twsk_alloc);
227696ab2d3SArnaldo Carvalho de Melo
228696ab2d3SArnaldo Carvalho de Melo /* These are always called from BH context. See callers in
229696ab2d3SArnaldo Carvalho de Melo * tcp_input.c to verify this.
230696ab2d3SArnaldo Carvalho de Melo */
231696ab2d3SArnaldo Carvalho de Melo
232dbe7faa4SEric Dumazet /* This is for handling early-kills of TIME_WAIT sockets.
233dbe7faa4SEric Dumazet * Warning : consume reference.
234dbe7faa4SEric Dumazet * Caller should not access tw anymore.
235dbe7faa4SEric Dumazet */
inet_twsk_deschedule_put(struct inet_timewait_sock * tw)236dbe7faa4SEric Dumazet void inet_twsk_deschedule_put(struct inet_timewait_sock *tw)
237696ab2d3SArnaldo Carvalho de Melo {
238b334b924SValentin Schneider struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
239b334b924SValentin Schneider spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
240b334b924SValentin Schneider
241b334b924SValentin Schneider /* inet_twsk_purge() walks over all sockets, including tw ones,
242b334b924SValentin Schneider * and removes them via inet_twsk_deschedule_put() after a
243b334b924SValentin Schneider * refcount_inc_not_zero().
244b334b924SValentin Schneider *
245b334b924SValentin Schneider * inet_twsk_hashdance_schedule() must (re)init the refcount before
246b334b924SValentin Schneider * arming the timer, i.e. inet_twsk_purge can obtain a reference to
247b334b924SValentin Schneider * a twsk that did not yet schedule the timer.
248b334b924SValentin Schneider *
249b334b924SValentin Schneider * The ehash lock synchronizes these two:
250b334b924SValentin Schneider * After acquiring the lock, the timer is always scheduled (else
251b334b924SValentin Schneider * timer_shutdown returns false), because hashdance_schedule releases
252b334b924SValentin Schneider * the ehash lock only after completing the timer initialization.
253b334b924SValentin Schneider *
254b334b924SValentin Schneider * Without grabbing the ehash lock, we get:
255b334b924SValentin Schneider * 1) cpu x sets twsk refcount to 3
256b334b924SValentin Schneider * 2) cpu y bumps refcount to 4
257b334b924SValentin Schneider * 3) cpu y calls inet_twsk_deschedule_put() and shuts timer down
258b334b924SValentin Schneider * 4) cpu x tries to start timer, but mod_timer is a noop post-shutdown
259b334b924SValentin Schneider * -> timer refcount is never decremented.
260b334b924SValentin Schneider */
261b334b924SValentin Schneider spin_lock(lock);
262b334b924SValentin Schneider /* Makes sure hashdance_schedule() has completed */
263b334b924SValentin Schneider spin_unlock(lock);
264b334b924SValentin Schneider
265b334b924SValentin Schneider if (timer_shutdown_sync(&tw->tw_timer))
266789f558cSEric Dumazet inet_twsk_kill(tw);
267dbe7faa4SEric Dumazet inet_twsk_put(tw);
268696ab2d3SArnaldo Carvalho de Melo }
269dbe7faa4SEric Dumazet EXPORT_SYMBOL(inet_twsk_deschedule_put);
270696ab2d3SArnaldo Carvalho de Melo
__inet_twsk_schedule(struct inet_timewait_sock * tw,int timeo,bool rearm)271ed2e9239SEric Dumazet void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
272696ab2d3SArnaldo Carvalho de Melo {
273696ab2d3SArnaldo Carvalho de Melo /* timeout := RTO * 3.5
274696ab2d3SArnaldo Carvalho de Melo *
275696ab2d3SArnaldo Carvalho de Melo * 3.5 = 1+2+0.5 to wait for two retransmits.
276696ab2d3SArnaldo Carvalho de Melo *
277696ab2d3SArnaldo Carvalho de Melo * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
278696ab2d3SArnaldo Carvalho de Melo * our ACK acking that FIN can be lost. If N subsequent retransmitted
279696ab2d3SArnaldo Carvalho de Melo * FINs (or previous seqments) are lost (probability of such event
280696ab2d3SArnaldo Carvalho de Melo * is p^(N+1), where p is probability to lose single packet and
281696ab2d3SArnaldo Carvalho de Melo * time to detect the loss is about RTO*(2^N - 1) with exponential
282696ab2d3SArnaldo Carvalho de Melo * backoff). Normal timewait length is calculated so, that we
283696ab2d3SArnaldo Carvalho de Melo * waited at least for one retransmitted FIN (maximal RTO is 120sec).
284696ab2d3SArnaldo Carvalho de Melo * [ BTW Linux. following BSD, violates this requirement waiting
285696ab2d3SArnaldo Carvalho de Melo * only for 60sec, we should wait at least for 240 secs.
286696ab2d3SArnaldo Carvalho de Melo * Well, 240 consumes too much of resources 8)
287696ab2d3SArnaldo Carvalho de Melo * ]
288696ab2d3SArnaldo Carvalho de Melo * This interval is not reduced to catch old duplicate and
289696ab2d3SArnaldo Carvalho de Melo * responces to our wandering segments living for two MSLs.
290696ab2d3SArnaldo Carvalho de Melo * However, if we use PAWS to detect
291696ab2d3SArnaldo Carvalho de Melo * old duplicates, we can reduce the interval to bounds required
292696ab2d3SArnaldo Carvalho de Melo * by RTO, rather than MSL. So, if peer understands PAWS, we
293696ab2d3SArnaldo Carvalho de Melo * kill tw bucket after 3.5*RTO (it is important that this number
294696ab2d3SArnaldo Carvalho de Melo * is greater than TS tick!) and detect old duplicates with help
295696ab2d3SArnaldo Carvalho de Melo * of PAWS.
296696ab2d3SArnaldo Carvalho de Melo */
297696ab2d3SArnaldo Carvalho de Melo
298ed2e9239SEric Dumazet if (!rearm) {
29927dd35e0SEric Dumazet bool kill = timeo <= 4*HZ;
30027dd35e0SEric Dumazet
30127dd35e0SEric Dumazet __NET_INC_STATS(twsk_net(tw), kill ? LINUX_MIB_TIMEWAITKILLED :
30227dd35e0SEric Dumazet LINUX_MIB_TIMEWAITED);
303f3438bc7SThomas Gleixner BUG_ON(mod_timer(&tw->tw_timer, jiffies + timeo));
304fbb82952SEric Dumazet refcount_inc(&tw->tw_dr->tw_refcount);
305ed2e9239SEric Dumazet } else {
306ed2e9239SEric Dumazet mod_timer_pending(&tw->tw_timer, jiffies + timeo);
307696ab2d3SArnaldo Carvalho de Melo }
308696ab2d3SArnaldo Carvalho de Melo }
309ed2e9239SEric Dumazet EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
31004c494e6SEric Dumazet
3111c4e97ddSEric Dumazet /* Remove all non full sockets (TIME_WAIT and NEW_SYN_RECV) for dead netns */
inet_twsk_purge(struct inet_hashinfo * hashinfo)3121eeb5043SEric Dumazet void inet_twsk_purge(struct inet_hashinfo *hashinfo)
31304c494e6SEric Dumazet {
31450e2907eSEric Dumazet struct inet_ehash_bucket *head = &hashinfo->ehash[0];
31550e2907eSEric Dumazet unsigned int ehash_mask = hashinfo->ehash_mask;
31604c494e6SEric Dumazet struct hlist_nulls_node *node;
31704c494e6SEric Dumazet unsigned int slot;
3181c4e97ddSEric Dumazet struct sock *sk;
31904c494e6SEric Dumazet
32050e2907eSEric Dumazet for (slot = 0; slot <= ehash_mask; slot++, head++) {
32150e2907eSEric Dumazet if (hlist_nulls_empty(&head->chain))
32250e2907eSEric Dumazet continue;
32350e2907eSEric Dumazet
32404c494e6SEric Dumazet restart_rcu:
32504c494e6SEric Dumazet cond_resched();
32604c494e6SEric Dumazet rcu_read_lock();
32704c494e6SEric Dumazet restart:
32804c494e6SEric Dumazet sk_nulls_for_each_rcu(sk, node, &head->chain) {
3291c4e97ddSEric Dumazet int state = inet_sk_state_load(sk);
330740ea3c4SKuniyuki Iwashima
3311c4e97ddSEric Dumazet if ((1 << state) & ~(TCPF_TIME_WAIT |
3321c4e97ddSEric Dumazet TCPF_NEW_SYN_RECV))
33304c494e6SEric Dumazet continue;
33404c494e6SEric Dumazet
3351eeb5043SEric Dumazet if (refcount_read(&sock_net(sk)->ns.count))
33604c494e6SEric Dumazet continue;
33704c494e6SEric Dumazet
3381c4e97ddSEric Dumazet if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
3391c4e97ddSEric Dumazet continue;
3401c4e97ddSEric Dumazet
3411eeb5043SEric Dumazet if (refcount_read(&sock_net(sk)->ns.count)) {
3421c4e97ddSEric Dumazet sock_gen_put(sk);
34304c494e6SEric Dumazet goto restart;
34404c494e6SEric Dumazet }
34504c494e6SEric Dumazet
34604c494e6SEric Dumazet rcu_read_unlock();
34704c494e6SEric Dumazet local_bh_disable();
3481c4e97ddSEric Dumazet if (state == TCP_TIME_WAIT) {
3491c4e97ddSEric Dumazet inet_twsk_deschedule_put(inet_twsk(sk));
3501c4e97ddSEric Dumazet } else {
3511c4e97ddSEric Dumazet struct request_sock *req = inet_reqsk(sk);
3521c4e97ddSEric Dumazet
3531c4e97ddSEric Dumazet inet_csk_reqsk_queue_drop_and_put(req->rsk_listener,
3541c4e97ddSEric Dumazet req);
3551c4e97ddSEric Dumazet }
35604c494e6SEric Dumazet local_bh_enable();
35704c494e6SEric Dumazet goto restart_rcu;
35804c494e6SEric Dumazet }
35904c494e6SEric Dumazet /* If the nulls value we got at the end of this lookup is
36004c494e6SEric Dumazet * not the expected one, we must restart lookup.
36104c494e6SEric Dumazet * We probably met an item that was moved to another chain.
36204c494e6SEric Dumazet */
36304c494e6SEric Dumazet if (get_nulls_value(node) != slot)
36404c494e6SEric Dumazet goto restart;
36504c494e6SEric Dumazet rcu_read_unlock();
36604c494e6SEric Dumazet }
36704c494e6SEric Dumazet }
36804c494e6SEric Dumazet EXPORT_SYMBOL_GPL(inet_twsk_purge);
369