1 // SPDX-License-Identifier: GPL-2.0-only
9 #include <linux/percpu-refcount.h>
13  * don't try to detect the ref hitting 0 - which means that get/put can just
15  * particular cpu can (and will) wrap - this is fine, when we go to shutdown the
24  * the ref hitting 0 on every put - this would require global synchronization
37 #define PERCPU_COUNT_BIAS	(1LU << (BITS_PER_LONG - 1))
45 		(ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC_DEAD);  in percpu_count_ptr()
49  * percpu_ref_init - initialize a percpu refcount
60  * Note that @release must not sleep - it may potentially be called from RCU
71 	ref->percpu_count_ptr = (unsigned long)  in percpu_ref_init()
73 	if (!ref->percpu_count_ptr)  in percpu_ref_init()
74 		return -ENOMEM;  in percpu_ref_init()
76 	data = kzalloc(sizeof(*ref->data), gfp);  in percpu_ref_init()
78 		free_percpu((void __percpu *)ref->percpu_count_ptr);  in percpu_ref_init()
79 		ref->percpu_count_ptr = 0;  in percpu_ref_init()
80 		return -ENOMEM;  in percpu_ref_init()
83 	data->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;  in percpu_ref_init()
84 	data->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT;  in percpu_ref_init()
87 		ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;  in percpu_ref_init()
88 		data->allow_reinit = true;  in percpu_ref_init()
94 		ref->percpu_count_ptr |= __PERCPU_REF_DEAD;  in percpu_ref_init()
98 	atomic_long_set(&data->count, start_count);  in percpu_ref_init()
100 	data->release = release;  in percpu_ref_init()
101 	data->confirm_switch = NULL;  in percpu_ref_init()
102 	data->ref = ref;  in percpu_ref_init()
103 	ref->data = data;  in percpu_ref_init()
113 		/* non-NULL confirm_switch indicates switching in progress */  in __percpu_ref_exit()
114 		WARN_ON_ONCE(ref->data && ref->data->confirm_switch);  in __percpu_ref_exit()
116 		ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD;  in __percpu_ref_exit()
121  * percpu_ref_exit - undo percpu_ref_init()
126  * function from are the @ref->release() callback or in init failure path
132 	struct percpu_ref_data *data = ref->data;  in percpu_ref_exit()
141 	ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) <<  in percpu_ref_exit()
143 	ref->data = NULL;  in percpu_ref_exit()
154 	struct percpu_ref *ref = data->ref;  in percpu_ref_call_confirm_rcu()
156 	data->confirm_switch(ref);  in percpu_ref_call_confirm_rcu()
157 	data->confirm_switch = NULL;  in percpu_ref_call_confirm_rcu()
160 	if (!data->allow_reinit)  in percpu_ref_call_confirm_rcu()
171 	struct percpu_ref *ref = data->ref;  in percpu_ref_switch_to_atomic_rcu()
175 	int cpu;  in percpu_ref_switch_to_atomic_rcu()  local
177 	for_each_possible_cpu(cpu)  in percpu_ref_switch_to_atomic_rcu()
178 		count += *per_cpu_ptr(percpu_count, cpu);  in percpu_ref_switch_to_atomic_rcu()
181 		 atomic_long_read(&data->count), count);  in percpu_ref_switch_to_atomic_rcu()
185 	 * to &ref->count; since gets could be happening on one cpu while puts  in percpu_ref_switch_to_atomic_rcu()
186 	 * happen on another, adding a single cpu's count could cause  in percpu_ref_switch_to_atomic_rcu()
187 	 * @ref->count to hit 0 before we've got a consistent value - but the  in percpu_ref_switch_to_atomic_rcu()
191 	 * &ref->count; we need the bias value to prevent &ref->count from  in percpu_ref_switch_to_atomic_rcu()
195 	atomic_long_add((long)count - PERCPU_COUNT_BIAS, &data->count);  in percpu_ref_switch_to_atomic_rcu()
197 	if (WARN_ONCE(atomic_long_read(&data->count) <= 0,  in percpu_ref_switch_to_atomic_rcu()
199 		      data->release, atomic_long_read(&data->count)) &&  in percpu_ref_switch_to_atomic_rcu()
205 	/* @ref is viewed as dead on all CPUs, send out switch confirmation */  in percpu_ref_switch_to_atomic_rcu()
216 	if (ref->percpu_count_ptr & __PERCPU_REF_ATOMIC) {  in __percpu_ref_switch_to_atomic()
223 	ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;  in __percpu_ref_switch_to_atomic()
226 	 * Non-NULL ->confirm_switch is used to indicate that switching is  in __percpu_ref_switch_to_atomic()
229 	ref->data->confirm_switch = confirm_switch ?:  in __percpu_ref_switch_to_atomic()
233 	call_rcu_hurry(&ref->data->rcu,  in __percpu_ref_switch_to_atomic()
240 	int cpu;  in __percpu_ref_switch_to_percpu()  local
244 	if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC))  in __percpu_ref_switch_to_percpu()
247 	if (WARN_ON_ONCE(!ref->data->allow_reinit))  in __percpu_ref_switch_to_percpu()
250 	atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count);  in __percpu_ref_switch_to_percpu()
253 	 * Restore per-cpu operation.  smp_store_release() is paired  in __percpu_ref_switch_to_percpu()
258 	for_each_possible_cpu(cpu)  in __percpu_ref_switch_to_percpu()
259 		*per_cpu_ptr(percpu_count, cpu) = 0;  in __percpu_ref_switch_to_percpu()
261 	smp_store_release(&ref->percpu_count_ptr,  in __percpu_ref_switch_to_percpu()
262 			  ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC);  in __percpu_ref_switch_to_percpu()
268 	struct percpu_ref_data *data = ref->data;  in __percpu_ref_switch_mode()
277 	wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch,  in __percpu_ref_switch_mode()
280 	if (data->force_atomic || percpu_ref_is_dying(ref))  in __percpu_ref_switch_mode()
287  * percpu_ref_switch_to_atomic - switch a percpu_ref to atomic mode
313 	ref->data->force_atomic = true;  in percpu_ref_switch_to_atomic()
321  * percpu_ref_switch_to_atomic_sync - switch a percpu_ref to atomic mode
331 	wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch);  in percpu_ref_switch_to_atomic_sync()
336  * percpu_ref_switch_to_percpu - switch a percpu_ref to percpu mode
340  * To re-use an expired ref, use percpu_ref_reinit().
359 	ref->data->force_atomic = false;  in percpu_ref_switch_to_percpu()
367  * percpu_ref_kill_and_confirm - drop the initial ref and schedule confirmation
392 		  ref->data->release);  in percpu_ref_kill_and_confirm()
394 	ref->percpu_count_ptr |= __PERCPU_REF_DEAD;  in percpu_ref_kill_and_confirm()
403  * percpu_ref_is_zero - test whether a percpu refcount reached zero
420 	if (ref->data)  in percpu_ref_is_zero()
421 		count = atomic_long_read(&ref->data->count);  in percpu_ref_is_zero()
423 		count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS;  in percpu_ref_is_zero()
431  * percpu_ref_reinit - re-initialize a percpu refcount
432  * @ref: perpcu_ref to re-initialize
434  * Re-initialize @ref so that it's in the same state as when it finished
450  * percpu_ref_resurrect - modify a percpu refcount from dead to live
456  * If @ref->release() frees @ref then the caller is responsible for
457  * guaranteeing that @ref->release() does not get called while this
473 	ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD;  in percpu_ref_resurrect()