1 // SPDX-License-Identifier: GPL-2.0-only
6 #include <linux/percpu-rwsem.h>
15 			const char *name, struct lock_class_key *key)  in __percpu_init_rwsem()  argument
17 	sem->read_count = alloc_percpu(int);  in __percpu_init_rwsem()
18 	if (unlikely(!sem->read_count))  in __percpu_init_rwsem()
19 		return -ENOMEM;  in __percpu_init_rwsem()
21 	rcu_sync_init(&sem->rss);  in __percpu_init_rwsem()
22 	rcuwait_init(&sem->writer);  in __percpu_init_rwsem()
23 	init_waitqueue_head(&sem->waiters);  in __percpu_init_rwsem()
24 	atomic_set(&sem->block, 0);  in __percpu_init_rwsem()
27 	lockdep_init_map(&sem->dep_map, name, key, 0);  in __percpu_init_rwsem()
39 	if (!sem->read_count)  in percpu_free_rwsem()
42 	rcu_sync_dtor(&sem->rss);  in percpu_free_rwsem()
43 	free_percpu(sem->read_count);  in percpu_free_rwsem()
44 	sem->read_count = NULL; /* catch use after free bugs */  in percpu_free_rwsem()
50 	this_cpu_inc(*sem->read_count);  in __percpu_down_read_trylock()
55 	 * increment-on-one-CPU-and-decrement-on-another problem.  in __percpu_down_read_trylock()
57 	 * If the reader misses the writer's assignment of sem->block, then the  in __percpu_down_read_trylock()
60 	 * Conversely, any readers that increment their sem->read_count after  in __percpu_down_read_trylock()
61 	 * the writer looks are guaranteed to see the sem->block value, which  in __percpu_down_read_trylock()
63 	 * their sem->read_count, so that it doesn't matter that the writer  in __percpu_down_read_trylock()
70 	 * If !sem->block the critical section starts here, matched by the  in __percpu_down_read_trylock()
73 	if (likely(!atomic_read_acquire(&sem->block)))  in __percpu_down_read_trylock()
76 	this_cpu_dec(*sem->read_count);  in __percpu_down_read_trylock()
78 	/* Prod writer to re-evaluate readers_active_check() */  in __percpu_down_read_trylock()
79 	rcuwait_wake_up(&sem->writer);  in __percpu_down_read_trylock()
86 	if (atomic_read(&sem->block))  in __percpu_down_write_trylock()
89 	return atomic_xchg(&sem->block, 1) == 0;  in __percpu_down_write_trylock()
109  *  <0 - error, wakeup is terminated and the error is returned
110  *   0 - no wakeup, a next waiter is tried
111  *  >0 - woken, if EXCLUSIVE, counted towards @nr_exclusive.
121 				      void *key)  in percpu_rwsem_wake_function()  argument
123 	bool reader = wq_entry->flags & WQ_FLAG_CUSTOM;  in percpu_rwsem_wake_function()
124 	struct percpu_rw_semaphore *sem = key;  in percpu_rwsem_wake_function()
131 	p = get_task_struct(wq_entry->private);  in percpu_rwsem_wake_function()
132 	list_del_init(&wq_entry->entry);  in percpu_rwsem_wake_function()
133 	smp_store_release(&wq_entry->private, NULL);  in percpu_rwsem_wake_function()
146 	spin_lock_irq(&sem->waiters.lock);  in percpu_rwsem_wait()
148 	 * Serialize against the wakeup in percpu_up_write(), if we fail  in percpu_rwsem_wait()
149 	 * the trylock, the wakeup must see us on the list.  in percpu_rwsem_wait()
154 		__add_wait_queue_entry_tail(&sem->waiters, &wq_entry);  in percpu_rwsem_wait()
156 	spin_unlock_irq(&sem->waiters.lock);  in percpu_rwsem_wait()
197 	return per_cpu_sum(*sem->read_count) != 0 && !atomic_read(&sem->block);  in percpu_is_read_locked()
202  * Return true if the modular sum of the sem->read_count per-CPU variable is
207  * Assumes sem->block is set.
211 	if (per_cpu_sum(*sem->read_count) != 0)  in readers_active_check()
229 	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);  in percpu_down_write()
232 	rcu_sync_enter(&sem->rss);  in percpu_down_write()
235 	 * Try set sem->block; this provides writer-writer exclusion.  in percpu_down_write()
236 	 * Having sem->block set makes new readers block.  in percpu_down_write()
244 	/* smp_mb() implied by __percpu_down_write_trylock() on success -- D matches A */  in percpu_down_write()
247 	 * If they don't see our store of sem->block, then we are guaranteed to  in percpu_down_write()
248 	 * see their sem->read_count increment, and therefore will wait for  in percpu_down_write()
253 	rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);  in percpu_down_write()
261 	rwsem_release(&sem->dep_map, _RET_IP_);  in percpu_up_write()
273 	atomic_set_release(&sem->block, 0);  in percpu_up_write()
278 	__wake_up(&sem->waiters, TASK_NORMAL, 1, sem);  in percpu_up_write()
281 	 * Once this completes (at least one RCU-sched grace period hence) the  in percpu_up_write()
285 	rcu_sync_exit(&sem->rss);  in percpu_up_write()