11c6fdbd8SKent Overstreet /* SPDX-License-Identifier: GPL-2.0 */
21c6fdbd8SKent Overstreet
31c6fdbd8SKent Overstreet #ifndef _LINUX_SIX_H
41c6fdbd8SKent Overstreet #define _LINUX_SIX_H
51c6fdbd8SKent Overstreet
691d16f16SKent Overstreet /**
791d16f16SKent Overstreet * DOC: SIX locks overview
81c6fdbd8SKent Overstreet *
991d16f16SKent Overstreet * Shared/intent/exclusive locks: sleepable read/write locks, like rw semaphores
1091d16f16SKent Overstreet * but with an additional state: read/shared, intent, exclusive/write
1191d16f16SKent Overstreet *
1291d16f16SKent Overstreet * The purpose of the intent state is to allow for greater concurrency on tree
1391d16f16SKent Overstreet * structures without deadlocking. In general, a read can't be upgraded to a
1491d16f16SKent Overstreet * write lock without deadlocking, so an operation that updates multiple nodes
1591d16f16SKent Overstreet * will have to take write locks for the full duration of the operation.
1691d16f16SKent Overstreet *
1791d16f16SKent Overstreet * But by adding an intent state, which is exclusive with other intent locks but
18*b56cee70SRandy Dunlap * not with readers, we can take intent locks at the start of the operation,
1991d16f16SKent Overstreet * and then take write locks only for the actual update to each individual
2091d16f16SKent Overstreet * nodes, without deadlocking.
2191d16f16SKent Overstreet *
2291d16f16SKent Overstreet * Example usage:
231c6fdbd8SKent Overstreet * six_lock_read(&foo->lock);
241c6fdbd8SKent Overstreet * six_unlock_read(&foo->lock);
251c6fdbd8SKent Overstreet *
2691d16f16SKent Overstreet * An intent lock must be held before taking a write lock:
271c6fdbd8SKent Overstreet * six_lock_intent(&foo->lock);
281c6fdbd8SKent Overstreet * six_lock_write(&foo->lock);
291c6fdbd8SKent Overstreet * six_unlock_write(&foo->lock);
301c6fdbd8SKent Overstreet * six_unlock_intent(&foo->lock);
311c6fdbd8SKent Overstreet *
321c6fdbd8SKent Overstreet * Other operations:
331c6fdbd8SKent Overstreet * six_trylock_read()
341c6fdbd8SKent Overstreet * six_trylock_intent()
351c6fdbd8SKent Overstreet * six_trylock_write()
361c6fdbd8SKent Overstreet *
3791d16f16SKent Overstreet * six_lock_downgrade() convert from intent to read
3891d16f16SKent Overstreet * six_lock_tryupgrade() attempt to convert from read to intent, may fail
391c6fdbd8SKent Overstreet *
4091d16f16SKent Overstreet * There are also interfaces that take the lock type as an enum:
411c6fdbd8SKent Overstreet *
4291d16f16SKent Overstreet * six_lock_type(&foo->lock, SIX_LOCK_read);
4391d16f16SKent Overstreet * six_trylock_convert(&foo->lock, SIX_LOCK_read, SIX_LOCK_intent)
4491d16f16SKent Overstreet * six_lock_type(&foo->lock, SIX_LOCK_write);
4591d16f16SKent Overstreet * six_unlock_type(&foo->lock, SIX_LOCK_write);
4691d16f16SKent Overstreet * six_unlock_type(&foo->lock, SIX_LOCK_intent);
471c6fdbd8SKent Overstreet *
4891d16f16SKent Overstreet * Lock sequence numbers - unlock(), relock():
491c6fdbd8SKent Overstreet *
5091d16f16SKent Overstreet * Locks embed sequences numbers, which are incremented on write lock/unlock.
5191d16f16SKent Overstreet * This allows locks to be dropped and the retaken iff the state they protect
5291d16f16SKent Overstreet * hasn't changed; this makes it much easier to avoid holding locks while e.g.
5391d16f16SKent Overstreet * doing IO or allocating memory.
5491d16f16SKent Overstreet *
5591d16f16SKent Overstreet * Example usage:
5691d16f16SKent Overstreet * six_lock_read(&foo->lock);
5791d16f16SKent Overstreet * u32 seq = six_lock_seq(&foo->lock);
5891d16f16SKent Overstreet * six_unlock_read(&foo->lock);
5991d16f16SKent Overstreet *
6091d16f16SKent Overstreet * some_operation_that_may_block();
6191d16f16SKent Overstreet *
6291d16f16SKent Overstreet * if (six_relock_read(&foo->lock, seq)) { ... }
6391d16f16SKent Overstreet *
6491d16f16SKent Overstreet * If the relock operation succeeds, it is as if the lock was never unlocked.
6591d16f16SKent Overstreet *
6691d16f16SKent Overstreet * Reentrancy:
6791d16f16SKent Overstreet *
68*b56cee70SRandy Dunlap * Six locks are not by themselves reentrant, but have counters for both the
69*b56cee70SRandy Dunlap * read and intent states that can be used to provide reentrancy by an upper
7091d16f16SKent Overstreet * layer that tracks held locks. If a lock is known to already be held in the
7191d16f16SKent Overstreet * read or intent state, six_lock_increment() can be used to bump the "lock
7291d16f16SKent Overstreet * held in this state" counter, increasing the number of unlock calls that
7391d16f16SKent Overstreet * will be required to fully unlock it.
7491d16f16SKent Overstreet *
7591d16f16SKent Overstreet * Example usage:
7691d16f16SKent Overstreet * six_lock_read(&foo->lock);
7791d16f16SKent Overstreet * six_lock_increment(&foo->lock, SIX_LOCK_read);
7891d16f16SKent Overstreet * six_unlock_read(&foo->lock);
7991d16f16SKent Overstreet * six_unlock_read(&foo->lock);
8091d16f16SKent Overstreet * foo->lock is now fully unlocked.
8191d16f16SKent Overstreet *
8291d16f16SKent Overstreet * Since the intent state supercedes read, it's legal to increment the read
8391d16f16SKent Overstreet * counter when holding an intent lock, but not the reverse.
8491d16f16SKent Overstreet *
8591d16f16SKent Overstreet * A lock may only be held once for write: six_lock_increment(.., SIX_LOCK_write)
8691d16f16SKent Overstreet * is not legal.
8791d16f16SKent Overstreet *
8891d16f16SKent Overstreet * should_sleep_fn:
8991d16f16SKent Overstreet *
9091d16f16SKent Overstreet * There is a six_lock() variant that takes a function pointer that is called
9191d16f16SKent Overstreet * immediately prior to schedule() when blocking, and may return an error to
9291d16f16SKent Overstreet * abort.
9391d16f16SKent Overstreet *
9491d16f16SKent Overstreet * One possible use for this feature is when objects being locked are part of
9591d16f16SKent Overstreet * a cache and may reused, and lock ordering is based on a property of the
9691d16f16SKent Overstreet * object that will change when the object is reused - i.e. logical key order.
9791d16f16SKent Overstreet *
9891d16f16SKent Overstreet * If looking up an object in the cache may race with object reuse, and lock
9991d16f16SKent Overstreet * ordering is required to prevent deadlock, object reuse may change the
10091d16f16SKent Overstreet * correct lock order for that object and cause a deadlock. should_sleep_fn
10191d16f16SKent Overstreet * can be used to check if the object is still the object we want and avoid
10291d16f16SKent Overstreet * this deadlock.
10391d16f16SKent Overstreet *
10491d16f16SKent Overstreet * Wait list entry interface:
10591d16f16SKent Overstreet *
10691d16f16SKent Overstreet * There is a six_lock() variant, six_lock_waiter(), that takes a pointer to a
10791d16f16SKent Overstreet * wait list entry. By embedding six_lock_waiter into another object, and by
10891d16f16SKent Overstreet * traversing lock waitlists, it is then possible for an upper layer to
10991d16f16SKent Overstreet * implement full cycle detection for deadlock avoidance.
11091d16f16SKent Overstreet *
11191d16f16SKent Overstreet * should_sleep_fn should be used for invoking the cycle detector, walking the
11291d16f16SKent Overstreet * graph of held locks to check for a deadlock. The upper layer must track
11391d16f16SKent Overstreet * held locks for each thread, and each thread's held locks must be reachable
11491d16f16SKent Overstreet * from its six_lock_waiter object.
11591d16f16SKent Overstreet *
11691d16f16SKent Overstreet * six_lock_waiter() will add the wait object to the waitlist re-trying taking
11791d16f16SKent Overstreet * the lock, and before calling should_sleep_fn, and the wait object will not
11891d16f16SKent Overstreet * be removed from the waitlist until either the lock has been successfully
11991d16f16SKent Overstreet * acquired, or we aborted because should_sleep_fn returned an error.
12091d16f16SKent Overstreet *
12191d16f16SKent Overstreet * Also, six_lock_waiter contains a timestamp, and waiters on a waitlist will
12291d16f16SKent Overstreet * have timestamps in strictly ascending order - this is so the timestamp can
12391d16f16SKent Overstreet * be used as a cursor for lock graph traverse.
1241c6fdbd8SKent Overstreet */
1251c6fdbd8SKent Overstreet
1261c6fdbd8SKent Overstreet #include <linux/lockdep.h>
1271c6fdbd8SKent Overstreet #include <linux/sched.h>
1281c6fdbd8SKent Overstreet #include <linux/types.h>
1291c6fdbd8SKent Overstreet
1301c6fdbd8SKent Overstreet enum six_lock_type {
1311c6fdbd8SKent Overstreet SIX_LOCK_read,
1321c6fdbd8SKent Overstreet SIX_LOCK_intent,
1331c6fdbd8SKent Overstreet SIX_LOCK_write,
1341c6fdbd8SKent Overstreet };
1351c6fdbd8SKent Overstreet
1361c6fdbd8SKent Overstreet struct six_lock {
1372804d0f1SKent Overstreet atomic_t state;
1382804d0f1SKent Overstreet u32 seq;
1391c6fdbd8SKent Overstreet unsigned intent_lock_recurse;
1401c6fdbd8SKent Overstreet struct task_struct *owner;
14184a37cbfSKent Overstreet unsigned __percpu *readers;
1421c6fdbd8SKent Overstreet raw_spinlock_t wait_lock;
143ebc6f76aSKent Overstreet struct list_head wait_list;
1441c6fdbd8SKent Overstreet #ifdef CONFIG_DEBUG_LOCK_ALLOC
1451c6fdbd8SKent Overstreet struct lockdep_map dep_map;
1461c6fdbd8SKent Overstreet #endif
1471c6fdbd8SKent Overstreet };
1481c6fdbd8SKent Overstreet
149ebc6f76aSKent Overstreet struct six_lock_waiter {
150ebc6f76aSKent Overstreet struct list_head list;
151ebc6f76aSKent Overstreet struct task_struct *task;
152ebc6f76aSKent Overstreet enum six_lock_type lock_want;
15384a37cbfSKent Overstreet bool lock_acquired;
154f6ea2d57SKent Overstreet u64 start_time;
155ebc6f76aSKent Overstreet };
156ebc6f76aSKent Overstreet
1571c6fdbd8SKent Overstreet typedef int (*six_lock_should_sleep_fn)(struct six_lock *lock, void *);
1581c6fdbd8SKent Overstreet
1590d2234a7SKent Overstreet void six_lock_exit(struct six_lock *lock);
1601c6fdbd8SKent Overstreet
1610d2234a7SKent Overstreet enum six_lock_init_flags {
1620d2234a7SKent Overstreet SIX_LOCK_INIT_PCPU = 1U << 0,
1630d2234a7SKent Overstreet };
1640d2234a7SKent Overstreet
1650d2234a7SKent Overstreet void __six_lock_init(struct six_lock *lock, const char *name,
1660d2234a7SKent Overstreet struct lock_class_key *key, enum six_lock_init_flags flags);
1670d2234a7SKent Overstreet
16891d16f16SKent Overstreet /**
16991d16f16SKent Overstreet * six_lock_init - initialize a six lock
17091d16f16SKent Overstreet * @lock: lock to initialize
17191d16f16SKent Overstreet * @flags: optional flags, i.e. SIX_LOCK_INIT_PCPU
17291d16f16SKent Overstreet */
1730d2234a7SKent Overstreet #define six_lock_init(lock, flags) \
1741c6fdbd8SKent Overstreet do { \
1751c6fdbd8SKent Overstreet static struct lock_class_key __key; \
1761c6fdbd8SKent Overstreet \
1770d2234a7SKent Overstreet __six_lock_init((lock), #lock, &__key, flags); \
1781c6fdbd8SKent Overstreet } while (0)
1791c6fdbd8SKent Overstreet
18091d16f16SKent Overstreet /**
18191d16f16SKent Overstreet * six_lock_seq - obtain current lock sequence number
18291d16f16SKent Overstreet * @lock: six_lock to obtain sequence number for
18391d16f16SKent Overstreet *
18491d16f16SKent Overstreet * @lock should be held for read or intent, and not write
18591d16f16SKent Overstreet *
18691d16f16SKent Overstreet * By saving the lock sequence number, we can unlock @lock and then (typically
18791d16f16SKent Overstreet * after some blocking operation) attempt to relock it: the relock will succeed
18891d16f16SKent Overstreet * if the sequence number hasn't changed, meaning no write locks have been taken
18991d16f16SKent Overstreet * and state corresponding to what @lock protects is still valid.
19091d16f16SKent Overstreet */
six_lock_seq(const struct six_lock * lock)1911fb4fe63SKent Overstreet static inline u32 six_lock_seq(const struct six_lock *lock)
1921fb4fe63SKent Overstreet {
1932804d0f1SKent Overstreet return lock->seq;
1941fb4fe63SKent Overstreet }
1951fb4fe63SKent Overstreet
19691d16f16SKent Overstreet bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip);
197c4bd3491SKent Overstreet
19891d16f16SKent Overstreet /**
19991d16f16SKent Overstreet * six_trylock_type - attempt to take a six lock without blocking
20091d16f16SKent Overstreet * @lock: lock to take
20191d16f16SKent Overstreet * @type: SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
20291d16f16SKent Overstreet *
20391d16f16SKent Overstreet * Return: true on success, false on failure.
20491d16f16SKent Overstreet */
six_trylock_type(struct six_lock * lock,enum six_lock_type type)205c4bd3491SKent Overstreet static inline bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
206c4bd3491SKent Overstreet {
20791d16f16SKent Overstreet return six_trylock_ip(lock, type, _THIS_IP_);
208c4bd3491SKent Overstreet }
209c4bd3491SKent Overstreet
21091d16f16SKent Overstreet int six_lock_ip_waiter(struct six_lock *lock, enum six_lock_type type,
211c4bd3491SKent Overstreet struct six_lock_waiter *wait,
212c4bd3491SKent Overstreet six_lock_should_sleep_fn should_sleep_fn, void *p,
213c4bd3491SKent Overstreet unsigned long ip);
214c4bd3491SKent Overstreet
21591d16f16SKent Overstreet /**
21691d16f16SKent Overstreet * six_lock_waiter - take a lock, with full waitlist interface
21791d16f16SKent Overstreet * @lock: lock to take
21891d16f16SKent Overstreet * @type: SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
21991d16f16SKent Overstreet * @wait: pointer to wait object, which will be added to lock's waitlist
22091d16f16SKent Overstreet * @should_sleep_fn: callback run after adding to waitlist, immediately prior
22191d16f16SKent Overstreet * to scheduling
22291d16f16SKent Overstreet * @p: passed through to @should_sleep_fn
22391d16f16SKent Overstreet *
22491d16f16SKent Overstreet * This is a convenience wrapper around six_lock_ip_waiter(), see that function
22591d16f16SKent Overstreet * for full documentation.
22691d16f16SKent Overstreet *
22791d16f16SKent Overstreet * Return: 0 on success, or the return code from @should_sleep_fn on failure.
22891d16f16SKent Overstreet */
six_lock_waiter(struct six_lock * lock,enum six_lock_type type,struct six_lock_waiter * wait,six_lock_should_sleep_fn should_sleep_fn,void * p)22991d16f16SKent Overstreet static inline int six_lock_waiter(struct six_lock *lock, enum six_lock_type type,
230c4bd3491SKent Overstreet struct six_lock_waiter *wait,
231c4bd3491SKent Overstreet six_lock_should_sleep_fn should_sleep_fn, void *p)
232c4bd3491SKent Overstreet {
23391d16f16SKent Overstreet return six_lock_ip_waiter(lock, type, wait, should_sleep_fn, p, _THIS_IP_);
234c4bd3491SKent Overstreet }
235c4bd3491SKent Overstreet
23691d16f16SKent Overstreet /**
23791d16f16SKent Overstreet * six_lock_ip - take a six lock lock
23891d16f16SKent Overstreet * @lock: lock to take
23991d16f16SKent Overstreet * @type: SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
24091d16f16SKent Overstreet * @should_sleep_fn: callback run after adding to waitlist, immediately prior
24191d16f16SKent Overstreet * to scheduling
24291d16f16SKent Overstreet * @p: passed through to @should_sleep_fn
24391d16f16SKent Overstreet * @ip: ip parameter for lockdep/lockstat, i.e. _THIS_IP_
24491d16f16SKent Overstreet *
24591d16f16SKent Overstreet * Return: 0 on success, or the return code from @should_sleep_fn on failure.
24691d16f16SKent Overstreet */
six_lock_ip(struct six_lock * lock,enum six_lock_type type,six_lock_should_sleep_fn should_sleep_fn,void * p,unsigned long ip)24791d16f16SKent Overstreet static inline int six_lock_ip(struct six_lock *lock, enum six_lock_type type,
248c4bd3491SKent Overstreet six_lock_should_sleep_fn should_sleep_fn, void *p,
249c4bd3491SKent Overstreet unsigned long ip)
250c4bd3491SKent Overstreet {
251c4bd3491SKent Overstreet struct six_lock_waiter wait;
252c4bd3491SKent Overstreet
25391d16f16SKent Overstreet return six_lock_ip_waiter(lock, type, &wait, should_sleep_fn, p, ip);
254c4bd3491SKent Overstreet }
255c4bd3491SKent Overstreet
25691d16f16SKent Overstreet /**
25791d16f16SKent Overstreet * six_lock_type - take a six lock lock
25891d16f16SKent Overstreet * @lock: lock to take
25991d16f16SKent Overstreet * @type: SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
26091d16f16SKent Overstreet * @should_sleep_fn: callback run after adding to waitlist, immediately prior
26191d16f16SKent Overstreet * to scheduling
26291d16f16SKent Overstreet * @p: passed through to @should_sleep_fn
26391d16f16SKent Overstreet *
26491d16f16SKent Overstreet * Return: 0 on success, or the return code from @should_sleep_fn on failure.
26591d16f16SKent Overstreet */
six_lock_type(struct six_lock * lock,enum six_lock_type type,six_lock_should_sleep_fn should_sleep_fn,void * p)266c4bd3491SKent Overstreet static inline int six_lock_type(struct six_lock *lock, enum six_lock_type type,
267c4bd3491SKent Overstreet six_lock_should_sleep_fn should_sleep_fn, void *p)
268c4bd3491SKent Overstreet {
269c4bd3491SKent Overstreet struct six_lock_waiter wait;
270c4bd3491SKent Overstreet
27191d16f16SKent Overstreet return six_lock_ip_waiter(lock, type, &wait, should_sleep_fn, p, _THIS_IP_);
272c4bd3491SKent Overstreet }
273c4bd3491SKent Overstreet
27491d16f16SKent Overstreet bool six_relock_ip(struct six_lock *lock, enum six_lock_type type,
275c4bd3491SKent Overstreet unsigned seq, unsigned long ip);
276c4bd3491SKent Overstreet
27791d16f16SKent Overstreet /**
27891d16f16SKent Overstreet * six_relock_type - attempt to re-take a lock that was held previously
27991d16f16SKent Overstreet * @lock: lock to take
28091d16f16SKent Overstreet * @type: SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
28191d16f16SKent Overstreet * @seq: lock sequence number obtained from six_lock_seq() while lock was
28291d16f16SKent Overstreet * held previously
28391d16f16SKent Overstreet *
28491d16f16SKent Overstreet * Return: true on success, false on failure.
28591d16f16SKent Overstreet */
six_relock_type(struct six_lock * lock,enum six_lock_type type,unsigned seq)286c4bd3491SKent Overstreet static inline bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
287c4bd3491SKent Overstreet unsigned seq)
288c4bd3491SKent Overstreet {
28991d16f16SKent Overstreet return six_relock_ip(lock, type, seq, _THIS_IP_);
290c4bd3491SKent Overstreet }
291c4bd3491SKent Overstreet
29291d16f16SKent Overstreet void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip);
293c4bd3491SKent Overstreet
29491d16f16SKent Overstreet /**
29591d16f16SKent Overstreet * six_unlock_type - drop a six lock
29691d16f16SKent Overstreet * @lock: lock to unlock
29791d16f16SKent Overstreet * @type: SIX_LOCK_read, SIX_LOCK_intent, or SIX_LOCK_write
29891d16f16SKent Overstreet *
29991d16f16SKent Overstreet * When a lock is held multiple times (because six_lock_incement()) was used),
30091d16f16SKent Overstreet * this decrements the 'lock held' counter by one.
30191d16f16SKent Overstreet *
30291d16f16SKent Overstreet * For example:
30391d16f16SKent Overstreet * six_lock_read(&foo->lock); read count 1
30491d16f16SKent Overstreet * six_lock_increment(&foo->lock, SIX_LOCK_read); read count 2
30591d16f16SKent Overstreet * six_lock_unlock(&foo->lock, SIX_LOCK_read); read count 1
30691d16f16SKent Overstreet * six_lock_unlock(&foo->lock, SIX_LOCK_read); read count 0
30791d16f16SKent Overstreet */
six_unlock_type(struct six_lock * lock,enum six_lock_type type)308c4bd3491SKent Overstreet static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
309c4bd3491SKent Overstreet {
31091d16f16SKent Overstreet six_unlock_ip(lock, type, _THIS_IP_);
311c4bd3491SKent Overstreet }
312c4bd3491SKent Overstreet
3131c6fdbd8SKent Overstreet #define __SIX_LOCK(type) \
314c4bd3491SKent Overstreet static inline bool six_trylock_ip_##type(struct six_lock *lock, unsigned long ip)\
315c4bd3491SKent Overstreet { \
31691d16f16SKent Overstreet return six_trylock_ip(lock, SIX_LOCK_##type, ip); \
317c4bd3491SKent Overstreet } \
318f746c62cSKent Overstreet \
319f746c62cSKent Overstreet static inline bool six_trylock_##type(struct six_lock *lock) \
320f746c62cSKent Overstreet { \
32191d16f16SKent Overstreet return six_trylock_ip(lock, SIX_LOCK_##type, _THIS_IP_); \
322f746c62cSKent Overstreet } \
323c4bd3491SKent Overstreet \
324c4bd3491SKent Overstreet static inline int six_lock_ip_waiter_##type(struct six_lock *lock, \
325c4bd3491SKent Overstreet struct six_lock_waiter *wait, \
326c4bd3491SKent Overstreet six_lock_should_sleep_fn should_sleep_fn, void *p,\
327c4bd3491SKent Overstreet unsigned long ip) \
328c4bd3491SKent Overstreet { \
32991d16f16SKent Overstreet return six_lock_ip_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p, ip);\
330c4bd3491SKent Overstreet } \
331c4bd3491SKent Overstreet \
332c4bd3491SKent Overstreet static inline int six_lock_ip_##type(struct six_lock *lock, \
333c4bd3491SKent Overstreet six_lock_should_sleep_fn should_sleep_fn, void *p, \
334c4bd3491SKent Overstreet unsigned long ip) \
335c4bd3491SKent Overstreet { \
33691d16f16SKent Overstreet return six_lock_ip(lock, SIX_LOCK_##type, should_sleep_fn, p, ip);\
337c4bd3491SKent Overstreet } \
338c4bd3491SKent Overstreet \
339c4bd3491SKent Overstreet static inline bool six_relock_ip_##type(struct six_lock *lock, u32 seq, unsigned long ip)\
340c4bd3491SKent Overstreet { \
34191d16f16SKent Overstreet return six_relock_ip(lock, SIX_LOCK_##type, seq, ip); \
342c4bd3491SKent Overstreet } \
343c4bd3491SKent Overstreet \
344f746c62cSKent Overstreet static inline bool six_relock_##type(struct six_lock *lock, u32 seq) \
345f746c62cSKent Overstreet { \
34691d16f16SKent Overstreet return six_relock_ip(lock, SIX_LOCK_##type, seq, _THIS_IP_); \
347f746c62cSKent Overstreet } \
348c4bd3491SKent Overstreet \
349f746c62cSKent Overstreet static inline int six_lock_##type(struct six_lock *lock, \
350f746c62cSKent Overstreet six_lock_should_sleep_fn fn, void *p)\
351f746c62cSKent Overstreet { \
352f746c62cSKent Overstreet return six_lock_ip_##type(lock, fn, p, _THIS_IP_); \
353f746c62cSKent Overstreet } \
354c4bd3491SKent Overstreet \
355c4bd3491SKent Overstreet static inline void six_unlock_ip_##type(struct six_lock *lock, unsigned long ip) \
356c4bd3491SKent Overstreet { \
35791d16f16SKent Overstreet six_unlock_ip(lock, SIX_LOCK_##type, ip); \
358c4bd3491SKent Overstreet } \
359c4bd3491SKent Overstreet \
360f746c62cSKent Overstreet static inline void six_unlock_##type(struct six_lock *lock) \
361f746c62cSKent Overstreet { \
36291d16f16SKent Overstreet six_unlock_ip(lock, SIX_LOCK_##type, _THIS_IP_); \
363f746c62cSKent Overstreet }
3641c6fdbd8SKent Overstreet
3651c6fdbd8SKent Overstreet __SIX_LOCK(read)
3661c6fdbd8SKent Overstreet __SIX_LOCK(intent)
3671c6fdbd8SKent Overstreet __SIX_LOCK(write)
3681c6fdbd8SKent Overstreet #undef __SIX_LOCK
3691c6fdbd8SKent Overstreet
3701c6fdbd8SKent Overstreet void six_lock_downgrade(struct six_lock *);
3711c6fdbd8SKent Overstreet bool six_lock_tryupgrade(struct six_lock *);
3721c6fdbd8SKent Overstreet bool six_trylock_convert(struct six_lock *, enum six_lock_type,
3731c6fdbd8SKent Overstreet enum six_lock_type);
3741c6fdbd8SKent Overstreet
3751c6fdbd8SKent Overstreet void six_lock_increment(struct six_lock *, enum six_lock_type);
3761c6fdbd8SKent Overstreet
3771c6fdbd8SKent Overstreet void six_lock_wakeup_all(struct six_lock *);
3781c6fdbd8SKent Overstreet
3791c6fdbd8SKent Overstreet struct six_lock_count {
380e3738c69SKent Overstreet unsigned n[3];
3811c6fdbd8SKent Overstreet };
3821c6fdbd8SKent Overstreet
3831c6fdbd8SKent Overstreet struct six_lock_count six_lock_counts(struct six_lock *);
38401bf56a9SKent Overstreet void six_lock_readers_add(struct six_lock *, int);
3851c6fdbd8SKent Overstreet
3861c6fdbd8SKent Overstreet #endif /* _LINUX_SIX_H */
387