| /linux/include/linux/ |
| H A D | spinlock.h | 102 extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
105 # define raw_spin_lock_init(lock) \ argument
109 __raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \
113 # define raw_spin_lock_init(lock) \ argument
114 do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
117 #define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock) argument
120 #define raw_spin_is_contended(lock) arch_spin_is_contended(&(lock)->raw_lock) argument
122 #define raw_spin_is_contended(lock) (((void)(lock), 0)) argument
181 extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
182 extern int do_raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(true, lock);
[all …]
|
| H A D | local_lock_internal.h | 81 #define __local_lock_init(lock) \ argument
85 debug_check_no_locks_freed((void *)lock, sizeof(*lock));\
86 lockdep_init_map_type(&(lock)->dep_map, #lock, &__key, \
89 local_lock_debug_init(lock); \
92 #define __local_trylock_init(lock) \ argument
94 __local_lock_init((local_lock_t *)lock); \
97 #define __spinlock_nested_bh_init(lock) \ argument
101 debug_check_no_locks_freed((void *)lock, sizeof(*lock));\
102 lockdep_init_map_type(&(lock)->dep_map, #lock, &__key, \
105 local_lock_debug_init(lock); \
[all …]
|
| H A D | lockdep.h | 90 extern void lockdep_reset_lock(struct lockdep_map *lock);
128 extern void lockdep_init_map_type(struct lockdep_map *lock, const char *name,
132 lockdep_init_map_waits(struct lockdep_map *lock, const char *name, in lockdep_init_map_waits() argument
135 lockdep_init_map_type(lock, name, key, subclass, inner, outer, LD_LOCK_NORMAL); in lockdep_init_map_waits()
139 lockdep_init_map_wait(struct lockdep_map *lock, const char *name, in lockdep_init_map_wait() argument
142 lockdep_init_map_waits(lock, name, key, subclass, inner, LD_WAIT_INV); in lockdep_init_map_wait()
145 static inline void lockdep_init_map(struct lockdep_map *lock, const char *name, in lockdep_init_map() argument
148 lockdep_init_map_wait(lock, name, key, subclass, LD_WAIT_INV); in lockdep_init_map()
157 #define lockdep_set_class(lock, key) \ argument
158 lockdep_init_map_type(&(lock)->dep_map, #key, key, 0, \
[all …]
|
| /linux/kernel/locking/ |
| H A D | rtmutex_api.c | 40 static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, in __rt_mutex_lock_common() argument
44 __cond_acquires(0, lock) in __rt_mutex_lock_common()
49 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, _RET_IP_); in __rt_mutex_lock_common()
50 ret = __rt_mutex_lock(&lock->rtmutex, state); in __rt_mutex_lock_common()
52 mutex_release(&lock->dep_map, _RET_IP_); in __rt_mutex_lock_common()
69 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) in rt_mutex_lock_nested() argument
71 if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass) == 0) in rt_mutex_lock_nested()
78 __acquire(lock); in rt_mutex_lock_nested()
82 void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) in _rt_mutex_lock_nest_lock() argument
84 if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0) == 0) in _rt_mutex_lock_nest_lock()
[all …]
|
| H A D | qspinlock_paravirt.h | 81 static inline bool pv_hybrid_queued_unfair_trylock(struct qspinlock *lock) in pv_hybrid_queued_unfair_trylock() argument
88 int val = atomic_read(&lock->val); in pv_hybrid_queued_unfair_trylock()
92 try_cmpxchg_acquire(&lock->locked, &old, _Q_LOCKED_VAL)) { in pv_hybrid_queued_unfair_trylock()
110 static __always_inline void set_pending(struct qspinlock *lock) in set_pending() argument
112 WRITE_ONCE(lock->pending, 1); in set_pending()
120 static __always_inline bool trylock_clear_pending(struct qspinlock *lock) in trylock_clear_pending() argument
124 return !READ_ONCE(lock->locked) && in trylock_clear_pending()
125 try_cmpxchg_acquire(&lock->locked_pending, &old, _Q_LOCKED_VAL); in trylock_clear_pending()
128 static __always_inline void set_pending(struct qspinlock *lock) in set_pending() argument
130 atomic_or(_Q_PENDING_VAL, &lock->val); in set_pending()
[all …]
|
| H A D | qrwlock.c | 21 void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock) in queued_read_lock_slowpath() argument
33 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); in queued_read_lock_slowpath()
36 atomic_sub(_QR_BIAS, &lock->cnts); in queued_read_lock_slowpath()
38 trace_contention_begin(lock, LCB_F_SPIN | LCB_F_READ); in queued_read_lock_slowpath()
43 arch_spin_lock(&lock->wait_lock); in queued_read_lock_slowpath()
44 atomic_add(_QR_BIAS, &lock->cnts); in queued_read_lock_slowpath()
51 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); in queued_read_lock_slowpath()
56 arch_spin_unlock(&lock->wait_lock); in queued_read_lock_slowpath()
58 trace_contention_end(lock, 0); in queued_read_lock_slowpath()
66 void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock) in queued_write_lock_slowpath() argument
[all …]
|
| /linux/arch/alpha/include/asm/ |
| H A D | spinlock.h | 17 #define arch_spin_is_locked(x) ((x)->lock != 0)
19 static inline int arch_spin_value_unlocked(arch_spinlock_t lock) in arch_spin_value_unlocked() argument
21 return lock.lock == 0; in arch_spin_value_unlocked()
24 static inline void arch_spin_unlock(arch_spinlock_t * lock) in arch_spin_unlock() argument
27 lock->lock = 0; in arch_spin_unlock()
30 static inline void arch_spin_lock(arch_spinlock_t * lock) in arch_spin_lock() argument
46 : "=&r" (tmp), "=m" (lock->lock) in arch_spin_lock()
47 : "m"(lock->lock) : "memory"); in arch_spin_lock()
50 static inline int arch_spin_trylock(arch_spinlock_t *lock) in arch_spin_trylock() argument
52 return !test_and_set_bit(0, &lock->lock); in arch_spin_trylock()
[all …]
|
| /linux/drivers/md/persistent-data/ |
| H A D | dm-block-manager.c | 45 spinlock_t lock; member
61 static unsigned int __find_holder(struct block_lock *lock, in __find_holder() argument
67 if (lock->holders[i] == task) in __find_holder()
75 static void __add_holder(struct block_lock *lock, struct task_struct *task) in __add_holder() argument
77 unsigned int h = __find_holder(lock, NULL); in __add_holder()
83 lock->holders[h] = task; in __add_holder()
86 t = lock->traces + h; in __add_holder()
92 static void __del_holder(struct block_lock *lock, struct task_struct *task) in __del_holder() argument
94 unsigned int h = __find_holder(lock, task); in __del_holder()
96 lock->holders[h] = NULL; in __del_holder()
[all …]
|
| /linux/fs/ocfs2/dlm/ |
| H A D | dlmast.c | 35 struct dlm_lock *lock);
36 static int dlm_should_cancel_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock);
47 static int dlm_should_cancel_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock) in dlm_should_cancel_bast() argument
50 assert_spin_locked(&lock->spinlock); in dlm_should_cancel_bast()
52 if (lock->ml.highest_blocked == LKM_IVMODE) in dlm_should_cancel_bast()
54 BUG_ON(lock->ml.highest_blocked == LKM_NLMODE); in dlm_should_cancel_bast()
56 if (lock->bast_pending && in dlm_should_cancel_bast()
57 list_empty(&lock->bast_list)) in dlm_should_cancel_bast()
61 if (lock->ml.type == LKM_EXMODE) in dlm_should_cancel_bast()
64 else if (lock->ml.type == LKM_NLMODE) in dlm_should_cancel_bast()
[all …]
|
| H A D | dlmlock.c | 45 struct dlm_lock *lock, int flags);
49 static void dlm_lock_detach_lockres(struct dlm_lock *lock);
74 struct dlm_lock *lock) in dlm_can_grant_new_lock() argument
79 if (!dlm_lock_compatible(tmplock->ml.type, lock->ml.type)) in dlm_can_grant_new_lock()
84 if (!dlm_lock_compatible(tmplock->ml.type, lock->ml.type)) in dlm_can_grant_new_lock()
87 lock->ml.type)) in dlm_can_grant_new_lock()
103 struct dlm_lock *lock, int flags) in dlmlock_master() argument
108 mlog(0, "type=%d\n", lock->ml.type); in dlmlock_master()
115 lock->ml.node != dlm->node_num) { in dlmlock_master()
124 if (dlm_can_grant_new_lock(res, lock)) { in dlmlock_master()
[all …]
|
| H A D | dlmconvert.c | 43 struct dlm_lock *lock, int flags,
48 struct dlm_lock *lock, int flags, int type);
61 struct dlm_lock *lock, int flags, int type) in dlmconvert_master() argument
72 status = __dlmconvert_master(dlm, res, lock, flags, type, in dlmconvert_master()
83 dlm_queue_ast(dlm, lock); in dlmconvert_master()
104 struct dlm_lock *lock, int flags, in __dlmconvert_master() argument
114 lock->ml.type, lock->ml.convert_type, type); in __dlmconvert_master()
116 spin_lock(&lock->spinlock); in __dlmconvert_master()
119 if (lock->ml.convert_type != LKM_IVMODE) { in __dlmconvert_master()
127 if (!dlm_lock_on_list(&res->granted, lock)) { in __dlmconvert_master()
[all …]
|
| H A D | dlmunlock.c | 43 struct dlm_lock *lock,
48 struct dlm_lock *lock,
54 struct dlm_lock *lock,
83 struct dlm_lock *lock, in dlmunlock_common() argument
105 in_use = !list_empty(&lock->ast_list); in dlmunlock_common()
125 spin_lock(&lock->spinlock); in dlmunlock_common()
140 status = dlm_get_cancel_actions(dlm, res, lock, lksb, &actions); in dlmunlock_common()
142 status = dlm_get_unlock_actions(dlm, res, lock, lksb, &actions); in dlmunlock_common()
161 lock->cancel_pending = 1; in dlmunlock_common()
163 lock->unlock_pending = 1; in dlmunlock_common()
[all …]
|
| /linux/arch/hexagon/include/asm/ |
| H A D | spinlock.h | 28 static inline void arch_read_lock(arch_rwlock_t *lock) in arch_read_lock() argument
37 : "r" (&lock->lock) in arch_read_lock()
43 static inline void arch_read_unlock(arch_rwlock_t *lock) in arch_read_unlock() argument
51 : "r" (&lock->lock) in arch_read_unlock()
58 static inline int arch_read_trylock(arch_rwlock_t *lock) in arch_read_trylock() argument
69 : "r" (&lock->lock) in arch_read_trylock()
76 static inline void arch_write_lock(arch_rwlock_t *lock) in arch_write_lock() argument
85 : "r" (&lock->lock) in arch_write_lock()
91 static inline int arch_write_trylock(arch_rwlock_t *lock) in arch_write_trylock() argument
102 : "r" (&lock->lock) in arch_write_trylock()
[all …]
|
| /linux/arch/powerpc/include/asm/ |
| H A D | simple_spinlock.h | 35 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) in arch_spin_value_unlocked() argument
37 return lock.slock == 0; in arch_spin_value_unlocked()
40 static inline int arch_spin_is_locked(arch_spinlock_t *lock) in arch_spin_is_locked() argument
42 return !arch_spin_value_unlocked(READ_ONCE(*lock)); in arch_spin_is_locked()
49 static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) in __arch_spin_trylock() argument
64 : "r" (token), "r" (&lock->slock), [eh] "n" (eh) in __arch_spin_trylock()
70 static inline int arch_spin_trylock(arch_spinlock_t *lock) in arch_spin_trylock() argument
72 return __arch_spin_trylock(lock) == 0; in arch_spin_trylock()
91 void splpar_spin_yield(arch_spinlock_t *lock);
92 void splpar_rw_yield(arch_rwlock_t *lock);
[all …]
|
| /linux/tools/testing/selftests/bpf/progs/ |
| H A D | linked_list.c | 12 struct bpf_spin_lock lock; member
28 int list_push_pop(struct bpf_spin_lock *lock, struct bpf_list_head *head, bool leave_in_map) in list_push_pop() argument
37 bpf_spin_lock(lock); in list_push_pop()
39 bpf_spin_unlock(lock); in list_push_pop()
46 bpf_spin_lock(lock); in list_push_pop()
48 bpf_spin_unlock(lock); in list_push_pop()
56 bpf_spin_lock(lock); in list_push_pop()
59 bpf_spin_unlock(lock); in list_push_pop()
62 bpf_spin_lock(lock); in list_push_pop()
64 bpf_spin_unlock(lock); in list_push_pop()
109 list_push_pop_multiple(struct bpf_spin_lock * lock,struct bpf_list_head * head,bool leave_in_map) list_push_pop_multiple() argument
191 list_in_list(struct bpf_spin_lock * lock,struct bpf_list_head * head,bool leave_in_map) list_in_list() argument
261 test_list_push_pop(struct bpf_spin_lock * lock,struct bpf_list_head * head) test_list_push_pop() argument
272 test_list_push_pop_multiple(struct bpf_spin_lock * lock,struct bpf_list_head * head) test_list_push_pop_multiple() argument
283 test_list_in_list(struct bpf_spin_lock * lock,struct bpf_list_head * head) test_list_in_list() argument
[all...] |
| H A D | refcounted_kptr.c | 57 private(A) struct bpf_spin_lock lock; variable
91 struct bpf_spin_lock *lock) in __insert_in_tree_and_list() argument
103 bpf_spin_lock(lock); in __insert_in_tree_and_list()
106 bpf_spin_unlock(lock); in __insert_in_tree_and_list()
110 bpf_spin_unlock(lock); in __insert_in_tree_and_list()
112 bpf_spin_lock(lock); in __insert_in_tree_and_list()
115 bpf_spin_unlock(lock); in __insert_in_tree_and_list()
118 bpf_spin_unlock(lock); in __insert_in_tree_and_list()
123 struct bpf_spin_lock *lock) in __stash_map_insert_tree() argument
146 bpf_spin_lock(lock); in __stash_map_insert_tree()
[all …]
|
| /linux/arch/arm/include/asm/ |
| H A D | spinlock.h | 56 static inline void arch_spin_lock(arch_spinlock_t *lock) in arch_spin_lock() argument
62 prefetchw(&lock->slock); in arch_spin_lock()
70 : "r" (&lock->slock), "I" (1 << TICKET_SHIFT) in arch_spin_lock()
75 lockval.tickets.owner = READ_ONCE(lock->tickets.owner); in arch_spin_lock()
81 static inline int arch_spin_trylock(arch_spinlock_t *lock) in arch_spin_trylock() argument
86 prefetchw(&lock->slock); in arch_spin_trylock()
95 : "r" (&lock->slock), "I" (1 << TICKET_SHIFT) in arch_spin_trylock()
107 static inline void arch_spin_unlock(arch_spinlock_t *lock) in arch_spin_unlock() argument
110 lock->tickets.owner++; in arch_spin_unlock()
114 static inline int arch_spin_value_unlocked(arch_spinlock_t lock) in arch_spin_value_unlocked() argument
[all …]
|
| /linux/tools/perf/tests/shell/ |
| H A D | lock_contention.sh | 2 # kernel lock contention analysis test
12 # Workload to generate lock contention.
13 # Using 1 group (-g 1) keeps runtime low while generating sufficient lock events.
14 # We include -p (pipes) because socketpairs don't generate enough lock events on s390.
41 if ! perf list tracepoint | grep -q lock:contention_begin; then
42 echo "[Skip] No lock contention tracepoints"
49 echo "[Skip] Low number of CPUs (`nproc`), lock event cannot be triggered certainly"
57 echo "Testing perf lock record and perf lock contention"
58 perf lock recor
[all...] |
| /linux/tools/virtio/linux/ |
| H A D | spinlock.h | 8 static inline void spin_lock_init(spinlock_t *lock) in spin_lock_init() argument
10 int r = pthread_spin_init(lock, 0); in spin_lock_init()
14 static inline void spin_lock(spinlock_t *lock) in spin_lock() argument
16 int ret = pthread_spin_lock(lock); in spin_lock()
20 static inline void spin_unlock(spinlock_t *lock) in spin_unlock() argument
22 int ret = pthread_spin_unlock(lock); in spin_unlock()
26 static inline void spin_lock_bh(spinlock_t *lock) in spin_lock_bh() argument
28 spin_lock(lock); in spin_lock_bh()
31 static inline void spin_unlock_bh(spinlock_t *lock) in spin_unlock_bh() argument
33 spin_unlock(lock); in spin_unlock_bh()
[all …]
|
| /linux/include/asm-generic/ |
| H A D | qrwlock.h | 36 extern void queued_read_lock_slowpath(struct qrwlock *lock);
37 extern void queued_write_lock_slowpath(struct qrwlock *lock);
44 static inline int queued_read_trylock(struct qrwlock *lock) in queued_read_trylock() argument
48 cnts = atomic_read(&lock->cnts); in queued_read_trylock()
50 cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts); in queued_read_trylock()
53 atomic_sub(_QR_BIAS, &lock->cnts); in queued_read_trylock()
63 static inline int queued_write_trylock(struct qrwlock *lock) in queued_write_trylock() argument
67 cnts = atomic_read(&lock->cnts); in queued_write_trylock()
71 return likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, in queued_write_trylock()
78 static inline void queued_read_lock(struct qrwlock *lock) in queued_read_lock() argument
[all …]
|
| /linux/fs/btrfs/ |
| H A D | locking.h | 104 #define btrfs_might_wait_for_event(owner, lock) \ argument
106 rwsem_acquire(&owner->lock##_map, 0, 0, _THIS_IP_); \
107 rwsem_release(&owner->lock##_map, _THIS_IP_); \
121 #define btrfs_lockdep_acquire(owner, lock) \ argument
122 rwsem_acquire_read(&owner->lock##_map, 0, 0, _THIS_IP_)
128 #define btrfs_lockdep_release(owner, lock) \ argument
129 rwsem_release(&owner->lock##_map, _THIS_IP_)
135 #define btrfs_lockdep_inode_acquire(owner, lock) \ argument
136 rwsem_acquire_read(&owner->vfs_inode.lock.dep_map, 0, 0, _THIS_IP_)
138 #define btrfs_lockdep_inode_release(owner, lock) \ argument
[all …]
|
| /linux/kernel/bpf/ |
| H A D | rqspinlock.c | 92 static bool is_lock_released(rqspinlock_t *lock, u32 mask) in is_lock_released() argument
94 if (!(atomic_read_acquire(&lock->val) & (mask))) in is_lock_released()
99 static noinline int check_deadlock_AA(rqspinlock_t *lock) in check_deadlock_AA() argument
109 if (rqh->locks[i] == lock) in check_deadlock_AA()
120 static noinline int check_deadlock_ABBA(rqspinlock_t *lock, u32 mask) in check_deadlock_ABBA() argument
143 if (is_lock_released(lock, mask)) in check_deadlock_ABBA()
178 if (READ_ONCE(rqh_cpu->locks[i]) != lock) in check_deadlock_ABBA()
199 static noinline int check_timeout(rqspinlock_t *lock, u32 mask, in check_timeout() argument
206 if (check_deadlock_AA(lock)) in check_timeout()
223 return check_deadlock_ABBA(lock, mask); in check_timeout()
[all …]
|
| /linux/arch/sh/include/asm/ |
| H A D | spinlock-cas.h | 26 #define arch_spin_is_locked(x) ((x)->lock <= 0)
28 static inline void arch_spin_lock(arch_spinlock_t *lock) in arch_spin_lock() argument
30 while (!__sl_cas(&lock->lock, 1, 0)); in arch_spin_lock()
33 static inline void arch_spin_unlock(arch_spinlock_t *lock) in arch_spin_unlock() argument
35 __sl_cas(&lock->lock, 0, 1); in arch_spin_unlock()
38 static inline int arch_spin_trylock(arch_spinlock_t *lock) in arch_spin_trylock() argument
40 return __sl_cas(&lock->lock, 1, 0); in arch_spin_trylock()
55 do old = rw->lock; in arch_read_lock()
56 while (!old || __sl_cas(&rw->lock, old, old-1) != old); in arch_read_lock()
62 do old = rw->lock; in arch_read_unlock()
[all …]
|
| /linux/lib/ |
| H A D | atomic64.c | 28 arch_spinlock_t lock; member
32 .lock = __ARCH_SPIN_LOCK_UNLOCKED,
42 return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; in lock_addr()
48 arch_spinlock_t *lock = lock_addr(v); in generic_atomic64_read() local
52 arch_spin_lock(lock); in generic_atomic64_read()
54 arch_spin_unlock(lock); in generic_atomic64_read()
63 arch_spinlock_t *lock = lock_addr(v); in generic_atomic64_set() local
66 arch_spin_lock(lock); in generic_atomic64_set()
68 arch_spin_unlock(lock); in generic_atomic64_set()
77 arch_spinlock_t *lock = lock_addr(v); \
[all …]
|
| /linux/Documentation/locking/ |
| H A D | lockstat.rst | 14 Because things like lock contention can severely impact performance.
19 Lockdep already has hooks in the lock functions and maps lock instances to
20 lock classes. We build on that (see Documentation/locking/lockdep-design.rst).
21 The graph below shows the relation between the lock functions and the various
26 lock _____
44 lock, unlock - the regular lock functions
51 - number of lock contention that involved x-cpu data
53 - number of lock acquisitions that had to wait
56 - shortest (non-0) time we ever had to wait for a lock
58 - longest time we ever had to wait for a lock
[all …]
|