Lines Matching full:lock
35 #include <trace/events/lock.h>
46 static void __mutex_init_generic(struct mutex *lock)
48 atomic_long_set(&lock->owner, 0);
49 raw_spin_lock_init(&lock->wait_lock);
50 INIT_LIST_HEAD(&lock->wait_list);
52 osq_lock_init(&lock->osq);
54 debug_mutex_init(lock);
62 bool mutex_is_locked(struct mutex *lock)
64 return __mutex_owner(lock) != NULL;
74 unsigned long mutex_get_owner(struct mutex *lock)
76 unsigned long owner = atomic_long_read(&lock->owner);
82 * Returns: __mutex_owner(lock) on failure or NULL on success.
84 static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)
88 owner = atomic_long_read(&lock->owner);
110 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {
123 static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff)
125 return !__mutex_trylock_common(lock, handoff);
131 static inline bool __mutex_trylock(struct mutex *lock)
133 return !__mutex_trylock_common(lock, false);
142 void mutex_init_generic(struct mutex *lock)
144 __mutex_init_generic(lock);
152 static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
157 MUTEX_WARN_ON(lock->magic != lock);
159 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
165 static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
169 return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL);
174 void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key)
176 __mutex_init_generic(lock);
179 * Make sure we are not reinitializing a held lock:
181 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
182 lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP);
187 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
189 atomic_long_or(flag, &lock->owner);
192 static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
194 atomic_long_andnot(flag, &lock->owner);
197 static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
199 return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
203 * Add @waiter to a given location in the lock wait_list and set the
207 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
210 hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX);
211 debug_mutex_add_waiter(lock, waiter, current);
214 if (__mutex_waiter_is_first(lock, waiter))
215 __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
219 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
222 if (likely(list_empty(&lock->wait_list)))
223 __mutex_clear_flag(lock, MUTEX_FLAGS);
225 debug_mutex_remove_waiter(lock, waiter, current);
235 static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
237 unsigned long owner = atomic_long_read(&lock->owner);
250 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))
257 * We split the mutex lock/unlock logic into separate fastpath and
262 static void __sched __mutex_lock_slowpath(struct mutex *lock);
266 * @lock: the mutex to be acquired
268 * Lock the mutex exclusively for this task. If the mutex is not
285 void __sched mutex_lock(struct mutex *lock)
289 if (!__mutex_trylock_fast(lock))
290 __mutex_lock_slowpath(lock);
302 static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)
304 return __mutex_trylock_common(lock, false);
308 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
313 ww = container_of(lock, struct ww_mutex, base);
332 * lock from a waiter with an earlier stamp, since the
333 * other thread may already own a lock that we also
336 if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS))
343 if (waiter && !__mutex_waiter_is_first(lock, waiter))
356 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,
363 while (__mutex_owner(lock) == owner) {
366 * checking lock->owner still matches owner. And we already
375 * Use vcpu_is_preempted to detect lock holder preemption issue.
382 if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {
396 static inline int mutex_can_spin_on_owner(struct mutex *lock)
411 owner = __mutex_owner(lock);
416 * If lock->owner is not set, the mutex has been released. Return true
426 * We try to spin for acquisition when we find that the lock owner
428 * need to reschedule. The rationale is that if the lock owner is
429 * running, it is likely to release the lock soon.
431 * The mutex spinners are queued up using MCS lock so that only one
433 * going to happen, there is no point in going through the lock/unlock
436 * Returns true when the lock was taken, otherwise false, indicating
440 * queue. The waiter-spinner will spin on the lock directly and concurrently
445 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
453 * is not going to take OSQ lock anyway, there is no need
456 if (!mutex_can_spin_on_owner(lock))
462 * MCS (queued) lock first before spinning on the owner field.
464 if (!osq_lock(&lock->osq))
472 owner = __mutex_trylock_or_owner(lock);
478 * release the lock or go to sleep.
480 if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))
493 osq_unlock(&lock->osq);
500 osq_unlock(&lock->osq);
505 * reschedule now, before we try-lock the mutex. This avoids getting
521 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
528 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
532 * @lock: the mutex to be released
546 void __sched mutex_unlock(struct mutex *lock)
549 if (__mutex_unlock_fast(lock))
552 __mutex_unlock_slowpath(lock, _RET_IP_);
558 * @lock: the mutex to be released
567 void __sched ww_mutex_unlock(struct ww_mutex *lock)
569 __ww_mutex_unlock(lock);
570 mutex_unlock(&lock->base);
575 * Lock a mutex (possibly interruptible), slowpath:
578 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,
593 MUTEX_WARN_ON(lock->magic != lock);
595 ww = container_of(lock, struct ww_mutex, base);
614 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
616 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
617 if (__mutex_trylock(lock) ||
618 mutex_optimistic_spin(lock, ww_ctx, NULL)) {
619 /* got the lock, yay! */
620 lock_acquired(&lock->dep_map, ip);
623 trace_contention_end(lock, 0);
628 raw_spin_lock_irqsave(&lock->wait_lock, flags);
632 if (__mutex_trylock(lock)) {
634 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q);
639 debug_mutex_lock_common(lock, &waiter);
644 lock_contended(&lock->dep_map, ip);
648 __mutex_add_waiter(lock, &waiter, &lock->wait_list);
654 ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx, &wake_q);
659 __set_task_blocked_on(current, lock);
661 trace_contention_begin(lock, LCB_F_MUTEX);
667 * mutex_unlock() handing the lock off to us, do a trylock
671 if (__mutex_trylock(lock))
676 * wait_lock. This ensures the lock cancellation is ordered
685 ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);
690 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
694 first = __mutex_waiter_is_first(lock, &waiter);
699 * it to the lock we are trying to acquire.
701 set_task_blocked_on(current, lock);
708 if (__mutex_trylock_or_handoff(lock, first))
712 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
718 clear_task_blocked_on(current, lock);
719 if (mutex_optimistic_spin(lock, ww_ctx, &waiter))
721 set_task_blocked_on(current, lock);
722 trace_contention_begin(lock, LCB_F_MUTEX);
725 raw_spin_lock_irqsave(&lock->wait_lock, flags);
727 raw_spin_lock_irqsave(&lock->wait_lock, flags);
729 __clear_task_blocked_on(current, lock);
734 * Wound-Wait; we stole the lock (!first_waiter), check the
738 !__mutex_waiter_is_first(lock, &waiter))
739 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q);
742 __mutex_remove_waiter(lock, &waiter);
747 /* got the lock - cleanup and rejoice! */
748 lock_acquired(&lock->dep_map, ip);
749 trace_contention_end(lock, 0);
754 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
759 __clear_task_blocked_on(current, lock);
761 __mutex_remove_waiter(lock, &waiter);
764 trace_contention_end(lock, ret);
765 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
767 mutex_release(&lock->dep_map, ip);
773 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
776 return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);
780 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
783 return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true);
788 * @ww: mutex to lock
826 mutex_lock_nested(struct mutex *lock, unsigned int subclass)
828 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
834 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
836 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
841 _mutex_lock_killable(struct mutex *lock, unsigned int subclass,
844 return __mutex_lock(lock, TASK_KILLABLE, subclass, nest, _RET_IP_);
849 mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
851 return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);
856 mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
863 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
870 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
884 ctx->contending_lock = lock;
886 ww_mutex_unlock(lock);
896 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
901 ret = __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE,
904 return ww_mutex_deadlock_injection(lock, ctx);
911 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
916 ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE,
920 return ww_mutex_deadlock_injection(lock, ctx);
929 * Release the lock, slowpath:
931 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
938 mutex_release(&lock->dep_map, ip);
941 * Release the lock before (potentially) taking the spinlock such that
947 owner = atomic_long_read(&lock->owner);
955 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
963 raw_spin_lock_irqsave(&lock->wait_lock, flags);
964 debug_mutex_unlock(lock);
965 if (!list_empty(&lock->wait_list)) {
968 list_first_entry(&lock->wait_list,
973 debug_mutex_wake_waiter(lock, waiter);
974 __clear_task_blocked_on(next, lock);
979 __mutex_handoff(lock, next);
981 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
990 __mutex_lock_killable_slowpath(struct mutex *lock);
993 __mutex_lock_interruptible_slowpath(struct mutex *lock);
997 * @lock: The mutex to be acquired.
999 * Lock the mutex like mutex_lock(). If a signal is delivered while the
1004 * Return: 0 if the lock was successfully acquired or %-EINTR if a
1007 int __sched mutex_lock_interruptible(struct mutex *lock)
1011 if (__mutex_trylock_fast(lock))
1014 return __mutex_lock_interruptible_slowpath(lock);
1021 * @lock: The mutex to be acquired.
1023 * Lock the mutex like mutex_lock(). If a signal which will be fatal to
1028 * Return: 0 if the lock was successfully acquired or %-EINTR if a
1031 int __sched mutex_lock_killable(struct mutex *lock)
1035 if (__mutex_trylock_fast(lock))
1038 return __mutex_lock_killable_slowpath(lock);
1044 * @lock: The mutex to be acquired.
1046 * Lock the mutex like mutex_lock(). While the task is waiting for this
1052 void __sched mutex_lock_io(struct mutex *lock)
1057 mutex_lock(lock);
1063 __mutex_lock_slowpath(struct mutex *lock)
1065 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
1069 __mutex_lock_killable_slowpath(struct mutex *lock)
1071 return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
1075 __mutex_lock_interruptible_slowpath(struct mutex *lock)
1077 return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
1081 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1083 return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0,
1088 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
1091 return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0,
1100 * @lock: the mutex to be acquired
1112 int __sched mutex_trylock(struct mutex *lock)
1114 MUTEX_WARN_ON(lock->magic != lock);
1115 return __mutex_trylock(lock);
1119 int __sched _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock)
1123 MUTEX_WARN_ON(lock->magic != lock);
1124 locked = __mutex_trylock(lock);
1126 mutex_acquire_nest(&lock->dep_map, 0, 1, nest_lock, _RET_IP_);
1135 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1139 if (__mutex_trylock_fast(&lock->base)) {
1141 ww_mutex_set_context_fastpath(lock, ctx);
1145 return __ww_mutex_lock_slowpath(lock, ctx);
1150 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1154 if (__mutex_trylock_fast(&lock->base)) {
1156 ww_mutex_set_context_fastpath(lock, ctx);
1160 return __ww_mutex_lock_interruptible_slowpath(lock, ctx);
1173 * @lock: the mutex to return holding if we dec to 0
1175 * return true and hold lock if we dec to 0, return false otherwise
1177 int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
1182 /* we might hit 0, so take the lock */
1183 mutex_lock(lock);
1186 mutex_unlock(lock);
1189 /* we hit 0, and we hold the lock */