Lines Matching refs:lock
29 #include <trace/events/lock.h>
46 * MCS lock. A copy of the original MCS lock paper ("Algorithms for Scalable
52 * This queued spinlock implementation is based on the MCS lock, however to
56 * In particular; where the traditional MCS lock consists of a tail pointer
65 * number. With one byte for the lock value and 3 bytes for the tail, only a
66 * 32-bit word is now needed. Even though we only need 1 bit for the lock,
70 * We also change the first spinner to spin on the lock bit instead of its
71 * node; whereby avoiding the need to carry a node from lock to unlock, and
72 * preserving existing lock API. This also makes the unlock code simpler and
92 static bool is_lock_released(rqspinlock_t *lock, u32 mask)
94 if (!(atomic_read_acquire(&lock->val) & (mask)))
99 static noinline int check_deadlock_AA(rqspinlock_t *lock)
105 * Return an error if we hold the lock we are attempting to acquire.
109 if (rqh->locks[i] == lock)
120 static noinline int check_deadlock_ABBA(rqspinlock_t *lock, u32 mask)
128 * Find the CPU holding the lock that we want to acquire. If there is a
131 * O(NR_CPUS) if we could determine the owning CPU from a lock value, but
132 * that requires increasing the size of the lock word.
140 * Let's ensure to break out of this loop if the lock is available for
143 if (is_lock_released(lock, mask))
148 * least one held lock and one acquisition attempt (reflected as top
151 * If cnt is more than RES_NR_HELD, it means the current lock being
159 * Obtain the entry at the top, this corresponds to the lock the
171 * Find if the lock we're attempting to acquire is held by this CPU.
172 * Don't consider the topmost entry, as that must be the latest lock
174 * attempt to acquire a lock we hold, so for this search only 'cnt - 1'
178 if (READ_ONCE(rqh_cpu->locks[i]) != lock)
181 * We found our lock as held on the remote CPU. Is the
182 * acquisition attempt on the remote CPU for a lock held
199 static noinline int check_timeout(rqspinlock_t *lock, u32 mask,
206 if (check_deadlock_AA(lock))
223 return check_deadlock_ABBA(lock, mask);
237 (ret) = check_timeout((lock), (mask), &(ts)); \
242 ({ (ret) = check_timeout((lock), (mask), &(ts)); })
258 * Provide a test-and-set fallback for cases when queued spin lock support is
261 int __lockfunc resilient_tas_spin_lock(rqspinlock_t *lock)
281 val = atomic_read(&lock->val);
283 if (val || !atomic_try_cmpxchg(&lock->val, &val, 1)) {
315 * @lock: Pointer to queued spinlock structure
323 * (queue tail, pending bit, lock value)
339 int __lockfunc resilient_queued_spin_lock_slowpath(rqspinlock_t *lock, u32 val)
349 return resilient_virt_spin_lock(lock);
361 val = atomic_cond_read_relaxed(&lock->val,
376 val = queued_fetch_set_pending_acquire(lock);
389 clear_pending(lock);
402 * store-release that clears the locked bit and create lock
409 res_smp_cond_load_acquire(&lock->locked, !VAL || RES_CHECK_TIMEOUT(ts, ret, _Q_LOCKED_MASK));
416 * we own it. Once a stuck owner has been recovered, the lock
422 clear_pending(lock);
432 clear_pending_set_locked(lock);
442 * Do not queue if we're a waiter and someone is attempting this lock on
449 if (check_deadlock_AA(lock)) {
460 trace_contention_begin(lock, LCB_F_SPIN);
475 if (!queued_spin_trylock(lock)) {
504 if (queued_spin_trylock(lock))
521 old = xchg_tail(lock, tail);
543 * While waiting for the MCS lock, the next pointer may have
544 * been set by another lock waiter. We optimistically load
560 * store-release that clears the locked bit and create lock
566 * have both the owner of the lock and the pending bit waiter ahead of
570 val = res_atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK) ||
595 * will queue behind us. This will leave the lock owner in
607 if (!try_cmpxchg_tail(lock, tail, 0)) {
616 * claim the lock:
618 * n,0,0 -> 0,0,1 : lock, uncontended
619 * *,*,0 -> *,*,1 : lock, contended
621 * If the queue head is the only one in the queue (lock value == tail)
622 * and nobody is pending, clear the tail code and grab the lock.
623 * Otherwise, we only need to grab the lock.
632 if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
641 set_locked(lock);
652 trace_contention_end(lock, 0);
660 trace_contention_end(lock, ret);
672 static void bpf_prog_report_rqspinlock_violation(const char *str, void *lock, bool irqsave)
683 bpf_stream_printk(ss, "Attempted lock = 0x%px\n", lock);
686 bpf_stream_printk(ss, "Held lock[%2d] = 0x%px\n", i, rqh->locks[i]);
693 __bpf_kfunc int bpf_res_spin_lock(struct bpf_res_spin_lock *lock)
700 ret = res_spin_lock((rqspinlock_t *)lock);
702 bpf_prog_report_rqspinlock_violation(REPORT_STR(ret), lock, false);
709 __bpf_kfunc void bpf_res_spin_unlock(struct bpf_res_spin_lock *lock)
711 res_spin_unlock((rqspinlock_t *)lock);
715 __bpf_kfunc int bpf_res_spin_lock_irqsave(struct bpf_res_spin_lock *lock, unsigned long *flags__irq_flag)
723 ret = res_spin_lock((rqspinlock_t *)lock);
725 bpf_prog_report_rqspinlock_violation(REPORT_STR(ret), lock, true);
734 __bpf_kfunc void bpf_res_spin_unlock_irqrestore(struct bpf_res_spin_lock *lock, unsigned long *flags__irq_flag)
739 res_spin_unlock((rqspinlock_t *)lock);