Lines Matching full:lock

25 #include <trace/events/lock.h>
36  * MCS lock. A copy of the original MCS lock paper ("Algorithms for Scalable
42  * This queued spinlock implementation is based on the MCS lock, however to
46  * In particular; where the traditional MCS lock consists of a tail pointer
55  * number. With one byte for the lock value and 3 bytes for the tail, only a
56  * 32-bit word is now needed. Even though we only need 1 bit for the lock,
60  * We also change the first spinner to spin on the lock bit instead of its
61  * node; whereby avoiding the need to carry a node from lock to unlock, and
62  * preserving existing lock API. This also makes the unlock code simpler and
90 static __always_inline void __pv_kick_node(struct qspinlock *lock,
92 static __always_inline u32  __pv_wait_head_or_lock(struct qspinlock *lock,
116  * tracepoint-disabled hot path never has to keep lock live across a call
120 void __lockfunc queued_spin_release_traced(struct qspinlock *lock)
122 	if (queued_spin_is_contended(lock))
123 		trace_call__contended_release(lock);
124 	queued_spin_release(lock);
133  * @lock: Pointer to queued spinlock structure
136  * (queue tail, pending bit, lock value)
152 void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
163 	if (virt_spin_lock(lock))
174 		val = atomic_cond_read_relaxed(&lock->val,
189 	val = queued_fetch_set_pending_acquire(lock);
202 			clear_pending(lock);
213 	 * store-release that clears the locked bit and create lock
219 		smp_cond_load_acquire(&lock->locked, !VAL);
226 	clear_pending_set_locked(lock);
241 	trace_contention_begin(lock, LCB_F_SPIN);
248 	 * we fall back to spinning on the lock directly without using
254 		while (!queued_spin_trylock(lock))
282 	if (queued_spin_trylock(lock))
299 	old = xchg_tail(lock, tail);
316 		 * While waiting for the MCS lock, the next pointer may have
317 		 * been set by another lock waiter. We optimistically load
333 	 * store-release that clears the locked bit and create lock
338 	 * the lock and return a non-zero value. So we have to skip the
347 	if ((val = pv_wait_head_or_lock(lock, node)))
350 	val = atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK));
354 	 * claim the lock:
356 	 * n,0,0 -> 0,0,1 : lock, uncontended
357 	 * *,*,0 -> *,*,1 : lock, contended
359 	 * If the queue head is the only one in the queue (lock value == tail)
360 	 * and nobody is pending, clear the tail code and grab the lock.
361 	 * Otherwise, we only need to grab the lock.
366 	 * of lock stealing; therefore we must also allow:
375 		if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
384 	set_locked(lock);
393 	pv_kick_node(lock, next);
396 	trace_contention_end(lock, 0);