1 /* 2 * Queued read/write locks 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. 15 * 16 * Authors: Waiman Long <waiman.long@hp.com> 17 */ 18 #include <linux/smp.h> 19 #include <linux/bug.h> 20 #include <linux/cpumask.h> 21 #include <linux/percpu.h> 22 #include <linux/hardirq.h> 23 #include <linux/spinlock.h> 24 #include <asm/qrwlock.h> 25 26 /** 27 * queued_read_lock_slowpath - acquire read lock of a queue rwlock 28 * @lock: Pointer to queue rwlock structure 29 */ 30 void queued_read_lock_slowpath(struct qrwlock *lock) 31 { 32 /* 33 * Readers come here when they cannot get the lock without waiting 34 */ 35 if (unlikely(in_interrupt())) { 36 /* 37 * Readers in interrupt context will get the lock immediately 38 * if the writer is just waiting (not holding the lock yet), 39 * so spin with ACQUIRE semantics until the lock is available 40 * without waiting in the queue. 41 */ 42 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); 43 return; 44 } 45 atomic_sub(_QR_BIAS, &lock->cnts); 46 47 /* 48 * Put the reader into the wait queue 49 */ 50 arch_spin_lock(&lock->wait_lock); 51 atomic_add(_QR_BIAS, &lock->cnts); 52 53 /* 54 * The ACQUIRE semantics of the following spinning code ensure 55 * that accesses can't leak upwards out of our subsequent critical 56 * section in the case that the lock is currently held for write. 57 */ 58 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); 59 60 /* 61 * Signal the next one in queue to become queue head 62 */ 63 arch_spin_unlock(&lock->wait_lock); 64 } 65 EXPORT_SYMBOL(queued_read_lock_slowpath); 66 67 /** 68 * queued_write_lock_slowpath - acquire write lock of a queue rwlock 69 * @lock : Pointer to queue rwlock structure 70 */ 71 void queued_write_lock_slowpath(struct qrwlock *lock) 72 { 73 /* Put the writer into the wait queue */ 74 arch_spin_lock(&lock->wait_lock); 75 76 /* Try to acquire the lock directly if no reader is present */ 77 if (!atomic_read(&lock->cnts) && 78 (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)) 79 goto unlock; 80 81 /* Set the waiting flag to notify readers that a writer is pending */ 82 atomic_add(_QW_WAITING, &lock->cnts); 83 84 /* When no more readers or writers, set the locked flag */ 85 do { 86 atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING); 87 } while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING, 88 _QW_LOCKED) != _QW_WAITING); 89 unlock: 90 arch_spin_unlock(&lock->wait_lock); 91 } 92 EXPORT_SYMBOL(queued_write_lock_slowpath); 93