1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * A stand-alone ticket spinlock implementation for use by the non-VHE 4 * KVM hypervisor code running at EL2. 5 * 6 * Copyright (C) 2020 Google LLC 7 * Author: Will Deacon <will@kernel.org> 8 * 9 * Heavily based on the implementation removed by c11090474d70 which was: 10 * Copyright (C) 2012 ARM Ltd. 11 */ 12 13 #ifndef __ARM64_KVM_NVHE_SPINLOCK_H__ 14 #define __ARM64_KVM_NVHE_SPINLOCK_H__ 15 16 #include <asm/alternative.h> 17 #include <asm/lse.h> 18 #include <asm/rwonce.h> 19 20 #include <linux/cleanup.h> 21 22 typedef union hyp_spinlock { 23 u32 __val; 24 struct { 25 u16 owner, next; 26 }; 27 } hyp_spinlock_t; 28 29 #define __HYP_SPIN_LOCK_INITIALIZER \ 30 { .__val = 0 } 31 32 #define __HYP_SPIN_LOCK_UNLOCKED \ 33 ((hyp_spinlock_t) __HYP_SPIN_LOCK_INITIALIZER) 34 35 #define DEFINE_HYP_SPINLOCK(x) hyp_spinlock_t x = __HYP_SPIN_LOCK_UNLOCKED 36 37 #define hyp_spin_lock_init(l) \ 38 do { \ 39 *(l) = __HYP_SPIN_LOCK_UNLOCKED; \ 40 } while (0) 41 42 static inline void hyp_spin_lock(hyp_spinlock_t *lock) 43 { 44 u32 tmp; 45 hyp_spinlock_t lockval, newval; 46 47 asm volatile( 48 /* Atomically increment the next ticket. */ 49 ARM64_LSE_ATOMIC_INSN( 50 /* LL/SC */ 51 " prfm pstl1strm, %3\n" 52 "1: ldaxr %w0, %3\n" 53 " add %w1, %w0, #(1 << 16)\n" 54 " stxr %w2, %w1, %3\n" 55 " cbnz %w2, 1b\n", 56 /* LSE atomics */ 57 " mov %w2, #(1 << 16)\n" 58 " ldadda %w2, %w0, %3\n" 59 __nops(3)) 60 61 /* Did we get the lock? */ 62 " eor %w1, %w0, %w0, ror #16\n" 63 " cbz %w1, 3f\n" 64 /* 65 * No: spin on the owner. Send a local event to avoid missing an 66 * unlock before the exclusive load. 67 */ 68 " sevl\n" 69 "2: wfe\n" 70 " ldaxrh %w2, %4\n" 71 " eor %w1, %w2, %w0, lsr #16\n" 72 " cbnz %w1, 2b\n" 73 /* We got the lock. Critical section starts here. */ 74 "3:" 75 : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 76 : "Q" (lock->owner) 77 : "memory"); 78 } 79 80 static inline void hyp_spin_unlock(hyp_spinlock_t *lock) 81 { 82 u64 tmp; 83 84 asm volatile( 85 ARM64_LSE_ATOMIC_INSN( 86 /* LL/SC */ 87 " ldrh %w1, %0\n" 88 " add %w1, %w1, #1\n" 89 " stlrh %w1, %0", 90 /* LSE atomics */ 91 " mov %w1, #1\n" 92 " staddlh %w1, %0\n" 93 __nops(1)) 94 : "=Q" (lock->owner), "=&r" (tmp) 95 : 96 : "memory"); 97 } 98 99 static inline bool hyp_spin_is_locked(hyp_spinlock_t *lock) 100 { 101 hyp_spinlock_t lockval = READ_ONCE(*lock); 102 103 return lockval.owner != lockval.next; 104 } 105 106 #ifdef CONFIG_NVHE_EL2_DEBUG 107 static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) 108 { 109 /* 110 * The __pkvm_init() path accesses protected data-structures without 111 * holding locks as the other CPUs are guaranteed to not enter EL2 112 * concurrently at this point in time. The point by which EL2 is 113 * initialized on all CPUs is reflected in the pkvm static key, so 114 * wait until it is set before checking the lock state. 115 */ 116 if (static_branch_likely(&kvm_protected_mode_initialized)) 117 BUG_ON(!hyp_spin_is_locked(lock)); 118 } 119 #else 120 static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { } 121 #endif 122 123 DEFINE_LOCK_GUARD_1(hyp_spinlock, hyp_spinlock_t, hyp_spin_lock(_T->lock), 124 hyp_spin_unlock(_T->lock)) 125 #endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */ 126