1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * A stand-alone ticket spinlock implementation for use by the non-VHE 4 * KVM hypervisor code running at EL2. 5 * 6 * Copyright (C) 2020 Google LLC 7 * Author: Will Deacon <will@kernel.org> 8 * 9 * Heavily based on the implementation removed by c11090474d70 which was: 10 * Copyright (C) 2012 ARM Ltd. 11 */ 12 13 #ifndef __ARM64_KVM_NVHE_SPINLOCK_H__ 14 #define __ARM64_KVM_NVHE_SPINLOCK_H__ 15 16 #include <asm/alternative.h> 17 #include <asm/lse.h> 18 #include <asm/rwonce.h> 19 20 typedef union hyp_spinlock { 21 u32 __val; 22 struct { 23 #ifdef __AARCH64EB__ 24 u16 next, owner; 25 #else 26 u16 owner, next; 27 #endif 28 }; 29 } hyp_spinlock_t; 30 31 #define __HYP_SPIN_LOCK_INITIALIZER \ 32 { .__val = 0 } 33 34 #define __HYP_SPIN_LOCK_UNLOCKED \ 35 ((hyp_spinlock_t) __HYP_SPIN_LOCK_INITIALIZER) 36 37 #define DEFINE_HYP_SPINLOCK(x) hyp_spinlock_t x = __HYP_SPIN_LOCK_UNLOCKED 38 39 #define hyp_spin_lock_init(l) \ 40 do { \ 41 *(l) = __HYP_SPIN_LOCK_UNLOCKED; \ 42 } while (0) 43 44 static inline void hyp_spin_lock(hyp_spinlock_t *lock) 45 { 46 u32 tmp; 47 hyp_spinlock_t lockval, newval; 48 49 asm volatile( 50 /* Atomically increment the next ticket. */ 51 ARM64_LSE_ATOMIC_INSN( 52 /* LL/SC */ 53 " prfm pstl1strm, %3\n" 54 "1: ldaxr %w0, %3\n" 55 " add %w1, %w0, #(1 << 16)\n" 56 " stxr %w2, %w1, %3\n" 57 " cbnz %w2, 1b\n", 58 /* LSE atomics */ 59 " mov %w2, #(1 << 16)\n" 60 " ldadda %w2, %w0, %3\n" 61 __nops(3)) 62 63 /* Did we get the lock? */ 64 " eor %w1, %w0, %w0, ror #16\n" 65 " cbz %w1, 3f\n" 66 /* 67 * No: spin on the owner. Send a local event to avoid missing an 68 * unlock before the exclusive load. 69 */ 70 " sevl\n" 71 "2: wfe\n" 72 " ldaxrh %w2, %4\n" 73 " eor %w1, %w2, %w0, lsr #16\n" 74 " cbnz %w1, 2b\n" 75 /* We got the lock. Critical section starts here. */ 76 "3:" 77 : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 78 : "Q" (lock->owner) 79 : "memory"); 80 } 81 82 static inline void hyp_spin_unlock(hyp_spinlock_t *lock) 83 { 84 u64 tmp; 85 86 asm volatile( 87 ARM64_LSE_ATOMIC_INSN( 88 /* LL/SC */ 89 " ldrh %w1, %0\n" 90 " add %w1, %w1, #1\n" 91 " stlrh %w1, %0", 92 /* LSE atomics */ 93 " mov %w1, #1\n" 94 " staddlh %w1, %0\n" 95 __nops(1)) 96 : "=Q" (lock->owner), "=&r" (tmp) 97 : 98 : "memory"); 99 } 100 101 static inline bool hyp_spin_is_locked(hyp_spinlock_t *lock) 102 { 103 hyp_spinlock_t lockval = READ_ONCE(*lock); 104 105 return lockval.owner != lockval.next; 106 } 107 108 #ifdef CONFIG_NVHE_EL2_DEBUG 109 static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) 110 { 111 /* 112 * The __pkvm_init() path accesses protected data-structures without 113 * holding locks as the other CPUs are guaranteed to not enter EL2 114 * concurrently at this point in time. The point by which EL2 is 115 * initialized on all CPUs is reflected in the pkvm static key, so 116 * wait until it is set before checking the lock state. 117 */ 118 if (static_branch_likely(&kvm_protected_mode_initialized)) 119 BUG_ON(!hyp_spin_is_locked(lock)); 120 } 121 #else 122 static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { } 123 #endif 124 125 #endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */ 126