1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #ifndef __ASM_BARRIER_H 6 #define __ASM_BARRIER_H 7 8 #define __sync() __asm__ __volatile__("dbar 0" : : : "memory") 9 10 #define fast_wmb() __sync() 11 #define fast_rmb() __sync() 12 #define fast_mb() __sync() 13 #define fast_iob() __sync() 14 #define wbflush() __sync() 15 16 #define wmb() fast_wmb() 17 #define rmb() fast_rmb() 18 #define mb() fast_mb() 19 #define iob() fast_iob() 20 21 #define __smp_mb() __asm__ __volatile__("dbar 0" : : : "memory") 22 #define __smp_rmb() __asm__ __volatile__("dbar 0" : : : "memory") 23 #define __smp_wmb() __asm__ __volatile__("dbar 0" : : : "memory") 24 25 #ifdef CONFIG_SMP 26 #define __WEAK_LLSC_MB " dbar 0 \n" 27 #else 28 #define __WEAK_LLSC_MB " \n" 29 #endif 30 31 #define __smp_mb__before_atomic() barrier() 32 #define __smp_mb__after_atomic() barrier() 33 34 /** 35 * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise 36 * @index: array element index 37 * @size: number of elements in array 38 * 39 * Returns: 40 * 0 - (@index < @size) 41 */ 42 #define array_index_mask_nospec array_index_mask_nospec 43 static inline unsigned long array_index_mask_nospec(unsigned long index, 44 unsigned long size) 45 { 46 unsigned long mask; 47 48 __asm__ __volatile__( 49 "sltu %0, %1, %2\n\t" 50 #if (__SIZEOF_LONG__ == 4) 51 "sub.w %0, $zero, %0\n\t" 52 #elif (__SIZEOF_LONG__ == 8) 53 "sub.d %0, $zero, %0\n\t" 54 #endif 55 : "=r" (mask) 56 : "r" (index), "r" (size) 57 :); 58 59 return mask; 60 } 61 62 #define __smp_load_acquire(p) \ 63 ({ \ 64 union { typeof(*p) __val; char __c[1]; } __u; \ 65 unsigned long __tmp = 0; \ 66 compiletime_assert_atomic_type(*p); \ 67 switch (sizeof(*p)) { \ 68 case 1: \ 69 *(__u8 *)__u.__c = *(volatile __u8 *)p; \ 70 __smp_mb(); \ 71 break; \ 72 case 2: \ 73 *(__u16 *)__u.__c = *(volatile __u16 *)p; \ 74 __smp_mb(); \ 75 break; \ 76 case 4: \ 77 __asm__ __volatile__( \ 78 "amor_db.w %[val], %[tmp], %[mem] \n" \ 79 : [val] "=&r" (*(__u32 *)__u.__c) \ 80 : [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp) \ 81 : "memory"); \ 82 break; \ 83 case 8: \ 84 __asm__ __volatile__( \ 85 "amor_db.d %[val], %[tmp], %[mem] \n" \ 86 : [val] "=&r" (*(__u64 *)__u.__c) \ 87 : [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp) \ 88 : "memory"); \ 89 break; \ 90 } \ 91 (typeof(*p))__u.__val; \ 92 }) 93 94 #define __smp_store_release(p, v) \ 95 do { \ 96 union { typeof(*p) __val; char __c[1]; } __u = \ 97 { .__val = (__force typeof(*p)) (v) }; \ 98 unsigned long __tmp; \ 99 compiletime_assert_atomic_type(*p); \ 100 switch (sizeof(*p)) { \ 101 case 1: \ 102 __smp_mb(); \ 103 *(volatile __u8 *)p = *(__u8 *)__u.__c; \ 104 break; \ 105 case 2: \ 106 __smp_mb(); \ 107 *(volatile __u16 *)p = *(__u16 *)__u.__c; \ 108 break; \ 109 case 4: \ 110 __asm__ __volatile__( \ 111 "amswap_db.w %[tmp], %[val], %[mem] \n" \ 112 : [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp) \ 113 : [val] "r" (*(__u32 *)__u.__c) \ 114 : ); \ 115 break; \ 116 case 8: \ 117 __asm__ __volatile__( \ 118 "amswap_db.d %[tmp], %[val], %[mem] \n" \ 119 : [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp) \ 120 : [val] "r" (*(__u64 *)__u.__c) \ 121 : ); \ 122 break; \ 123 } \ 124 } while (0) 125 126 #define __smp_store_mb(p, v) \ 127 do { \ 128 union { typeof(p) __val; char __c[1]; } __u = \ 129 { .__val = (__force typeof(p)) (v) }; \ 130 unsigned long __tmp; \ 131 switch (sizeof(p)) { \ 132 case 1: \ 133 *(volatile __u8 *)&p = *(__u8 *)__u.__c; \ 134 __smp_mb(); \ 135 break; \ 136 case 2: \ 137 *(volatile __u16 *)&p = *(__u16 *)__u.__c; \ 138 __smp_mb(); \ 139 break; \ 140 case 4: \ 141 __asm__ __volatile__( \ 142 "amswap_db.w %[tmp], %[val], %[mem] \n" \ 143 : [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp) \ 144 : [val] "r" (*(__u32 *)__u.__c) \ 145 : ); \ 146 break; \ 147 case 8: \ 148 __asm__ __volatile__( \ 149 "amswap_db.d %[tmp], %[val], %[mem] \n" \ 150 : [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp) \ 151 : [val] "r" (*(__u64 *)__u.__c) \ 152 : ); \ 153 break; \ 154 } \ 155 } while (0) 156 157 #include <asm-generic/barrier.h> 158 159 #endif /* __ASM_BARRIER_H */ 160