1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright IBM Corp. 1999, 2009 4 * 5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 6 */ 7 8 #ifndef __ASM_BARRIER_H 9 #define __ASM_BARRIER_H 10 11 #include <asm/alternative.h> 12 #include <asm/march.h> 13 14 /* 15 * Force strict CPU ordering. 16 * And yes, this is required on UP too when we're talking 17 * to devices. 18 */ 19 20 static __always_inline void bcr_serialize(void) 21 { 22 asm_inline volatile( 23 ALTERNATIVE("bcr 15,0", "bcr 14,0", ALT_FACILITY(45)) 24 : : : "memory"); 25 } 26 27 #define __mb() bcr_serialize() 28 #define __rmb() barrier() 29 #define __wmb() barrier() 30 #define __dma_rmb() __mb() 31 #define __dma_wmb() __mb() 32 #define __smp_mb() __mb() 33 #define __smp_rmb() __rmb() 34 #define __smp_wmb() __wmb() 35 36 #define __smp_store_release(p, v) \ 37 do { \ 38 compiletime_assert_atomic_type(*p); \ 39 barrier(); \ 40 WRITE_ONCE(*p, v); \ 41 } while (0) 42 43 #define __smp_load_acquire(p) \ 44 ({ \ 45 typeof(*p) ___p1 = READ_ONCE(*p); \ 46 compiletime_assert_atomic_type(*p); \ 47 barrier(); \ 48 ___p1; \ 49 }) 50 51 #define __smp_mb__before_atomic() barrier() 52 #define __smp_mb__after_atomic() barrier() 53 54 /** 55 * array_index_mask_nospec - generate a mask for array_idx() that is 56 * ~0UL when the bounds check succeeds and 0 otherwise 57 * @index: array element index 58 * @size: number of elements in array 59 */ 60 #define array_index_mask_nospec array_index_mask_nospec 61 static __always_inline unsigned long array_index_mask_nospec(unsigned long index, 62 unsigned long size) 63 { 64 unsigned long mask; 65 66 if (__builtin_constant_p(size) && size > 0) { 67 asm(" clgr %2,%1\n" 68 " slbgr %0,%0" 69 :"=d" (mask) : "d" (size-1), "d" (index) :"cc"); 70 return mask; 71 } 72 asm(" clgr %1,%2\n" 73 " slbgr %0,%0" 74 :"=d" (mask) : "d" (size), "d" (index) :"cc"); 75 return ~mask; 76 } 77 78 #include <asm-generic/barrier.h> 79 80 #endif /* __ASM_BARRIER_H */ 81