1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARCH_S390_PERCPU__ 3 #define __ARCH_S390_PERCPU__ 4 5 #include <linux/preempt.h> 6 #include <asm/cmpxchg.h> 7 #include <asm/march.h> 8 9 /* 10 * s390 uses its own implementation for per cpu data, the offset of 11 * the cpu local data area is cached in the cpu's lowcore memory. 12 */ 13 #define __my_cpu_offset get_lowcore()->percpu_offset 14 15 /* 16 * For 64 bit module code, the module may be more than 4G above the 17 * per cpu area, use weak definitions to force the compiler to 18 * generate external references. 19 */ 20 #if defined(MODULE) 21 #define ARCH_NEEDS_WEAK_PER_CPU 22 #endif 23 24 /* 25 * We use a compare-and-swap loop since that uses less cpu cycles than 26 * disabling and enabling interrupts like the generic variant would do. 27 */ 28 #define arch_this_cpu_to_op_simple(pcp, val, op) \ 29 ({ \ 30 typedef typeof(pcp) pcp_op_T__; \ 31 pcp_op_T__ old__, new__, prev__; \ 32 pcp_op_T__ *ptr__; \ 33 preempt_disable_notrace(); \ 34 ptr__ = raw_cpu_ptr(&(pcp)); \ 35 prev__ = READ_ONCE(*ptr__); \ 36 do { \ 37 old__ = prev__; \ 38 new__ = old__ op (val); \ 39 prev__ = cmpxchg(ptr__, old__, new__); \ 40 } while (prev__ != old__); \ 41 preempt_enable_notrace(); \ 42 new__; \ 43 }) 44 45 #define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 46 #define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 47 #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 48 #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 49 #define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 50 #define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 51 #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 52 #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 53 54 #ifndef MARCH_HAS_Z196_FEATURES 55 56 #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 57 #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 58 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 59 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 60 #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 61 #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 62 #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 63 #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 64 65 #else /* MARCH_HAS_Z196_FEATURES */ 66 67 #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ 68 { \ 69 typedef typeof(pcp) pcp_op_T__; \ 70 pcp_op_T__ val__ = (val); \ 71 pcp_op_T__ old__, *ptr__; \ 72 preempt_disable_notrace(); \ 73 ptr__ = raw_cpu_ptr(&(pcp)); \ 74 if (__builtin_constant_p(val__) && \ 75 ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ 76 asm volatile( \ 77 op2 " %[ptr__],%[val__]\n" \ 78 : [ptr__] "+Q" (*ptr__) \ 79 : [val__] "i" ((szcast)val__) \ 80 : "cc"); \ 81 } else { \ 82 asm volatile( \ 83 op1 " %[old__],%[val__],%[ptr__]\n" \ 84 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 85 : [val__] "d" (val__) \ 86 : "cc"); \ 87 } \ 88 preempt_enable_notrace(); \ 89 } 90 91 #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) 92 #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) 93 94 #define arch_this_cpu_add_return(pcp, val, op) \ 95 ({ \ 96 typedef typeof(pcp) pcp_op_T__; \ 97 pcp_op_T__ val__ = (val); \ 98 pcp_op_T__ old__, *ptr__; \ 99 preempt_disable_notrace(); \ 100 ptr__ = raw_cpu_ptr(&(pcp)); \ 101 asm volatile( \ 102 op " %[old__],%[val__],%[ptr__]\n" \ 103 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 104 : [val__] "d" (val__) \ 105 : "cc"); \ 106 preempt_enable_notrace(); \ 107 old__ + val__; \ 108 }) 109 110 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa") 111 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") 112 113 #define arch_this_cpu_to_op(pcp, val, op) \ 114 { \ 115 typedef typeof(pcp) pcp_op_T__; \ 116 pcp_op_T__ val__ = (val); \ 117 pcp_op_T__ old__, *ptr__; \ 118 preempt_disable_notrace(); \ 119 ptr__ = raw_cpu_ptr(&(pcp)); \ 120 asm volatile( \ 121 op " %[old__],%[val__],%[ptr__]\n" \ 122 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 123 : [val__] "d" (val__) \ 124 : "cc"); \ 125 preempt_enable_notrace(); \ 126 } 127 128 #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") 129 #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") 130 #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") 131 #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") 132 133 #endif /* MARCH_HAS_Z196_FEATURES */ 134 135 #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ 136 ({ \ 137 typedef typeof(pcp) pcp_op_T__; \ 138 pcp_op_T__ ret__; \ 139 pcp_op_T__ *ptr__; \ 140 preempt_disable_notrace(); \ 141 ptr__ = raw_cpu_ptr(&(pcp)); \ 142 ret__ = cmpxchg(ptr__, oval, nval); \ 143 preempt_enable_notrace(); \ 144 ret__; \ 145 }) 146 147 #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 148 #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 149 #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 150 #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 151 152 #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) 153 154 #define this_cpu_cmpxchg128(pcp, oval, nval) \ 155 ({ \ 156 typedef typeof(pcp) pcp_op_T__; \ 157 u128 old__, new__, ret__; \ 158 pcp_op_T__ *ptr__; \ 159 old__ = oval; \ 160 new__ = nval; \ 161 preempt_disable_notrace(); \ 162 ptr__ = raw_cpu_ptr(&(pcp)); \ 163 ret__ = cmpxchg128((void *)ptr__, old__, new__); \ 164 preempt_enable_notrace(); \ 165 ret__; \ 166 }) 167 168 #define arch_this_cpu_xchg(pcp, nval) \ 169 ({ \ 170 typeof(pcp) *ptr__; \ 171 typeof(pcp) ret__; \ 172 preempt_disable_notrace(); \ 173 ptr__ = raw_cpu_ptr(&(pcp)); \ 174 ret__ = xchg(ptr__, nval); \ 175 preempt_enable_notrace(); \ 176 ret__; \ 177 }) 178 179 #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) 180 #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) 181 #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) 182 #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) 183 184 #include <asm-generic/percpu.h> 185 186 #endif /* __ARCH_S390_PERCPU__ */ 187