1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARCH_S390_PERCPU__ 3 #define __ARCH_S390_PERCPU__ 4 5 #include <linux/preempt.h> 6 #include <asm/cmpxchg.h> 7 #include <asm/march.h> 8 #include <asm/asm.h> 9 10 /* 11 * s390 uses its own implementation for per cpu data, the offset of 12 * the cpu local data area is cached in the cpu's lowcore memory. 13 */ 14 #define __my_cpu_offset get_lowcore()->percpu_offset 15 16 #define arch_raw_cpu_ptr(_ptr) \ 17 ({ \ 18 unsigned long tcp_ptr__; \ 19 \ 20 tcp_ptr__ = (__force unsigned long)(_ptr); \ 21 asm_inline volatile( \ 22 ALTERNATIVE("ag %[__ptr__],%[offzero](%%r0)\n", \ 23 "ag %[__ptr__],%[offalt](%%r0)\n", \ 24 ALT_FEATURE(MFEATURE_LOWCORE)) \ 25 : [__ptr__] "+d" (tcp_ptr__) \ 26 : [offzero] "i" (LC_PERCPU_OFFSET), \ 27 [offalt] "i" (LC_PERCPU_OFFSET + LOWCORE_ALT_ADDRESS), \ 28 "m" (((struct lowcore *)0)->percpu_offset) \ 29 : "cc"); \ 30 (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \ 31 }) 32 33 /* 34 * We use a compare-and-swap loop since that uses less cpu cycles than 35 * disabling and enabling interrupts like the generic variant would do. 36 */ 37 #define arch_this_cpu_to_op_simple(pcp, val, op) \ 38 ({ \ 39 typedef typeof(pcp) pcp_op_T__; \ 40 pcp_op_T__ old__, new__, prev__; \ 41 pcp_op_T__ *ptr__; \ 42 preempt_disable_notrace(); \ 43 ptr__ = raw_cpu_ptr(&(pcp)); \ 44 prev__ = READ_ONCE(*ptr__); \ 45 do { \ 46 old__ = prev__; \ 47 new__ = old__ op (val); \ 48 prev__ = cmpxchg(ptr__, old__, new__); \ 49 } while (prev__ != old__); \ 50 preempt_enable_notrace(); \ 51 new__; \ 52 }) 53 54 #define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 55 #define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 56 #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 57 #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 58 #define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 59 #define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 60 #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 61 #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 62 63 /* 64 * Macros to be used for percpu code section based on atomic instructions. 65 * 66 * Avoid the need to use preempt_disable() / preempt_enable() pairs and the 67 * conditional preempt_schedule_notrace() function calls which come with 68 * this. The idea is that this_cpu operations based on atomic instructions are 69 * guarded with mviy instructions: 70 * 71 * - The first mviy instruction writes the register number of the percpu address 72 * variable and the even register number of a register pair (which encodes two 73 * registers: the even register for the percpu offset and the odd register for 74 * the percpu pointer) to lowcore. This also indicates that a percpu code 75 * section is executed. 76 * 77 * - The mviy instruction is followed by the lg instruction which loads the 78 * percpu offset into the even register of the pair and the la instruction 79 * which adds the percpu offset and the percpu address into the odd register 80 * of the pair (the percpu pointer register). 81 * 82 * - Afterwards the atomic percpu operation follows. 83 * 84 * - Then a second mviy instruction writes a zero to lowcore, which indicates 85 * the end of the percpu code section. 86 * 87 * - In case of an interrupt/exception/nmi the encoded register numbers which 88 * were written to lowcore are copied to the exception frame (pt_regs), and a 89 * zero is written to lowcore. 90 * 91 * - On return to the previous context it is checked if a percpu code section 92 * was executed (saved register value not zero), and if the process was 93 * migrated to a different cpu. The content of the percpu offset register 94 * (even register of the pair) is reloaded with the current cpu's percpu 95 * offset and the percpu pointer register (odd register of the pair) is 96 * recalculated so it points to the percpu variable of the new cpu. 97 * 98 * Inline assemblies making use of this typically have a code sequence like: 99 * 100 * __PCPU_BEGIN(...) <- start of percpu code section 101 * atomic_op <- atomic op 102 * __PCPU_END(...) <- end of percpu code section 103 */ 104 105 #define LC_ALT_ADDR __stringify(LOWCORE_ALT_ADDRESS) 106 107 #define DEFINE_GR_NUM \ 108 ".macro _GR_NUM opd, gr\n" \ 109 ".set \\opd,255\n" \ 110 ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ 111 " .ifc \\gr,%%r\\rs\n" \ 112 " \\opd = \\rs\n" \ 113 " .endif\n" \ 114 ".endr\n" \ 115 ".if \\opd == 255\n" \ 116 " .error \"Illegal register number\"\n" \ 117 ".endif\n" \ 118 ".endm\n" 119 120 #define UNDEF_GR_NUM \ 121 ".purgem _GR_NUM\n" 122 123 #define PCPU_REG_PCP_SHIFT 0 124 #define PCPU_REG_PCP GENMASK(3, 0) 125 #define PCPU_REG_OFF_SHIFT 4 126 #define PCPU_REG_OFF GENMASK(7, 4) 127 128 #define __PCPU_MVIY(lcreg, imm) \ 129 ALTERNATIVE(" mviy " lcreg "(%%r0)," imm "\n", \ 130 " mviy " lcreg "+" LC_ALT_ADDR "(%%r0)," imm "\n", \ 131 ALT_FEATURE(MFEATURE_LOWCORE)) 132 133 #define __PCPU_LG(regoff, lcoff) \ 134 ALTERNATIVE(" lg " regoff ", " lcoff "(%%r0)\n", \ 135 " lg " regoff ", " lcoff "+" LC_ALT_ADDR "(%%r0)\n", \ 136 ALT_FEATURE(MFEATURE_LOWCORE)) 137 138 #define __PCPU_LA(regptr, regoff, regpcp) \ 139 " la " regptr ",0(" regoff "," regpcp ")\n" 140 141 #define __PCPU_CALC_REGVAL(regpcp, regoff) \ 142 "(" regpcp " << " __stringify(PCPU_REG_PCP_SHIFT) ") |" \ 143 "(" regoff " << " __stringify(PCPU_REG_OFF_SHIFT) ")" 144 145 #define __PCPU_CHECK_REGS(regpcp, regoff, regptr) \ 146 ".if " regoff " & 1\n" \ 147 " .error \"Percpu offset register must be even\"\n" \ 148 ".endif\n" \ 149 ".if (" regpcp " == 0) || (" regoff " == 0)\n" \ 150 " .error \"Percpu address and offset register must be non-zero\"\n"\ 151 ".endif\n" \ 152 ".if " regptr " != " regoff " + 1\n" \ 153 " .error \"Percpu pointer and offset register must be a pair\"\n" \ 154 ".endif\n" \ 155 ".if (" regpcp " == " regoff ") || (" regpcp " == " regptr ")\n" \ 156 " .error \"Percpu registers must be distinct\"\n" \ 157 ".endif\n" 158 159 #define __PCPU_BEGIN(lcreg, lcoff, regpcp, regoff, regptr) \ 160 DEFINE_GR_NUM \ 161 "_GR_NUM .Lregpcp, " regpcp "\n" \ 162 "_GR_NUM .Lregoff, " regoff "\n" \ 163 "_GR_NUM .Lregptr, " regptr "\n" \ 164 UNDEF_GR_NUM \ 165 __PCPU_CHECK_REGS(".Lregpcp", ".Lregoff", ".Lregptr") \ 166 ".set .Lregval, " __PCPU_CALC_REGVAL(".Lregpcp", ".Lregoff") "\n" \ 167 __PCPU_MVIY(lcreg, ".Lregval") \ 168 __PCPU_LG(regoff, lcoff) \ 169 __PCPU_LA(regptr, regoff, regpcp) 170 171 #define __PCPU_END(lcreg) \ 172 __PCPU_MVIY(lcreg, "0") 173 174 #ifndef MARCH_HAS_Z196_FEATURES 175 176 #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 177 #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 178 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 179 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 180 #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 181 #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 182 #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 183 #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 184 185 #else /* MARCH_HAS_Z196_FEATURES */ 186 187 #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ 188 do { \ 189 typedef typeof(pcp) pcp_op_T__; \ 190 union register_pair rp__; \ 191 pcp_op_T__ val__ = (val); \ 192 pcp_op_T__ *ptr__; \ 193 \ 194 ptr__ = PERCPU_PTR(&(pcp)); \ 195 if (__builtin_constant_p(val__) && \ 196 ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ 197 asm_inline volatile( \ 198 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 199 "%[pair__]","%N[pair__]") \ 200 op2 " 0(%N[pair__]),%[val__]\n" \ 201 __PCPU_END("%[lcreg]") \ 202 : [pair__] "=&a" (rp__.pair), "+m" (*ptr__), \ 203 "=m" (((struct lowcore *)0)->percpu_register) \ 204 : [val__] "i" ((szcast)val__), [ptr__] "a" (ptr__), \ 205 [lcreg] "i" (LC_PERCPU_REGISTER), \ 206 [lcoff] "i" (LC_PERCPU_OFFSET), \ 207 "m" (((struct lowcore *)0)->percpu_offset) \ 208 : "cc"); \ 209 } else { \ 210 asm_inline volatile( \ 211 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 212 "%[pair__]","%N[pair__]") \ 213 op1 " %[val__],%[val__],0(%N[pair__])\n" \ 214 __PCPU_END("%[lcreg]") \ 215 : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \ 216 "+m" (*ptr__), \ 217 "=m" (((struct lowcore *)0)->percpu_register) \ 218 : [ptr__] "a" (ptr__), \ 219 [lcreg] "i" (LC_PERCPU_REGISTER), \ 220 [lcoff] "i" (LC_PERCPU_OFFSET), \ 221 "m" (((struct lowcore *)0)->percpu_offset) \ 222 : "cc"); \ 223 } \ 224 } while (0) 225 226 #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) 227 #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) 228 229 #define arch_this_cpu_add_return(pcp, val, op) \ 230 ({ \ 231 typedef typeof(pcp) pcp_op_T__; \ 232 union register_pair rp__; \ 233 pcp_op_T__ val__ = (val); \ 234 pcp_op_T__ old__, *ptr__; \ 235 \ 236 ptr__ = PERCPU_PTR(&(pcp)); \ 237 asm_inline volatile( \ 238 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 239 "%[pair__]","%N[pair__]") \ 240 op " %[old__],%[val__],0(%N[pair__])\n" \ 241 __PCPU_END("%[lcreg]") \ 242 : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ 243 "+m" (*ptr__), \ 244 "=m" (((struct lowcore *)0)->percpu_register) \ 245 : [val__] "d" (val__), [ptr__] "a" (ptr__), \ 246 [lcreg] "i" (LC_PERCPU_REGISTER), \ 247 [lcoff] "i" (LC_PERCPU_OFFSET), \ 248 "m" (((struct lowcore *)0)->percpu_offset) \ 249 : "cc"); \ 250 old__ + val__; \ 251 }) 252 253 #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa") 254 #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") 255 256 #define arch_this_cpu_to_op(pcp, val, op) \ 257 do { \ 258 typedef typeof(pcp) pcp_op_T__; \ 259 union register_pair rp__; \ 260 pcp_op_T__ val__ = (val); \ 261 pcp_op_T__ *ptr__; \ 262 \ 263 ptr__ = PERCPU_PTR(&(pcp)); \ 264 asm_inline volatile( \ 265 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 266 "%[pair__]","%N[pair__]") \ 267 op " %[val__],%[val__],0(%N[pair__])\n" \ 268 __PCPU_END("%[lcreg]") \ 269 : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \ 270 "+m" (*ptr__), \ 271 "=m" (((struct lowcore *)0)->percpu_register) \ 272 : [ptr__] "a" (ptr__), \ 273 [lcreg] "i" (LC_PERCPU_REGISTER), \ 274 [lcoff] "i" (LC_PERCPU_OFFSET), \ 275 "m" (((struct lowcore *)0)->percpu_offset) \ 276 : "cc"); \ 277 } while (0) 278 279 #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") 280 #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") 281 #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") 282 #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") 283 284 #endif /* MARCH_HAS_Z196_FEATURES */ 285 286 #define arch_this_cpu_read(pcp, op) \ 287 ({ \ 288 typedef typeof(pcp) pcp_op_T__; \ 289 union register_pair rp__; \ 290 unsigned long res__; \ 291 pcp_op_T__ *ptr__; \ 292 \ 293 ptr__ = PERCPU_PTR(&(pcp)); \ 294 asm_inline volatile( \ 295 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 296 "%[pair__]","%N[pair__]") \ 297 op " %[res__],0(%N[pair__])\n" \ 298 __PCPU_END("%[lcreg]") \ 299 : [res__] "=&d" (res__), [pair__] "=&a" (rp__.pair), \ 300 "=m" (((struct lowcore *)0)->percpu_register) \ 301 : [ptr__] "a" (ptr__), \ 302 [lcreg] "i" (LC_PERCPU_REGISTER), \ 303 [lcoff] "i" (LC_PERCPU_OFFSET), \ 304 "m" (*ptr__), \ 305 "m" (((struct lowcore *)0)->percpu_offset) \ 306 : "cc"); \ 307 (pcp_op_T__)res__; \ 308 }) 309 310 #define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc") 311 #define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh") 312 #define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf") 313 #define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg") 314 315 #define arch_this_cpu_write(pcp, val, op) \ 316 do { \ 317 typedef typeof(pcp) pcp_op_T__; \ 318 union register_pair rp__; \ 319 pcp_op_T__ *ptr__, val__ = (val); \ 320 \ 321 ptr__ = PERCPU_PTR(&(pcp)); \ 322 asm_inline volatile( \ 323 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 324 "%[pair__]","%N[pair__]") \ 325 op " %[val__],0(%N[pair__])\n" \ 326 __PCPU_END("%[lcreg]") \ 327 : [pair__] "=&a" (rp__.pair), "=m" (*ptr__), \ 328 "=m" (((struct lowcore *)0)->percpu_register) \ 329 : [val__] "d" (val__), [ptr__] "a" (ptr__), \ 330 [lcreg] "i" (LC_PERCPU_REGISTER), \ 331 [lcoff] "i" (LC_PERCPU_OFFSET), \ 332 "m" (((struct lowcore *)0)->percpu_offset) \ 333 : "cc"); \ 334 } while (0) 335 336 #define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc") 337 #define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth") 338 #define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") 339 #define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") 340 341 #define arch_this_cpu_cmpxchg_simple(pcp, oval, nval) \ 342 ({ \ 343 typedef typeof(pcp) pcp_op_T__; \ 344 pcp_op_T__ ret__; \ 345 pcp_op_T__ *ptr__; \ 346 preempt_disable_notrace(); \ 347 ptr__ = raw_cpu_ptr(&(pcp)); \ 348 ret__ = cmpxchg(ptr__, oval, nval); \ 349 preempt_enable_notrace(); \ 350 ret__; \ 351 }) 352 353 #define arch_this_cpu_cmpxchg(pcp, oval, nval, op) \ 354 ({ \ 355 typedef typeof(pcp) pcp_op_T__; \ 356 pcp_op_T__ old__ = (oval), new__ = (nval); \ 357 union register_pair rp__; \ 358 pcp_op_T__ *ptr__; \ 359 \ 360 ptr__ = PERCPU_PTR(&(pcp)); \ 361 asm_inline volatile( \ 362 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 363 "%[pair__]","%N[pair__]") \ 364 op " %[old__],%[new__],0(%N[pair__])\n" \ 365 __PCPU_END("%[lcreg]") \ 366 : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ 367 "+m" (*ptr__), \ 368 "=m" (((struct lowcore *)0)->percpu_register) \ 369 : [new__] "d" (new__), [ptr__] "a" (ptr__), \ 370 [lcreg] "i" (LC_PERCPU_REGISTER), \ 371 [lcoff] "i" (LC_PERCPU_OFFSET), \ 372 "m" (((struct lowcore *)0)->percpu_offset) \ 373 : "memory", "cc"); \ 374 old__; \ 375 }) 376 377 #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval) 378 #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval) 379 #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "cs") 380 #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "csg") 381 382 #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) 383 384 #define this_cpu_cmpxchg128(pcp, oval, nval) \ 385 ({ \ 386 typedef typeof(pcp) pcp_op_T__; \ 387 u128 old__ = (oval), new__ = (nval); \ 388 union register_pair rp__; \ 389 pcp_op_T__ *ptr__; \ 390 \ 391 ptr__ = PERCPU_PTR(&(pcp)); \ 392 asm_inline volatile( \ 393 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 394 "%[pair__]","%N[pair__]") \ 395 " cdsg %[old__],%[new__],0(%N[pair__])\n" \ 396 __PCPU_END("%[lcreg]") \ 397 : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ 398 "+m" (*ptr__), \ 399 "=m" (((struct lowcore *)0)->percpu_register) \ 400 : [new__] "d" (new__), [ptr__] "a" (ptr__), \ 401 [lcreg] "i" (LC_PERCPU_REGISTER), \ 402 [lcoff] "i" (LC_PERCPU_OFFSET), \ 403 "m" (((struct lowcore *)0)->percpu_offset) \ 404 : "memory", "cc"); \ 405 old__; \ 406 }) 407 408 #ifdef __HAVE_ASM_FLAG_OUTPUTS__ 409 410 #define arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, op) \ 411 ({ \ 412 typedef typeof(pcp) pcp_op_T__; \ 413 typeof(ovalp) ovalp__ = (ovalp); \ 414 pcp_op_T__ old__ = *ovalp__; \ 415 pcp_op_T__ new__ = (nval); \ 416 union register_pair rp__; \ 417 pcp_op_T__ *ptr__; \ 418 int cc__; \ 419 \ 420 ptr__ = PERCPU_PTR(&(pcp)); \ 421 asm_inline volatile( \ 422 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 423 "%[pair__]","%N[pair__]") \ 424 op " %[old__],%[new__],0(%N[pair__])\n" \ 425 __PCPU_END("%[lcreg]") \ 426 : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ 427 "+m" (*ptr__), "=@cc" (cc__), \ 428 "=m" (((struct lowcore *)0)->percpu_register) \ 429 : [new__] "d" (new__), [ptr__] "a" (ptr__), \ 430 [lcreg] "i" (LC_PERCPU_REGISTER), \ 431 [lcoff] "i" (LC_PERCPU_OFFSET), \ 432 "m" (((struct lowcore *)0)->percpu_offset) \ 433 : "memory"); \ 434 if (unlikely(cc__)) \ 435 *ovalp__ = old__; \ 436 likely(cc__ == 0); \ 437 }) 438 439 #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "cs") 440 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "csg") 441 442 #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) this_cpu_try_cmpxchg_8(pcp, ovalp, nval) 443 444 #endif /* __HAVE_ASM_FLAG_OUTPUTS__ */ 445 446 #define arch_this_cpu_xchg_simple(pcp, nval) \ 447 ({ \ 448 typeof(pcp) *ptr__; \ 449 typeof(pcp) ret__; \ 450 preempt_disable_notrace(); \ 451 ptr__ = raw_cpu_ptr(&(pcp)); \ 452 ret__ = xchg(ptr__, nval); \ 453 preempt_enable_notrace(); \ 454 ret__; \ 455 }) 456 457 #define arch_this_cpu_xchg(pcp, nval, ldop, csop) \ 458 ({ \ 459 typedef typeof(pcp) pcp_op_T__; \ 460 pcp_op_T__ old__, new__ = (nval); \ 461 union register_pair rp__; \ 462 pcp_op_T__ *ptr__; \ 463 \ 464 ptr__ = PERCPU_PTR(&(pcp)); \ 465 asm_inline volatile( \ 466 __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ 467 "%[pair__]","%N[pair__]") \ 468 " " ldop " %[old__],0(%N[pair__])\n" \ 469 "0: " csop " %[old__],%[new__],0(%N[pair__])\n"\ 470 " jnz 0b\n" \ 471 __PCPU_END("%[lcreg]") \ 472 : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ 473 "+m" (*ptr__), \ 474 "=m" (((struct lowcore *)0)->percpu_register) \ 475 : [new__] "d" (new__), [ptr__] "a" (ptr__), \ 476 [lcreg] "i" (LC_PERCPU_REGISTER), \ 477 [lcoff] "i" (LC_PERCPU_OFFSET), \ 478 "m" (((struct lowcore *)0)->percpu_offset) \ 479 : "memory", "cc"); \ 480 old__; \ 481 }) 482 483 #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval) 484 #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval) 485 #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval, "l", "cs") 486 #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval, "lg", "csg") 487 488 #include <asm-generic/percpu.h> 489 490 #endif /* __ARCH_S390_PERCPU__ */ 491