1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2013 ARM Ltd. 4 */ 5 #ifndef __ASM_PERCPU_H 6 #define __ASM_PERCPU_H 7 8 #include <linux/bits.h> 9 #include <linux/preempt.h> 10 #include <linux/stringify.h> 11 12 #include <asm/alternative.h> 13 #include <asm/cmpxchg.h> 14 #include <asm/gpr-num.h> 15 #include <asm/stack_pointer.h> 16 #include <asm/sysreg.h> 17 18 static inline void set_my_cpu_offset(unsigned long off) 19 { 20 asm volatile(ALTERNATIVE("msr tpidr_el1, %0", 21 "msr tpidr_el2, %0", 22 ARM64_HAS_VIRT_HOST_EXTN) 23 :: "r" (off) : "memory"); 24 } 25 26 static inline unsigned long __hyp_my_cpu_offset(void) 27 { 28 /* 29 * Non-VHE hyp code runs with preemption disabled. No need to hazard 30 * the register access against barrier() as in __kern_my_cpu_offset. 31 */ 32 return read_sysreg(tpidr_el2); 33 } 34 35 #define __KERN_ASM_CPU_OFFSET(dst) \ 36 ALTERNATIVE("mrs " dst ", tpidr_el1", \ 37 "mrs " dst ", tpidr_el2", \ 38 ARM64_HAS_VIRT_HOST_EXTN) 39 40 static inline unsigned long __kern_my_cpu_offset(void) 41 { 42 unsigned long off; 43 44 /* 45 * We want to allow caching the value, so avoid using volatile and 46 * instead use a fake stack read to hazard against barrier(). 47 */ 48 asm( 49 __KERN_ASM_CPU_OFFSET("%0") 50 : "=r" (off) 51 : "Q" (*(const unsigned long *)current_stack_pointer) 52 ); 53 54 return off; 55 } 56 57 #define PCPU_GPR_PCP_SHIFT 0 58 #define PCPU_GPR_PCP GENMASK(4, 0) 59 #define PCPU_GPR_OFF_SHIFT 5 60 #define PCPU_GPR_OFF GENMASK(9, 5) 61 #define PCPU_GPR_ADDR_SHIFT 10 62 #define PCPU_GPR_ADDR GENMASK(14, 10) 63 64 #define __VAL_PCPU_GPRS(pcp, off, addr) \ 65 "(" \ 66 "(" __GPR_NUM(pcp) " << " __stringify(PCPU_GPR_PCP_SHIFT) ") | " \ 67 "(" __GPR_NUM(off) " << " __stringify(PCPU_GPR_OFF_SHIFT) ") | " \ 68 "(" __GPR_NUM(addr) " << " __stringify(PCPU_GPR_ADDR_SHIFT) ")" \ 69 ")" 70 71 #define __ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr) \ 72 ".if (" \ 73 "(" __GPR_NUM(pcp) " == " __GPR_NUM(off) ") || " \ 74 "(" __GPR_NUM(pcp) " == " __GPR_NUM(addr) ") || " \ 75 "(" __GPR_NUM(off) " == " __GPR_NUM(addr) ")" \ 76 " )\n" \ 77 ".error \"PCPU GPRS overlap: {" pcp "," off "," addr "}\"\n" \ 78 ".endif\n" 79 80 #define ____PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \ 81 "// ____PCPU_GPRS_BEGIN(" gprs ", " pcp ", " off ", " addr")\n" \ 82 __DEFINE_ASM_GPR_NUMS \ 83 __DEFINE_ASM_GPR_ALIASES \ 84 __ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr) \ 85 " mov w" off ", #" __VAL_PCPU_GPRS(pcp, off, addr) "\n" \ 86 " strh w" off ", " gprs "\n" \ 87 __KERN_ASM_CPU_OFFSET(off) "\n" 88 89 /* 90 * Begin a PCPU GPR critical section which requires <addr> (and <off>). 91 * 92 * At the start of the critical section, and upon any (preemptible) exception 93 * until __PCPU_GPRS_END(): 94 * - <off> will be set to the current CPU's percpu offset. 95 * - <addr> will be set to <pcp> + <off>. 96 * 97 * The <pcp>, <off>, and <addr> registers must be distinct GPRs. 98 * 99 * <gprs> must be '¤t_thread_info()->pcpu_gprs', as a memory operand 100 * which can be written both at the start and end of the critical section 101 * (e.g. using "=Qo" constraints). 102 */ 103 #define __PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \ 104 ____PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \ 105 " add " addr ", " pcp ", " off "\n" 106 107 /* 108 * Begin a PCPU GPR critical section which only requires <off> and does not 109 * require <addr>. 110 * 111 * This is only for operations that can use register-offset addressing, 112 * e.g. STR <Xt>, [<Xn>, <Xm>]. 113 * 114 * All other details are the same as __PCPU_GPRS_BEGIN(). 115 */ 116 #define __PCPU_GPRS_BEGIN_OFFSET(gprs, pcp, off) \ 117 ____PCPU_GPRS_BEGIN(gprs, pcp, off, "xzr") 118 119 /* 120 * End a PCPU GPR critical section. 121 */ 122 #define __PCPU_GPRS_END(gprs) \ 123 " strh wzr, " gprs "\n" 124 125 #ifdef __KVM_NVHE_HYPERVISOR__ 126 #define __my_cpu_offset __hyp_my_cpu_offset() 127 #else 128 #define __my_cpu_offset __kern_my_cpu_offset() 129 #endif 130 131 #define PERCPU_RW_OPS(w, sfx, sz) \ 132 static inline unsigned long __percpu_read_##sz(void __percpu *pcp) \ 133 { \ 134 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \ 135 unsigned long off; \ 136 unsigned long val; \ 137 \ 138 asm volatile( \ 139 __PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]") \ 140 " ldr" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n" \ 141 __PCPU_GPRS_END("%[gprs]") \ 142 : [gprs] "=Qo" (*gprs), \ 143 [off] "=&r" (off), \ 144 [val] "=&r" (val) \ 145 : [pcp] "r" (pcp) \ 146 : "memory" \ 147 ); \ 148 \ 149 return val; \ 150 } \ 151 \ 152 static inline void __percpu_write_##sz(void __percpu *pcp, unsigned long val) \ 153 { \ 154 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \ 155 unsigned long off; \ 156 \ 157 asm volatile( \ 158 __PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]") \ 159 " str" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n" \ 160 __PCPU_GPRS_END("%[gprs]") \ 161 : [gprs] "=Qo" (*gprs), \ 162 [off] "=&r" (off) \ 163 : [pcp] "r" (pcp), \ 164 [val] "r" ((u##sz)val) \ 165 : "memory" \ 166 ); \ 167 } 168 169 #define __PERCPU_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \ 170 static inline void \ 171 __percpu_##name##_case_##sz(void __percpu *pcp, unsigned long val) \ 172 { \ 173 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \ 174 unsigned long addr; \ 175 unsigned long off; \ 176 unsigned int loop; \ 177 u##sz tmp; \ 178 \ 179 asm volatile ( \ 180 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \ 181 ARM64_LSE_ATOMIC_INSN( \ 182 /* LL/SC */ \ 183 "1: ldxr" #sfx "\t%" #w "[tmp], [%[addr]]\n" \ 184 #op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \ 185 " stxr" #sfx "\t%w[loop], %" #w "[tmp], [%[addr]]\n" \ 186 " cbnz %w[loop], 1b", \ 187 /* LSE atomics */ \ 188 #op_lse #sfx "\t%" #w "[val], %" #w "[tmp], [%[addr]]\n"\ 189 __nops(3)) \ 190 __PCPU_GPRS_END("%[gprs]") \ 191 : [gprs] "=Qo" (*gprs), \ 192 [addr] "=&r" (addr), \ 193 [off] "=&r" (off), \ 194 [loop] "=&r" (loop), \ 195 [tmp] "=&r" (tmp) \ 196 : [pcp] "r" (pcp), \ 197 [val] "r" ((u##sz)(val)) \ 198 : "memory" \ 199 ); \ 200 } 201 202 #define __PERCPU_RET_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \ 203 static inline u##sz \ 204 __percpu_##name##_return_case_##sz(void __percpu *pcp, unsigned long val) \ 205 { \ 206 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \ 207 unsigned long addr; \ 208 unsigned long off; \ 209 unsigned int loop; \ 210 u##sz ret; \ 211 \ 212 asm volatile ( \ 213 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \ 214 ARM64_LSE_ATOMIC_INSN( \ 215 /* LL/SC */ \ 216 "1: ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n" \ 217 #op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \ 218 " stxr" #sfx "\t%w[loop], %" #w "[ret], [%[addr]]\n" \ 219 " cbnz %w[loop], 1b", \ 220 /* LSE atomics */ \ 221 #op_lse #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n"\ 222 #op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \ 223 __nops(2)) \ 224 __PCPU_GPRS_END("%[gprs]") \ 225 : [gprs] "=Qo" (*gprs), \ 226 [addr] "=&r" (addr), \ 227 [off] "=&r" (off), \ 228 [loop] "=&r" (loop), \ 229 [ret] "=&r" (ret) \ 230 : [pcp] "r" (pcp), \ 231 [val] "r" ((u##sz)(val)) \ 232 : "memory" \ 233 ); \ 234 \ 235 return ret; \ 236 } 237 238 #define PERCPU_OP(name, op_llsc, op_lse) \ 239 __PERCPU_OP_CASE(w, b, name, 8, op_llsc, op_lse) \ 240 __PERCPU_OP_CASE(w, h, name, 16, op_llsc, op_lse) \ 241 __PERCPU_OP_CASE(w, , name, 32, op_llsc, op_lse) \ 242 __PERCPU_OP_CASE( , , name, 64, op_llsc, op_lse) 243 244 #define PERCPU_RET_OP(name, op_llsc, op_lse) \ 245 __PERCPU_RET_OP_CASE(w, b, name, 8, op_llsc, op_lse) \ 246 __PERCPU_RET_OP_CASE(w, h, name, 16, op_llsc, op_lse) \ 247 __PERCPU_RET_OP_CASE(w, , name, 32, op_llsc, op_lse) \ 248 __PERCPU_RET_OP_CASE( , , name, 64, op_llsc, op_lse) 249 250 PERCPU_RW_OPS(w, b, 8) 251 PERCPU_RW_OPS(w, h, 16) 252 PERCPU_RW_OPS(w, , 32) 253 PERCPU_RW_OPS( , , 64) 254 255 /* 256 * Use value-returning atomics for CPU-local ops as they are more likely 257 * to execute "near" to the CPU (e.g. in L1$). 258 * 259 * https://lore.kernel.org/r/e7d539ed-ced0-4b96-8ecd-048a5b803b85@paulmck-laptop 260 */ 261 PERCPU_OP(add, add, ldadd) 262 PERCPU_OP(andnot, bic, ldclr) 263 PERCPU_OP(or, orr, ldset) 264 PERCPU_RET_OP(add, add, ldadd) 265 266 #undef PERCPU_RW_OPS 267 #undef __PERCPU_OP_CASE 268 #undef __PERCPU_RET_OP_CASE 269 #undef PERCPU_OP 270 #undef PERCPU_RET_OP 271 272 #define PERCPU_XCHG_OP(w, sfx, sz) \ 273 static inline unsigned long \ 274 __percpu_xchg_case_##sz(void __percpu *pcp, u##sz val) \ 275 { \ 276 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \ 277 unsigned long addr; \ 278 unsigned long off; \ 279 unsigned int loop; \ 280 unsigned long ret; \ 281 \ 282 asm volatile ( \ 283 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \ 284 ARM64_LSE_ATOMIC_INSN( \ 285 /* LL/SC */ \ 286 " prfm pstl1strm, [%[addr]]\n" \ 287 "1: ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n" \ 288 " stxr" #sfx "\t%w[loop], %" #w "[val], [%[addr]]\n" \ 289 " cbnz %w[loop], 1b\n" \ 290 , \ 291 /* LSE atomics */ \ 292 " swp" #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n" \ 293 __nops(3) \ 294 ) \ 295 __PCPU_GPRS_END("%[gprs]") \ 296 : [gprs] "=Qo" (*gprs), \ 297 [addr] "=&r" (addr), \ 298 [off] "=&r" (off), \ 299 [loop] "=&r" (loop), \ 300 [ret] "=&r" (ret) \ 301 : [pcp] "r" (pcp), \ 302 [val] "r" (val) \ 303 : "memory" \ 304 ); \ 305 \ 306 return ret; \ 307 } 308 309 #define PERCPU_CMPXCHG_OP(w, sfx, sz) \ 310 static inline unsigned long \ 311 __percpu_cmpxchg_case_##sz(void __percpu *pcp, \ 312 u##sz old, \ 313 u##sz new) \ 314 { \ 315 /* \ 316 * Sub-word sizes require zero extension so that EOR+CBNZ don't \ 317 * consume non-zero upper bits of the register containing "old".\ 318 */ \ 319 xwreg_t(w) cmpval = xwreg_zero_extend(old, w, sz); \ 320 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \ 321 unsigned long addr; \ 322 unsigned long off; \ 323 unsigned long tmp; \ 324 unsigned long oldval; \ 325 \ 326 asm volatile ( \ 327 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \ 328 ARM64_LSE_ATOMIC_INSN( \ 329 /* LL/SC */ \ 330 " prfm pstl1strm, [%[addr]]\n" \ 331 "1: ldxr" #sfx "\t%" #w "[oldval], [%[addr]]\n" \ 332 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ 333 " cbnz %" #w "[tmp], 2f\n" \ 334 " stxr" #sfx "\t%w[tmp], %" #w "[new], [%[addr]]\n" \ 335 " cbnz %w[tmp], 1b\n" \ 336 "2:\n" \ 337 , \ 338 /* LSE atomics */ \ 339 " mov %" #w "[oldval], %" #w "[old]\n" \ 340 " cas" #sfx "\t%" #w "[oldval], %" #w "[new], [%[addr]]\n"\ 341 __nops(4) \ 342 ) \ 343 __PCPU_GPRS_END("%[gprs]") \ 344 : [gprs] "=Qo" (*gprs), \ 345 [addr] "=&r" (addr), \ 346 [off] "=&r" (off), \ 347 [tmp] "=&r" (tmp), \ 348 [oldval] "=&r" (oldval) \ 349 : [pcp] "r" (pcp), \ 350 [old] "r" (cmpval), \ 351 [new] "r" (new) \ 352 : "memory" \ 353 ); \ 354 \ 355 return oldval; \ 356 } 357 358 PERCPU_XCHG_OP(w, b, 8) 359 PERCPU_XCHG_OP(w, h, 16) 360 PERCPU_XCHG_OP(w, , 32) 361 PERCPU_XCHG_OP(x, , 64) 362 363 PERCPU_CMPXCHG_OP(w, b, 8) 364 PERCPU_CMPXCHG_OP(w, h, 16) 365 PERCPU_CMPXCHG_OP(w, , 32) 366 PERCPU_CMPXCHG_OP(x, , 64) 367 368 #undef PERCPU_XCHG_OP 369 #undef PERCPU_CMPXCHG_OP 370 371 #define _pcp_wrap(op, pcp, ...) \ 372 ({ \ 373 op(&(pcp), __VA_ARGS__); \ 374 }) 375 376 #define _pcp_wrap_return(op, pcp, args...) \ 377 ({ \ 378 (typeof(pcp))op(&(pcp), ##args); \ 379 }) 380 381 #define _pcp_wrap_xchg(op, pcp, val) \ 382 ({ \ 383 (typeof(pcp))op(&(pcp), (unsigned long)(val)); \ 384 }) 385 386 #define _pcp_wrap_cmpxchg(op, pcp, old, new) \ 387 ({ \ 388 (typeof(pcp))op(&(pcp), (unsigned long)(old), \ 389 (unsigned long)(new)); \ 390 }) 391 392 #define this_cpu_read_1(pcp) \ 393 _pcp_wrap_return(__percpu_read_8, pcp) 394 #define this_cpu_read_2(pcp) \ 395 _pcp_wrap_return(__percpu_read_16, pcp) 396 #define this_cpu_read_4(pcp) \ 397 _pcp_wrap_return(__percpu_read_32, pcp) 398 #define this_cpu_read_8(pcp) \ 399 _pcp_wrap_return(__percpu_read_64, pcp) 400 401 #define this_cpu_write_1(pcp, val) \ 402 _pcp_wrap(__percpu_write_8, pcp, (unsigned long)(val)) 403 #define this_cpu_write_2(pcp, val) \ 404 _pcp_wrap(__percpu_write_16, pcp, (unsigned long)(val)) 405 #define this_cpu_write_4(pcp, val) \ 406 _pcp_wrap(__percpu_write_32, pcp, (unsigned long)(val)) 407 #define this_cpu_write_8(pcp, val) \ 408 _pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val)) 409 410 #define this_cpu_add_1(pcp, val) \ 411 _pcp_wrap(__percpu_add_case_8, pcp, val) 412 #define this_cpu_add_2(pcp, val) \ 413 _pcp_wrap(__percpu_add_case_16, pcp, val) 414 #define this_cpu_add_4(pcp, val) \ 415 _pcp_wrap(__percpu_add_case_32, pcp, val) 416 #define this_cpu_add_8(pcp, val) \ 417 _pcp_wrap(__percpu_add_case_64, pcp, val) 418 419 #define this_cpu_add_return_1(pcp, val) \ 420 _pcp_wrap_return(__percpu_add_return_case_8, pcp, val) 421 #define this_cpu_add_return_2(pcp, val) \ 422 _pcp_wrap_return(__percpu_add_return_case_16, pcp, val) 423 #define this_cpu_add_return_4(pcp, val) \ 424 _pcp_wrap_return(__percpu_add_return_case_32, pcp, val) 425 #define this_cpu_add_return_8(pcp, val) \ 426 _pcp_wrap_return(__percpu_add_return_case_64, pcp, val) 427 428 #define this_cpu_and_1(pcp, val) \ 429 _pcp_wrap(__percpu_andnot_case_8, pcp, ~(u8)(val)) 430 #define this_cpu_and_2(pcp, val) \ 431 _pcp_wrap(__percpu_andnot_case_16, pcp, ~(u16)(val)) 432 #define this_cpu_and_4(pcp, val) \ 433 _pcp_wrap(__percpu_andnot_case_32, pcp, ~(u32)(val)) 434 #define this_cpu_and_8(pcp, val) \ 435 _pcp_wrap(__percpu_andnot_case_64, pcp, ~(u64)(val)) 436 437 #define this_cpu_or_1(pcp, val) \ 438 _pcp_wrap(__percpu_or_case_8, pcp, val) 439 #define this_cpu_or_2(pcp, val) \ 440 _pcp_wrap(__percpu_or_case_16, pcp, val) 441 #define this_cpu_or_4(pcp, val) \ 442 _pcp_wrap(__percpu_or_case_32, pcp, val) 443 #define this_cpu_or_8(pcp, val) \ 444 _pcp_wrap(__percpu_or_case_64, pcp, val) 445 446 #define this_cpu_xchg_1(pcp, val) \ 447 _pcp_wrap_xchg(__percpu_xchg_case_8, pcp, val) 448 #define this_cpu_xchg_2(pcp, val) \ 449 _pcp_wrap_xchg(__percpu_xchg_case_16, pcp, val) 450 #define this_cpu_xchg_4(pcp, val) \ 451 _pcp_wrap_xchg(__percpu_xchg_case_32, pcp, val) 452 #define this_cpu_xchg_8(pcp, val) \ 453 _pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val) 454 455 #define this_cpu_cmpxchg_1(pcp, o, n) \ 456 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_8, pcp, o, n) 457 #define this_cpu_cmpxchg_2(pcp, o, n) \ 458 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_16, pcp, o, n) 459 #define this_cpu_cmpxchg_4(pcp, o, n) \ 460 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_32, pcp, o, n) 461 #define this_cpu_cmpxchg_8(pcp, o, n) \ 462 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_64, pcp, o, n) 463 464 #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) 465 466 static inline u128 467 __percpu_cmpxchg_128(void __percpu *pcp, u128 old, u128 new) 468 { 469 u16 *gprs = ¤t_thread_info()->pcpu_gprs; 470 unsigned long addr; 471 unsigned long off; 472 union __u128_halves r, o = { .full = (old) }, 473 n = { .full = (new) }; 474 register unsigned long ol asm ("x0") = o.low; 475 register unsigned long oh asm ("x1") = o.high; 476 register unsigned long nl asm ("x2") = n.low; 477 register unsigned long nh asm ("x3") = n.high; 478 unsigned long rl, rh; 479 unsigned long tmp; 480 481 asm volatile ( 482 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") 483 ARM64_LSE_ATOMIC_INSN( 484 /* LL/SC */ 485 " prfm pstl1strm, [%[addr]]\n" 486 "1: ldxp %[rl], %[rh], [%[addr]]\n" 487 " cmp %[rl], %[ol]\n" 488 " ccmp %[rh], %[oh], 0, eq\n" 489 " b.ne 2f\n" 490 " stxp %w[tmp], %[nl], %[nh], [%[addr]]\n" 491 " cbnz %w[tmp], 1b\n" 492 "2:\n" 493 , 494 /* LSE atomics */ 495 " casp %[ol], %[oh], %[nl], %[nh], [%[addr]]\n" 496 " mov %[rl], %[ol]\n" 497 " mov %[rh], %[oh]\n" 498 __nops(4) 499 ) 500 __PCPU_GPRS_END("%[gprs]") 501 : [gprs] "=Qo" (*gprs), 502 [addr] "=&r" (addr), 503 [off] "=&r" (off), 504 [tmp] "=&r" (tmp), 505 [ol] "+&r" (ol), 506 [oh] "+&r" (oh), 507 [rl] "=&r" (rl), 508 [rh] "=&r" (rh) 509 : [pcp] "r" (pcp), 510 [nl] "r" (nl), 511 [nh] "r" (nh) 512 : "memory", "cc" 513 ); 514 515 r.low = rl; 516 r.high = rh; 517 518 return r.full; 519 } 520 521 #define this_cpu_cmpxchg128(pcp, o, n) \ 522 _pcp_wrap_return(__percpu_cmpxchg_128, pcp, o, n) 523 524 #ifdef __KVM_NVHE_HYPERVISOR__ 525 extern unsigned long __hyp_per_cpu_offset(unsigned int cpu); 526 #define __per_cpu_offset 527 #define per_cpu_offset(cpu) __hyp_per_cpu_offset((cpu)) 528 #endif 529 530 #include <asm-generic/percpu.h> 531 532 /* Redefine macros for nVHE hyp under DEBUG_PREEMPT to avoid its dependencies. */ 533 #if defined(__KVM_NVHE_HYPERVISOR__) && defined(CONFIG_DEBUG_PREEMPT) 534 #undef this_cpu_ptr 535 #define this_cpu_ptr raw_cpu_ptr 536 #undef __this_cpu_read 537 #define __this_cpu_read raw_cpu_read 538 #undef __this_cpu_write 539 #define __this_cpu_write raw_cpu_write 540 #endif 541 542 #endif /* __ASM_PERCPU_H */ 543