1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2013 ARM Ltd.
4 */
5 #ifndef __ASM_PERCPU_H
6 #define __ASM_PERCPU_H
7
8 #include <linux/bits.h>
9 #include <linux/preempt.h>
10 #include <linux/stringify.h>
11
12 #include <asm/alternative.h>
13 #include <asm/cmpxchg.h>
14 #include <asm/gpr-num.h>
15 #include <asm/stack_pointer.h>
16 #include <asm/sysreg.h>
17
set_my_cpu_offset(unsigned long off)18 static inline void set_my_cpu_offset(unsigned long off)
19 {
20 asm volatile(ALTERNATIVE("msr tpidr_el1, %0",
21 "msr tpidr_el2, %0",
22 ARM64_HAS_VIRT_HOST_EXTN)
23 :: "r" (off) : "memory");
24 }
25
__hyp_my_cpu_offset(void)26 static inline unsigned long __hyp_my_cpu_offset(void)
27 {
28 /*
29 * Non-VHE hyp code runs with preemption disabled. No need to hazard
30 * the register access against barrier() as in __kern_my_cpu_offset.
31 */
32 return read_sysreg(tpidr_el2);
33 }
34
35 #define __KERN_ASM_CPU_OFFSET(dst) \
36 ALTERNATIVE("mrs " dst ", tpidr_el1", \
37 "mrs " dst ", tpidr_el2", \
38 ARM64_HAS_VIRT_HOST_EXTN)
39
__kern_my_cpu_offset(void)40 static inline unsigned long __kern_my_cpu_offset(void)
41 {
42 unsigned long off;
43
44 /*
45 * We want to allow caching the value, so avoid using volatile and
46 * instead use a fake stack read to hazard against barrier().
47 */
48 asm(
49 __KERN_ASM_CPU_OFFSET("%0")
50 : "=r" (off)
51 : "Q" (*(const unsigned long *)current_stack_pointer)
52 );
53
54 return off;
55 }
56
57 #define PCPU_GPR_PCP_SHIFT 0
58 #define PCPU_GPR_PCP GENMASK(4, 0)
59 #define PCPU_GPR_OFF_SHIFT 5
60 #define PCPU_GPR_OFF GENMASK(9, 5)
61 #define PCPU_GPR_ADDR_SHIFT 10
62 #define PCPU_GPR_ADDR GENMASK(14, 10)
63
64 #define __VAL_PCPU_GPRS(pcp, off, addr) \
65 "(" \
66 "(" __GPR_NUM(pcp) " << " __stringify(PCPU_GPR_PCP_SHIFT) ") | " \
67 "(" __GPR_NUM(off) " << " __stringify(PCPU_GPR_OFF_SHIFT) ") | " \
68 "(" __GPR_NUM(addr) " << " __stringify(PCPU_GPR_ADDR_SHIFT) ")" \
69 ")"
70
71 #define __ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr) \
72 ".if (" \
73 "(" __GPR_NUM(pcp) " == " __GPR_NUM(off) ") || " \
74 "(" __GPR_NUM(pcp) " == " __GPR_NUM(addr) ") || " \
75 "(" __GPR_NUM(off) " == " __GPR_NUM(addr) ")" \
76 " )\n" \
77 ".error \"PCPU GPRS overlap: {" pcp "," off "," addr "}\"\n" \
78 ".endif\n"
79
80 #define ____PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \
81 "// ____PCPU_GPRS_BEGIN(" gprs ", " pcp ", " off ", " addr")\n" \
82 __DEFINE_ASM_GPR_NUMS \
83 __DEFINE_ASM_GPR_ALIASES \
84 __ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr) \
85 " mov w" off ", #" __VAL_PCPU_GPRS(pcp, off, addr) "\n" \
86 " strh w" off ", " gprs "\n" \
87 __KERN_ASM_CPU_OFFSET(off) "\n"
88
89 /*
90 * Begin a PCPU GPR critical section which requires <addr> (and <off>).
91 *
92 * At the start of the critical section, and upon any (preemptible) exception
93 * until __PCPU_GPRS_END():
94 * - <off> will be set to the current CPU's percpu offset.
95 * - <addr> will be set to <pcp> + <off>.
96 *
97 * The <pcp>, <off>, and <addr> registers must be distinct GPRs.
98 *
99 * <gprs> must be '¤t_thread_info()->pcpu_gprs', as a memory operand
100 * which can be written both at the start and end of the critical section
101 * (e.g. using "=Qo" constraints).
102 */
103 #define __PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \
104 ____PCPU_GPRS_BEGIN(gprs, pcp, off, addr) \
105 " add " addr ", " pcp ", " off "\n"
106
107 /*
108 * Begin a PCPU GPR critical section which only requires <off> and does not
109 * require <addr>.
110 *
111 * This is only for operations that can use register-offset addressing,
112 * e.g. STR <Xt>, [<Xn>, <Xm>].
113 *
114 * All other details are the same as __PCPU_GPRS_BEGIN().
115 */
116 #define __PCPU_GPRS_BEGIN_OFFSET(gprs, pcp, off) \
117 ____PCPU_GPRS_BEGIN(gprs, pcp, off, "xzr")
118
119 /*
120 * End a PCPU GPR critical section.
121 */
122 #define __PCPU_GPRS_END(gprs) \
123 " strh wzr, " gprs "\n"
124
125 #ifdef __KVM_NVHE_HYPERVISOR__
126 #define __my_cpu_offset __hyp_my_cpu_offset()
127 #else
128 #define __my_cpu_offset __kern_my_cpu_offset()
129 #endif
130
131 #define PERCPU_RW_OPS(w, sfx, sz) \
132 static inline unsigned long __percpu_read_##sz(void __percpu *pcp) \
133 { \
134 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \
135 unsigned long off; \
136 unsigned long val; \
137 \
138 asm volatile( \
139 __PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]") \
140 " ldr" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n" \
141 __PCPU_GPRS_END("%[gprs]") \
142 : [gprs] "=Qo" (*gprs), \
143 [off] "=&r" (off), \
144 [val] "=&r" (val) \
145 : [pcp] "r" (pcp) \
146 : "memory" \
147 ); \
148 \
149 return val; \
150 } \
151 \
152 static inline void __percpu_write_##sz(void __percpu *pcp, unsigned long val) \
153 { \
154 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \
155 unsigned long off; \
156 \
157 asm volatile( \
158 __PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]") \
159 " str" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n" \
160 __PCPU_GPRS_END("%[gprs]") \
161 : [gprs] "=Qo" (*gprs), \
162 [off] "=&r" (off) \
163 : [pcp] "r" (pcp), \
164 [val] "r" ((u##sz)val) \
165 : "memory" \
166 ); \
167 }
168
169 #define __PERCPU_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
170 static inline void \
171 __percpu_##name##_case_##sz(void __percpu *pcp, unsigned long val) \
172 { \
173 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \
174 unsigned long addr; \
175 unsigned long off; \
176 unsigned int loop; \
177 u##sz tmp; \
178 \
179 asm volatile ( \
180 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
181 ARM64_LSE_ATOMIC_INSN( \
182 /* LL/SC */ \
183 "1: ldxr" #sfx "\t%" #w "[tmp], [%[addr]]\n" \
184 #op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
185 " stxr" #sfx "\t%w[loop], %" #w "[tmp], [%[addr]]\n" \
186 " cbnz %w[loop], 1b", \
187 /* LSE atomics */ \
188 #op_lse #sfx "\t%" #w "[val], %" #w "[tmp], [%[addr]]\n"\
189 __nops(3)) \
190 __PCPU_GPRS_END("%[gprs]") \
191 : [gprs] "=Qo" (*gprs), \
192 [addr] "=&r" (addr), \
193 [off] "=&r" (off), \
194 [loop] "=&r" (loop), \
195 [tmp] "=&r" (tmp) \
196 : [pcp] "r" (pcp), \
197 [val] "r" ((u##sz)(val)) \
198 : "memory" \
199 ); \
200 }
201
202 #define __PERCPU_RET_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
203 static inline u##sz \
204 __percpu_##name##_return_case_##sz(void __percpu *pcp, unsigned long val) \
205 { \
206 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \
207 unsigned long addr; \
208 unsigned long off; \
209 unsigned int loop; \
210 u##sz ret; \
211 \
212 asm volatile ( \
213 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
214 ARM64_LSE_ATOMIC_INSN( \
215 /* LL/SC */ \
216 "1: ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n" \
217 #op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
218 " stxr" #sfx "\t%w[loop], %" #w "[ret], [%[addr]]\n" \
219 " cbnz %w[loop], 1b", \
220 /* LSE atomics */ \
221 #op_lse #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n"\
222 #op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
223 __nops(2)) \
224 __PCPU_GPRS_END("%[gprs]") \
225 : [gprs] "=Qo" (*gprs), \
226 [addr] "=&r" (addr), \
227 [off] "=&r" (off), \
228 [loop] "=&r" (loop), \
229 [ret] "=&r" (ret) \
230 : [pcp] "r" (pcp), \
231 [val] "r" ((u##sz)(val)) \
232 : "memory" \
233 ); \
234 \
235 return ret; \
236 }
237
238 #define PERCPU_OP(name, op_llsc, op_lse) \
239 __PERCPU_OP_CASE(w, b, name, 8, op_llsc, op_lse) \
240 __PERCPU_OP_CASE(w, h, name, 16, op_llsc, op_lse) \
241 __PERCPU_OP_CASE(w, , name, 32, op_llsc, op_lse) \
242 __PERCPU_OP_CASE( , , name, 64, op_llsc, op_lse)
243
244 #define PERCPU_RET_OP(name, op_llsc, op_lse) \
245 __PERCPU_RET_OP_CASE(w, b, name, 8, op_llsc, op_lse) \
246 __PERCPU_RET_OP_CASE(w, h, name, 16, op_llsc, op_lse) \
247 __PERCPU_RET_OP_CASE(w, , name, 32, op_llsc, op_lse) \
248 __PERCPU_RET_OP_CASE( , , name, 64, op_llsc, op_lse)
249
250 PERCPU_RW_OPS(w, b, 8)
251 PERCPU_RW_OPS(w, h, 16)
252 PERCPU_RW_OPS(w, , 32)
253 PERCPU_RW_OPS( , , 64)
254
255 /*
256 * Use value-returning atomics for CPU-local ops as they are more likely
257 * to execute "near" to the CPU (e.g. in L1$).
258 *
259 * https://lore.kernel.org/r/e7d539ed-ced0-4b96-8ecd-048a5b803b85@paulmck-laptop
260 */
PERCPU_OP(add,add,ldadd)261 PERCPU_OP(add, add, ldadd)
262 PERCPU_OP(andnot, bic, ldclr)
263 PERCPU_OP(or, orr, ldset)
264 PERCPU_RET_OP(add, add, ldadd)
265
266 #undef PERCPU_RW_OPS
267 #undef __PERCPU_OP_CASE
268 #undef __PERCPU_RET_OP_CASE
269 #undef PERCPU_OP
270 #undef PERCPU_RET_OP
271
272 #define PERCPU_XCHG_OP(w, sfx, sz) \
273 static inline unsigned long \
274 __percpu_xchg_case_##sz(void __percpu *pcp, u##sz val) \
275 { \
276 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \
277 unsigned long addr; \
278 unsigned long off; \
279 unsigned int loop; \
280 unsigned long ret; \
281 \
282 asm volatile ( \
283 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
284 ARM64_LSE_ATOMIC_INSN( \
285 /* LL/SC */ \
286 " prfm pstl1strm, [%[addr]]\n" \
287 "1: ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n" \
288 " stxr" #sfx "\t%w[loop], %" #w "[val], [%[addr]]\n" \
289 " cbnz %w[loop], 1b\n" \
290 , \
291 /* LSE atomics */ \
292 " swp" #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n" \
293 __nops(3) \
294 ) \
295 __PCPU_GPRS_END("%[gprs]") \
296 : [gprs] "=Qo" (*gprs), \
297 [addr] "=&r" (addr), \
298 [off] "=&r" (off), \
299 [loop] "=&r" (loop), \
300 [ret] "=&r" (ret) \
301 : [pcp] "r" (pcp), \
302 [val] "r" (val) \
303 : "memory" \
304 ); \
305 \
306 return ret; \
307 }
308
309 #define PERCPU_CMPXCHG_OP(w, sfx, sz) \
310 static inline unsigned long \
311 __percpu_cmpxchg_case_##sz(void __percpu *pcp, \
312 u##sz old, \
313 u##sz new) \
314 { \
315 /* \
316 * Sub-word sizes require zero extension so that EOR+CBNZ don't \
317 * consume non-zero upper bits of the register containing "old".\
318 */ \
319 xwreg_t(w) cmpval = xwreg_zero_extend(old, w, sz); \
320 u16 *gprs = ¤t_thread_info()->pcpu_gprs; \
321 unsigned long addr; \
322 unsigned long off; \
323 unsigned long tmp; \
324 unsigned long oldval; \
325 \
326 asm volatile ( \
327 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
328 ARM64_LSE_ATOMIC_INSN( \
329 /* LL/SC */ \
330 " prfm pstl1strm, [%[addr]]\n" \
331 "1: ldxr" #sfx "\t%" #w "[oldval], [%[addr]]\n" \
332 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
333 " cbnz %" #w "[tmp], 2f\n" \
334 " stxr" #sfx "\t%w[tmp], %" #w "[new], [%[addr]]\n" \
335 " cbnz %w[tmp], 1b\n" \
336 "2:\n" \
337 , \
338 /* LSE atomics */ \
339 " mov %" #w "[oldval], %" #w "[old]\n" \
340 " cas" #sfx "\t%" #w "[oldval], %" #w "[new], [%[addr]]\n"\
341 __nops(4) \
342 ) \
343 __PCPU_GPRS_END("%[gprs]") \
344 : [gprs] "=Qo" (*gprs), \
345 [addr] "=&r" (addr), \
346 [off] "=&r" (off), \
347 [tmp] "=&r" (tmp), \
348 [oldval] "=&r" (oldval) \
349 : [pcp] "r" (pcp), \
350 [old] "r" (cmpval), \
351 [new] "r" (new) \
352 : "memory" \
353 ); \
354 \
355 return oldval; \
356 }
357
358 PERCPU_XCHG_OP(w, b, 8)
359 PERCPU_XCHG_OP(w, h, 16)
360 PERCPU_XCHG_OP(w, , 32)
361 PERCPU_XCHG_OP(x, , 64)
362
363 PERCPU_CMPXCHG_OP(w, b, 8)
364 PERCPU_CMPXCHG_OP(w, h, 16)
365 PERCPU_CMPXCHG_OP(w, , 32)
366 PERCPU_CMPXCHG_OP(x, , 64)
367
368 #undef PERCPU_XCHG_OP
369 #undef PERCPU_CMPXCHG_OP
370
371 #define _pcp_wrap(op, pcp, ...) \
372 ({ \
373 op(&(pcp), __VA_ARGS__); \
374 })
375
376 #define _pcp_wrap_return(op, pcp, args...) \
377 ({ \
378 (typeof(pcp))op(&(pcp), ##args); \
379 })
380
381 #define _pcp_wrap_xchg(op, pcp, val) \
382 ({ \
383 (typeof(pcp))op(&(pcp), (unsigned long)(val)); \
384 })
385
386 #define _pcp_wrap_cmpxchg(op, pcp, old, new) \
387 ({ \
388 (typeof(pcp))op(&(pcp), (unsigned long)(old), \
389 (unsigned long)(new)); \
390 })
391
392 #define this_cpu_read_1(pcp) \
393 _pcp_wrap_return(__percpu_read_8, pcp)
394 #define this_cpu_read_2(pcp) \
395 _pcp_wrap_return(__percpu_read_16, pcp)
396 #define this_cpu_read_4(pcp) \
397 _pcp_wrap_return(__percpu_read_32, pcp)
398 #define this_cpu_read_8(pcp) \
399 _pcp_wrap_return(__percpu_read_64, pcp)
400
401 #define this_cpu_write_1(pcp, val) \
402 _pcp_wrap(__percpu_write_8, pcp, (unsigned long)(val))
403 #define this_cpu_write_2(pcp, val) \
404 _pcp_wrap(__percpu_write_16, pcp, (unsigned long)(val))
405 #define this_cpu_write_4(pcp, val) \
406 _pcp_wrap(__percpu_write_32, pcp, (unsigned long)(val))
407 #define this_cpu_write_8(pcp, val) \
408 _pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val))
409
410 #define this_cpu_add_1(pcp, val) \
411 _pcp_wrap(__percpu_add_case_8, pcp, val)
412 #define this_cpu_add_2(pcp, val) \
413 _pcp_wrap(__percpu_add_case_16, pcp, val)
414 #define this_cpu_add_4(pcp, val) \
415 _pcp_wrap(__percpu_add_case_32, pcp, val)
416 #define this_cpu_add_8(pcp, val) \
417 _pcp_wrap(__percpu_add_case_64, pcp, val)
418
419 #define this_cpu_add_return_1(pcp, val) \
420 _pcp_wrap_return(__percpu_add_return_case_8, pcp, val)
421 #define this_cpu_add_return_2(pcp, val) \
422 _pcp_wrap_return(__percpu_add_return_case_16, pcp, val)
423 #define this_cpu_add_return_4(pcp, val) \
424 _pcp_wrap_return(__percpu_add_return_case_32, pcp, val)
425 #define this_cpu_add_return_8(pcp, val) \
426 _pcp_wrap_return(__percpu_add_return_case_64, pcp, val)
427
428 #define this_cpu_and_1(pcp, val) \
429 _pcp_wrap(__percpu_andnot_case_8, pcp, ~(u8)(val))
430 #define this_cpu_and_2(pcp, val) \
431 _pcp_wrap(__percpu_andnot_case_16, pcp, ~(u16)(val))
432 #define this_cpu_and_4(pcp, val) \
433 _pcp_wrap(__percpu_andnot_case_32, pcp, ~(u32)(val))
434 #define this_cpu_and_8(pcp, val) \
435 _pcp_wrap(__percpu_andnot_case_64, pcp, ~(u64)(val))
436
437 #define this_cpu_or_1(pcp, val) \
438 _pcp_wrap(__percpu_or_case_8, pcp, val)
439 #define this_cpu_or_2(pcp, val) \
440 _pcp_wrap(__percpu_or_case_16, pcp, val)
441 #define this_cpu_or_4(pcp, val) \
442 _pcp_wrap(__percpu_or_case_32, pcp, val)
443 #define this_cpu_or_8(pcp, val) \
444 _pcp_wrap(__percpu_or_case_64, pcp, val)
445
446 #define this_cpu_xchg_1(pcp, val) \
447 _pcp_wrap_xchg(__percpu_xchg_case_8, pcp, val)
448 #define this_cpu_xchg_2(pcp, val) \
449 _pcp_wrap_xchg(__percpu_xchg_case_16, pcp, val)
450 #define this_cpu_xchg_4(pcp, val) \
451 _pcp_wrap_xchg(__percpu_xchg_case_32, pcp, val)
452 #define this_cpu_xchg_8(pcp, val) \
453 _pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val)
454
455 #define this_cpu_cmpxchg_1(pcp, o, n) \
456 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_8, pcp, o, n)
457 #define this_cpu_cmpxchg_2(pcp, o, n) \
458 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_16, pcp, o, n)
459 #define this_cpu_cmpxchg_4(pcp, o, n) \
460 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_32, pcp, o, n)
461 #define this_cpu_cmpxchg_8(pcp, o, n) \
462 _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_64, pcp, o, n)
463
464 #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
465
466 static inline u128
467 __percpu_cmpxchg_128(void __percpu *pcp, u128 old, u128 new)
468 {
469 u16 *gprs = ¤t_thread_info()->pcpu_gprs;
470 unsigned long addr;
471 unsigned long off;
472 union __u128_halves r, o = { .full = (old) },
473 n = { .full = (new) };
474 register unsigned long ol asm ("x0") = o.low;
475 register unsigned long oh asm ("x1") = o.high;
476 register unsigned long nl asm ("x2") = n.low;
477 register unsigned long nh asm ("x3") = n.high;
478 unsigned long rl, rh;
479 unsigned long tmp;
480
481 asm volatile (
482 __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")
483 ARM64_LSE_ATOMIC_INSN(
484 /* LL/SC */
485 " prfm pstl1strm, [%[addr]]\n"
486 "1: ldxp %[rl], %[rh], [%[addr]]\n"
487 " cmp %[rl], %[ol]\n"
488 " ccmp %[rh], %[oh], 0, eq\n"
489 " b.ne 2f\n"
490 " stxp %w[tmp], %[nl], %[nh], [%[addr]]\n"
491 " cbnz %w[tmp], 1b\n"
492 "2:\n"
493 ,
494 /* LSE atomics */
495 " casp %[ol], %[oh], %[nl], %[nh], [%[addr]]\n"
496 " mov %[rl], %[ol]\n"
497 " mov %[rh], %[oh]\n"
498 __nops(4)
499 )
500 __PCPU_GPRS_END("%[gprs]")
501 : [gprs] "=Qo" (*gprs),
502 [addr] "=&r" (addr),
503 [off] "=&r" (off),
504 [tmp] "=&r" (tmp),
505 [ol] "+&r" (ol),
506 [oh] "+&r" (oh),
507 [rl] "=&r" (rl),
508 [rh] "=&r" (rh)
509 : [pcp] "r" (pcp),
510 [nl] "r" (nl),
511 [nh] "r" (nh)
512 : "memory", "cc"
513 );
514
515 r.low = rl;
516 r.high = rh;
517
518 return r.full;
519 }
520
521 #define this_cpu_cmpxchg128(pcp, o, n) \
522 _pcp_wrap_return(__percpu_cmpxchg_128, pcp, o, n)
523
524 #ifdef __KVM_NVHE_HYPERVISOR__
525 extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
526 #define __per_cpu_offset
527 #define per_cpu_offset(cpu) __hyp_per_cpu_offset((cpu))
528 #endif
529
530 #include <asm-generic/percpu.h>
531
532 /* Redefine macros for nVHE hyp under DEBUG_PREEMPT to avoid its dependencies. */
533 #if defined(__KVM_NVHE_HYPERVISOR__) && defined(CONFIG_DEBUG_PREEMPT)
534 #undef this_cpu_ptr
535 #define this_cpu_ptr raw_cpu_ptr
536 #undef __this_cpu_read
537 #define __this_cpu_read raw_cpu_read
538 #undef __this_cpu_write
539 #define __this_cpu_write raw_cpu_write
540 #endif
541
542 #endif /* __ASM_PERCPU_H */
543