1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5 #ifndef __ASM_PERCPU_H
6 #define __ASM_PERCPU_H
7
8 #include <asm/loongarch.h>
9
10 /*
11 * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
12 * the loading address of main kernel image, but far from where the modules are
13 * loaded. Tell the compiler this fact when using explicit relocs.
14 */
15 #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) && defined(CONFIG_64BIT)
16 # if __has_attribute(model)
17 # define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
18 # else
19 # error compiler support for the model attribute is necessary when a recent assembler is used
20 # endif
21 #endif
22
23 /* Use r21 for fast access */
24 register unsigned long __my_cpu_offset __asm__("$r21");
25
set_my_cpu_offset(unsigned long off)26 static inline void set_my_cpu_offset(unsigned long off)
27 {
28 __my_cpu_offset = off;
29 csr_write(off, PERCPU_BASE_KS);
30 }
31
32 #define __my_cpu_offset \
33 ({ \
34 __asm__ __volatile__("":"+r"(__my_cpu_offset)); \
35 __my_cpu_offset; \
36 })
37
38 #ifdef CONFIG_CPU_HAS_AMO
39
40 #define PERCPU_OP(op, asm_op, c_op) \
41 static __always_inline unsigned long __percpu_##op(void *ptr, \
42 unsigned long val, int size) \
43 { \
44 unsigned long ret; \
45 \
46 switch (size) { \
47 case 4: \
48 __asm__ __volatile__( \
49 "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \
50 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \
51 : [val] "r" (val)); \
52 break; \
53 case 8: \
54 __asm__ __volatile__( \
55 "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \
56 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \
57 : [val] "r" (val)); \
58 break; \
59 default: \
60 ret = 0; \
61 BUILD_BUG(); \
62 } \
63 \
64 return ret c_op val; \
65 }
66
67 PERCPU_OP(add, add, +)
68 PERCPU_OP(and, and, &)
69 PERCPU_OP(or, or, |)
70 #undef PERCPU_OP
71
72 #endif
73
74 #ifdef CONFIG_64BIT
75
76 #define __pcpu_op_1(op) op ".b "
77 #define __pcpu_op_2(op) op ".h "
78 #define __pcpu_op_4(op) op ".w "
79 #define __pcpu_op_8(op) op ".d "
80
81 #define _percpu_read(size, _pcp) \
82 ({ \
83 typeof(_pcp) __pcp_ret; \
84 \
85 __asm__ __volatile__( \
86 __pcpu_op_##size("ldx") "%[ret], $r21, %[ptr] \n" \
87 : [ret] "=&r"(__pcp_ret) \
88 : [ptr] "r"(&(_pcp)) \
89 : "memory"); \
90 \
91 __pcp_ret; \
92 })
93
94 #define _percpu_write(size, _pcp, _val) \
95 do { \
96 __asm__ __volatile__( \
97 __pcpu_op_##size("stx") "%[val], $r21, %[ptr] \n" \
98 : \
99 : [val] "r"(_val), [ptr] "r"(&(_pcp)) \
100 : "memory"); \
101 } while (0)
102
103 #endif
104
105 #define __percpu_xchg __arch_xchg
106
107 /* this_cpu_cmpxchg */
108 #define _protect_cmpxchg_local(pcp, o, n) \
109 ({ \
110 typeof(*raw_cpu_ptr(&(pcp))) __ret; \
111 preempt_disable_notrace(); \
112 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
113 preempt_enable_notrace(); \
114 __ret; \
115 })
116
117 #define _pcp_protect(operation, pcp, val) \
118 ({ \
119 typeof(pcp) __retval; \
120 preempt_disable_notrace(); \
121 __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \
122 (val), sizeof(pcp)); \
123 preempt_enable_notrace(); \
124 __retval; \
125 })
126
127 #ifdef CONFIG_CPU_HAS_AMO
128
129 #define _percpu_add(pcp, val) \
130 _pcp_protect(__percpu_add, pcp, val)
131
132 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
133
134 #define _percpu_and(pcp, val) \
135 _pcp_protect(__percpu_and, pcp, val)
136
137 #define _percpu_or(pcp, val) \
138 _pcp_protect(__percpu_or, pcp, val)
139
140 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
141 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
142
143 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
144 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
145
146 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
147 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
148
149 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
150 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
151
152 #endif
153
154 #ifdef CONFIG_64BIT
155
156 #define this_cpu_read_1(pcp) _percpu_read(1, pcp)
157 #define this_cpu_read_2(pcp) _percpu_read(2, pcp)
158 #define this_cpu_read_4(pcp) _percpu_read(4, pcp)
159 #define this_cpu_read_8(pcp) _percpu_read(8, pcp)
160
161 #define this_cpu_write_1(pcp, val) _percpu_write(1, pcp, val)
162 #define this_cpu_write_2(pcp, val) _percpu_write(2, pcp, val)
163 #define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val)
164 #define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val)
165
166 #endif
167
168 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \
169 _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
170
171 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
172 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
173 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
174 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
175
176 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
177 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
178 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
179 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
180
181 #include <asm-generic/percpu.h>
182
183 #endif /* __ASM_PERCPU_H */
184