1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-arm64-bits.h
4 *
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */
8
9 #include "rseq-bits-template.h"
10
11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
13
14 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
16 {
17 RSEQ_INJECT_C(9)
18
19 __asm__ __volatile__ goto (
20 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
21 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
22 #ifdef RSEQ_COMPARE_TWICE
23 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
24 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
25 #endif
26 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
28 RSEQ_INJECT_ASM(3)
29 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
30 RSEQ_INJECT_ASM(4)
31 #ifdef RSEQ_COMPARE_TWICE
32 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
33 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
34 #endif
35 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
36 RSEQ_INJECT_ASM(5)
37 RSEQ_ASM_DEFINE_ABORT(4, abort)
38 : /* gcc asm goto does not allow outputs */
39 : [cpu_id] "r" (cpu),
40 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
41 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
42 [v] "Qo" (*v),
43 [expect] "r" (expect),
44 [newv] "r" (newv)
45 RSEQ_INJECT_INPUT
46 : "memory", RSEQ_ASM_TMP_REG
47 : abort, cmpfail
48 #ifdef RSEQ_COMPARE_TWICE
49 , error1, error2
50 #endif
51 );
52 rseq_after_asm_goto();
53 return 0;
54 abort:
55 rseq_after_asm_goto();
56 RSEQ_INJECT_FAILED
57 return -1;
58 cmpfail:
59 rseq_after_asm_goto();
60 return 1;
61 #ifdef RSEQ_COMPARE_TWICE
62 error1:
63 rseq_after_asm_goto();
64 rseq_bug("cpu_id comparison failed");
65 error2:
66 rseq_after_asm_goto();
67 rseq_bug("expected value comparison failed");
68 #endif
69 }
70
71 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)72 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
73 long voffp, intptr_t *load, int cpu)
74 {
75 RSEQ_INJECT_C(9)
76
77 __asm__ __volatile__ goto (
78 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
79 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
80 #ifdef RSEQ_COMPARE_TWICE
81 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
82 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
83 #endif
84 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
85 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
86 RSEQ_INJECT_ASM(3)
87 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
88 RSEQ_INJECT_ASM(4)
89 #ifdef RSEQ_COMPARE_TWICE
90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
91 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
92 #endif
93 RSEQ_ASM_OP_R_LOAD(v)
94 RSEQ_ASM_OP_R_STORE(load)
95 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
96 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
97 RSEQ_INJECT_ASM(5)
98 RSEQ_ASM_DEFINE_ABORT(4, abort)
99 : /* gcc asm goto does not allow outputs */
100 : [cpu_id] "r" (cpu),
101 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
102 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
103 [v] "Qo" (*v),
104 [expectnot] "r" (expectnot),
105 [load] "Qo" (*load),
106 [voffp] "r" (voffp)
107 RSEQ_INJECT_INPUT
108 : "memory", RSEQ_ASM_TMP_REG
109 : abort, cmpfail
110 #ifdef RSEQ_COMPARE_TWICE
111 , error1, error2
112 #endif
113 );
114 rseq_after_asm_goto();
115 return 0;
116 abort:
117 rseq_after_asm_goto();
118 RSEQ_INJECT_FAILED
119 return -1;
120 cmpfail:
121 rseq_after_asm_goto();
122 return 1;
123 #ifdef RSEQ_COMPARE_TWICE
124 error1:
125 rseq_after_asm_goto();
126 rseq_bug("cpu_id comparison failed");
127 error2:
128 rseq_after_asm_goto();
129 rseq_bug("expected value comparison failed");
130 #endif
131 }
132
133 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)134 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
135 {
136 RSEQ_INJECT_C(9)
137
138 __asm__ __volatile__ goto (
139 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
140 #ifdef RSEQ_COMPARE_TWICE
141 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
142 #endif
143 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
144 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
145 RSEQ_INJECT_ASM(3)
146 #ifdef RSEQ_COMPARE_TWICE
147 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
148 #endif
149 RSEQ_ASM_OP_R_LOAD(v)
150 RSEQ_ASM_OP_R_ADD(count)
151 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
152 RSEQ_INJECT_ASM(4)
153 RSEQ_ASM_DEFINE_ABORT(4, abort)
154 : /* gcc asm goto does not allow outputs */
155 : [cpu_id] "r" (cpu),
156 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
157 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
158 [v] "Qo" (*v),
159 [count] "r" (count)
160 RSEQ_INJECT_INPUT
161 : "memory", RSEQ_ASM_TMP_REG
162 : abort
163 #ifdef RSEQ_COMPARE_TWICE
164 , error1
165 #endif
166 );
167 rseq_after_asm_goto();
168 return 0;
169 abort:
170 rseq_after_asm_goto();
171 RSEQ_INJECT_FAILED
172 return -1;
173 #ifdef RSEQ_COMPARE_TWICE
174 error1:
175 rseq_after_asm_goto();
176 rseq_bug("cpu_id comparison failed");
177 #endif
178 }
179
180 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)181 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
182 intptr_t *v2, intptr_t expect2,
183 intptr_t newv, int cpu)
184 {
185 RSEQ_INJECT_C(9)
186
187 __asm__ __volatile__ goto (
188 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
189 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
190 #ifdef RSEQ_COMPARE_TWICE
191 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
192 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
193 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
194 #endif
195 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
196 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
197 RSEQ_INJECT_ASM(3)
198 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
199 RSEQ_INJECT_ASM(4)
200 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
201 RSEQ_INJECT_ASM(5)
202 #ifdef RSEQ_COMPARE_TWICE
203 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
204 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
205 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
206 #endif
207 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
208 RSEQ_INJECT_ASM(6)
209 RSEQ_ASM_DEFINE_ABORT(4, abort)
210 : /* gcc asm goto does not allow outputs */
211 : [cpu_id] "r" (cpu),
212 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
213 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
214 [v] "Qo" (*v),
215 [expect] "r" (expect),
216 [v2] "Qo" (*v2),
217 [expect2] "r" (expect2),
218 [newv] "r" (newv)
219 RSEQ_INJECT_INPUT
220 : "memory", RSEQ_ASM_TMP_REG
221 : abort, cmpfail
222 #ifdef RSEQ_COMPARE_TWICE
223 , error1, error2, error3
224 #endif
225 );
226 rseq_after_asm_goto();
227 return 0;
228 abort:
229 rseq_after_asm_goto();
230 RSEQ_INJECT_FAILED
231 return -1;
232 cmpfail:
233 rseq_after_asm_goto();
234 return 1;
235 #ifdef RSEQ_COMPARE_TWICE
236 error1:
237 rseq_after_asm_goto();
238 rseq_bug("cpu_id comparison failed");
239 error2:
240 rseq_after_asm_goto();
241 rseq_bug("expected value comparison failed");
242 error3:
243 rseq_after_asm_goto();
244 rseq_bug("2nd expected value comparison failed");
245 #endif
246 }
247
248 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
249 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
250
251 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
252 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
253
254 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)255 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
256 intptr_t *v2, intptr_t newv2,
257 intptr_t newv, int cpu)
258 {
259 RSEQ_INJECT_C(9)
260
261 __asm__ __volatile__ goto (
262 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
263 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
264 #ifdef RSEQ_COMPARE_TWICE
265 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
266 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
267 #endif
268 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
269 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
270 RSEQ_INJECT_ASM(3)
271 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
272 RSEQ_INJECT_ASM(4)
273 #ifdef RSEQ_COMPARE_TWICE
274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
275 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
276 #endif
277 RSEQ_ASM_OP_STORE(newv2, v2)
278 RSEQ_INJECT_ASM(5)
279 #ifdef RSEQ_TEMPLATE_MO_RELEASE
280 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
281 #else
282 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
283 #endif
284 RSEQ_INJECT_ASM(6)
285 RSEQ_ASM_DEFINE_ABORT(4, abort)
286 : /* gcc asm goto does not allow outputs */
287 : [cpu_id] "r" (cpu),
288 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
289 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
290 [expect] "r" (expect),
291 [v] "Qo" (*v),
292 [newv] "r" (newv),
293 [v2] "Qo" (*v2),
294 [newv2] "r" (newv2)
295 RSEQ_INJECT_INPUT
296 : "memory", RSEQ_ASM_TMP_REG
297 : abort, cmpfail
298 #ifdef RSEQ_COMPARE_TWICE
299 , error1, error2
300 #endif
301 );
302 rseq_after_asm_goto();
303 return 0;
304 abort:
305 rseq_after_asm_goto();
306 RSEQ_INJECT_FAILED
307 return -1;
308 cmpfail:
309 rseq_after_asm_goto();
310 return 1;
311 #ifdef RSEQ_COMPARE_TWICE
312 error1:
313 rseq_after_asm_goto();
314 rseq_bug("cpu_id comparison failed");
315 error2:
316 rseq_after_asm_goto();
317 rseq_bug("expected value comparison failed");
318 #endif
319 }
320
321 static inline __attribute__((always_inline))
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)322 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
323 void *dst, void *src, size_t len,
324 intptr_t newv, int cpu)
325 {
326 RSEQ_INJECT_C(9)
327
328 __asm__ __volatile__ goto (
329 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
330 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
331 #ifdef RSEQ_COMPARE_TWICE
332 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
333 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
334 #endif
335 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
337 RSEQ_INJECT_ASM(3)
338 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
339 RSEQ_INJECT_ASM(4)
340 #ifdef RSEQ_COMPARE_TWICE
341 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
342 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
343 #endif
344 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
345 RSEQ_INJECT_ASM(5)
346 #ifdef RSEQ_TEMPLATE_MO_RELEASE
347 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
348 #else
349 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
350 #endif
351 RSEQ_INJECT_ASM(6)
352 RSEQ_ASM_DEFINE_ABORT(4, abort)
353 : /* gcc asm goto does not allow outputs */
354 : [cpu_id] "r" (cpu),
355 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
356 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
357 [expect] "r" (expect),
358 [v] "Qo" (*v),
359 [newv] "r" (newv),
360 [dst] "r" (dst),
361 [src] "r" (src),
362 [len] "r" (len)
363 RSEQ_INJECT_INPUT
364 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
365 : abort, cmpfail
366 #ifdef RSEQ_COMPARE_TWICE
367 , error1, error2
368 #endif
369 );
370 rseq_after_asm_goto();
371 return 0;
372 abort:
373 rseq_after_asm_goto();
374 RSEQ_INJECT_FAILED
375 return -1;
376 cmpfail:
377 rseq_after_asm_goto();
378 return 1;
379 #ifdef RSEQ_COMPARE_TWICE
380 error1:
381 rseq_after_asm_goto();
382 rseq_bug("cpu_id comparison failed");
383 error2:
384 rseq_after_asm_goto();
385 rseq_bug("expected value comparison failed");
386 #endif
387 }
388
389 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
390 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
391
392 #include "rseq-bits-reset.h"
393