1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3 #include "rseq-bits-template.h" 4 5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 7 8 static inline __always_inline 9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, 10 int cpu) 11 { 12 RSEQ_INJECT_C(9) 13 14 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 15 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 16 #ifdef RSEQ_COMPARE_TWICE 17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 18 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 19 #endif 20 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 21 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 22 RSEQ_INJECT_ASM(3) 23 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 24 RSEQ_INJECT_ASM(4) 25 #ifdef RSEQ_COMPARE_TWICE 26 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 27 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 28 #endif 29 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3) 30 RSEQ_INJECT_ASM(5) 31 RSEQ_ASM_DEFINE_ABORT(4, abort) 32 : /* gcc asm goto does not allow outputs */ 33 : [cpu_id] "r" (cpu), 34 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 35 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 36 [v] "m" (*v), 37 [expect] "r" (expect), 38 [newv] "r" (newv) 39 RSEQ_INJECT_INPUT 40 : "memory", RSEQ_ASM_TMP_REG_1 41 RSEQ_INJECT_CLOBBER 42 : abort, cmpfail 43 #ifdef RSEQ_COMPARE_TWICE 44 , error1, error2 45 #endif 46 ); 47 48 return 0; 49 abort: 50 RSEQ_INJECT_FAILED 51 return -1; 52 cmpfail: 53 return 1; 54 #ifdef RSEQ_COMPARE_TWICE 55 error1: 56 rseq_bug("cpu_id comparison failed"); 57 error2: 58 rseq_bug("expected value comparison failed"); 59 #endif 60 } 61 62 static inline __always_inline 63 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 64 off_t voffp, intptr_t *load, int cpu) 65 { 66 RSEQ_INJECT_C(9) 67 68 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 69 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 70 #ifdef RSEQ_COMPARE_TWICE 71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 72 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 73 #endif 74 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 75 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 76 RSEQ_INJECT_ASM(3) 77 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") 78 RSEQ_INJECT_ASM(4) 79 #ifdef RSEQ_COMPARE_TWICE 80 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 81 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") 82 #endif 83 RSEQ_ASM_OP_R_LOAD(v) 84 RSEQ_ASM_OP_R_STORE(load) 85 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 86 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 87 RSEQ_INJECT_ASM(5) 88 RSEQ_ASM_DEFINE_ABORT(4, abort) 89 : /* gcc asm goto does not allow outputs */ 90 : [cpu_id] "r" (cpu), 91 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 92 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 93 [v] "m" (*v), 94 [expectnot] "r" (expectnot), 95 [load] "m" (*load), 96 [voffp] "Ir" (voffp) 97 RSEQ_INJECT_INPUT 98 : "memory", RSEQ_ASM_TMP_REG_1 99 RSEQ_INJECT_CLOBBER 100 : abort, cmpfail 101 #ifdef RSEQ_COMPARE_TWICE 102 , error1, error2 103 #endif 104 ); 105 return 0; 106 abort: 107 RSEQ_INJECT_FAILED 108 return -1; 109 cmpfail: 110 return 1; 111 #ifdef RSEQ_COMPARE_TWICE 112 error1: 113 rseq_bug("cpu_id comparison failed"); 114 error2: 115 rseq_bug("expected value comparison failed"); 116 #endif 117 } 118 119 static inline __always_inline 120 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 121 { 122 RSEQ_INJECT_C(9) 123 124 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 125 #ifdef RSEQ_COMPARE_TWICE 126 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 127 #endif 128 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 129 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 130 RSEQ_INJECT_ASM(3) 131 #ifdef RSEQ_COMPARE_TWICE 132 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 133 #endif 134 RSEQ_ASM_OP_R_LOAD(v) 135 RSEQ_ASM_OP_R_ADD(count) 136 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 137 RSEQ_INJECT_ASM(4) 138 RSEQ_ASM_DEFINE_ABORT(4, abort) 139 : /* gcc asm goto does not allow outputs */ 140 : [cpu_id] "r" (cpu), 141 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 142 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 143 [v] "m" (*v), 144 [count] "r" (count) 145 RSEQ_INJECT_INPUT 146 : "memory", RSEQ_ASM_TMP_REG_1 147 RSEQ_INJECT_CLOBBER 148 : abort 149 #ifdef RSEQ_COMPARE_TWICE 150 , error1 151 #endif 152 ); 153 return 0; 154 abort: 155 RSEQ_INJECT_FAILED 156 return -1; 157 #ifdef RSEQ_COMPARE_TWICE 158 error1: 159 rseq_bug("cpu_id comparison failed"); 160 #endif 161 } 162 163 static inline __always_inline 164 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 165 intptr_t *v2, intptr_t expect2, 166 intptr_t newv, int cpu) 167 { 168 RSEQ_INJECT_C(9) 169 170 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 171 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 172 #ifdef RSEQ_COMPARE_TWICE 173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 175 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") 176 #endif 177 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 178 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 179 RSEQ_INJECT_ASM(3) 180 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 181 RSEQ_INJECT_ASM(4) 182 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") 183 RSEQ_INJECT_ASM(5) 184 #ifdef RSEQ_COMPARE_TWICE 185 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 186 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 187 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") 188 #endif 189 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3) 190 RSEQ_INJECT_ASM(6) 191 RSEQ_ASM_DEFINE_ABORT(4, abort) 192 : /* gcc asm goto does not allow outputs */ 193 : [cpu_id] "r" (cpu), 194 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 195 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 196 [v] "m" (*v), 197 [expect] "r" (expect), 198 [v2] "m" (*v2), 199 [expect2] "r" (expect2), 200 [newv] "r" (newv) 201 RSEQ_INJECT_INPUT 202 : "memory", RSEQ_ASM_TMP_REG_1 203 RSEQ_INJECT_CLOBBER 204 : abort, cmpfail 205 #ifdef RSEQ_COMPARE_TWICE 206 , error1, error2, error3 207 #endif 208 ); 209 210 return 0; 211 abort: 212 RSEQ_INJECT_FAILED 213 return -1; 214 cmpfail: 215 return 1; 216 #ifdef RSEQ_COMPARE_TWICE 217 error1: 218 rseq_bug("cpu_id comparison failed"); 219 error2: 220 rseq_bug("expected value comparison failed"); 221 error3: 222 rseq_bug("2nd expected value comparison failed"); 223 #endif 224 } 225 226 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 227 228 /* 229 * pval = *(ptr+off) 230 * *pval += inc; 231 */ 232 static inline __always_inline 233 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, 234 int cpu) 235 { 236 RSEQ_INJECT_C(9) 237 238 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 239 #ifdef RSEQ_COMPARE_TWICE 240 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 241 #endif 242 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 244 RSEQ_INJECT_ASM(3) 245 #ifdef RSEQ_COMPARE_TWICE 246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 247 #endif 248 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, inc, 3) 249 RSEQ_INJECT_ASM(4) 250 RSEQ_ASM_DEFINE_ABORT(4, abort) 251 : /* gcc asm goto does not allow outputs */ 252 : [cpu_id] "r" (cpu), 253 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 254 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 255 [ptr] "r" (ptr), 256 [off] "r" (off), 257 [inc] "r" (inc) 258 RSEQ_INJECT_INPUT 259 : "memory", RSEQ_ASM_TMP_REG_1 260 RSEQ_INJECT_CLOBBER 261 : abort 262 #ifdef RSEQ_COMPARE_TWICE 263 , error1 264 #endif 265 ); 266 return 0; 267 abort: 268 RSEQ_INJECT_FAILED 269 return -1; 270 #ifdef RSEQ_COMPARE_TWICE 271 error1: 272 rseq_bug("cpu_id comparison failed"); 273 #endif 274 } 275 276 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 277 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 278 279 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 280 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 281 282 static inline __always_inline 283 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 284 intptr_t *v2, intptr_t newv2, 285 intptr_t newv, int cpu) 286 { 287 RSEQ_INJECT_C(9) 288 289 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 291 #ifdef RSEQ_COMPARE_TWICE 292 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 293 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 294 #endif 295 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 296 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 297 RSEQ_INJECT_ASM(3) 298 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 299 RSEQ_INJECT_ASM(4) 300 #ifdef RSEQ_COMPARE_TWICE 301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 302 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 303 #endif 304 RSEQ_ASM_OP_STORE(v2, newv2) 305 RSEQ_INJECT_ASM(5) 306 #ifdef RSEQ_TEMPLATE_MO_RELEASE 307 RSEQ_ASM_OP_FINAL_STORE_RELEASE(v, newv, 3) 308 #else 309 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3) 310 #endif 311 RSEQ_INJECT_ASM(6) 312 RSEQ_ASM_DEFINE_ABORT(4, abort) 313 : /* gcc asm goto does not allow outputs */ 314 : [cpu_id] "r" (cpu), 315 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 316 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 317 [expect] "r" (expect), 318 [v] "m" (*v), 319 [newv] "r" (newv), 320 [v2] "m" (*v2), 321 [newv2] "r" (newv2) 322 RSEQ_INJECT_INPUT 323 : "memory", RSEQ_ASM_TMP_REG_1 324 RSEQ_INJECT_CLOBBER 325 : abort, cmpfail 326 #ifdef RSEQ_COMPARE_TWICE 327 , error1, error2 328 #endif 329 ); 330 331 return 0; 332 abort: 333 RSEQ_INJECT_FAILED 334 return -1; 335 cmpfail: 336 return 1; 337 #ifdef RSEQ_COMPARE_TWICE 338 error1: 339 rseq_bug("cpu_id comparison failed"); 340 error2: 341 rseq_bug("expected value comparison failed"); 342 #endif 343 } 344 345 static inline __always_inline 346 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 347 void *dst, void *src, size_t len, 348 intptr_t newv, int cpu) 349 { 350 RSEQ_INJECT_C(9) 351 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 353 #ifdef RSEQ_COMPARE_TWICE 354 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 355 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 356 #endif 357 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 358 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 359 RSEQ_INJECT_ASM(3) 360 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 361 RSEQ_INJECT_ASM(4) 362 #ifdef RSEQ_COMPARE_TWICE 363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 364 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 365 #endif 366 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 367 RSEQ_INJECT_ASM(5) 368 #ifdef RSEQ_TEMPLATE_MO_RELEASE 369 RSEQ_ASM_OP_FINAL_STORE_RELEASE(v, newv, 3) 370 #else 371 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3) 372 #endif 373 RSEQ_INJECT_ASM(6) 374 RSEQ_ASM_DEFINE_ABORT(4, abort) 375 : /* gcc asm goto does not allow outputs */ 376 : [cpu_id] "r" (cpu), 377 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 378 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 379 [expect] "r" (expect), 380 [v] "m" (*v), 381 [newv] "r" (newv), 382 [dst] "r" (dst), 383 [src] "r" (src), 384 [len] "r" (len) 385 RSEQ_INJECT_INPUT 386 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, 387 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 388 RSEQ_INJECT_CLOBBER 389 : abort, cmpfail 390 #ifdef RSEQ_COMPARE_TWICE 391 , error1, error2 392 #endif 393 ); 394 395 return 0; 396 abort: 397 RSEQ_INJECT_FAILED 398 return -1; 399 cmpfail: 400 return 1; 401 #ifdef RSEQ_COMPARE_TWICE 402 error1: 403 rseq_bug("cpu_id comparison failed"); 404 error2: 405 rseq_bug("expected value comparison failed"); 406 #endif 407 } 408 409 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 410 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 411 412 #include "rseq-bits-reset.h" 413