1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-ppc.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> 7 */ 8 9 /* 10 * RSEQ_SIG is used with the following trap instruction: 11 * 12 * powerpc-be: 0f e5 00 0b twui r5,11 13 * powerpc64-le: 0b 00 e5 0f twui r5,11 14 * powerpc64-be: 0f e5 00 0b twui r5,11 15 */ 16 17 #define RSEQ_SIG 0x0fe5000b 18 19 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc") 20 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc") 21 #define rseq_smp_rmb() rseq_smp_lwsync() 22 #define rseq_smp_wmb() rseq_smp_lwsync() 23 24 #define rseq_smp_load_acquire(p) \ 25 __extension__ ({ \ 26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 27 rseq_smp_lwsync(); \ 28 ____p1; \ 29 }) 30 31 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync() 32 33 #define rseq_smp_store_release(p, v) \ 34 do { \ 35 rseq_smp_lwsync(); \ 36 RSEQ_WRITE_ONCE(*p, v); \ 37 } while (0) 38 39 #ifdef RSEQ_SKIP_FASTPATH 40 #include "rseq-skip.h" 41 #else /* !RSEQ_SKIP_FASTPATH */ 42 43 /* 44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to 45 * better handle single-stepping through the restartable critical sections. 46 */ 47 48 #ifdef __PPC64__ 49 50 #define STORE_WORD "std " 51 #define LOAD_WORD "ld " 52 #define LOADX_WORD "ldx " 53 #define CMP_WORD "cmpd " 54 55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 56 start_ip, post_commit_offset, abort_ip) \ 57 ".pushsection __rseq_cs, \"aw\"\n\t" \ 58 ".balign 32\n\t" \ 59 __rseq_str(label) ":\n\t" \ 60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 62 ".popsection\n\t" \ 63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 64 ".quad " __rseq_str(label) "b\n\t" \ 65 ".popsection\n\t" 66 67 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 68 RSEQ_INJECT_ASM(1) \ 69 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \ 70 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \ 71 "rldicr %%r17, %%r17, 32, 31\n\t" \ 72 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \ 73 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ 74 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 75 __rseq_str(label) ":\n\t" 76 77 /* 78 * Exit points of a rseq critical section consist of all instructions outside 79 * of the critical section where a critical section can either branch to or 80 * reach through the normal course of its execution. The abort IP and the 81 * post-commit IP are already part of the __rseq_cs section and should not be 82 * explicitly defined as additional exit points. Knowing all exit points is 83 * useful to assist debuggers stepping over the critical section. 84 */ 85 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 86 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 87 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 88 ".popsection\n\t" 89 90 #else /* #ifdef __PPC64__ */ 91 92 #define STORE_WORD "stw " 93 #define LOAD_WORD "lwz " 94 #define LOADX_WORD "lwzx " 95 #define CMP_WORD "cmpw " 96 97 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 98 start_ip, post_commit_offset, abort_ip) \ 99 ".pushsection __rseq_cs, \"aw\"\n\t" \ 100 ".balign 32\n\t" \ 101 __rseq_str(label) ":\n\t" \ 102 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 103 /* 32-bit only supported on BE */ \ 104 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 105 ".popsection\n\t" \ 106 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 107 ".long 0x0, " __rseq_str(label) "b\n\t" \ 108 ".popsection\n\t" 109 110 /* 111 * Exit points of a rseq critical section consist of all instructions outside 112 * of the critical section where a critical section can either branch to or 113 * reach through the normal course of its execution. The abort IP and the 114 * post-commit IP are already part of the __rseq_cs section and should not be 115 * explicitly defined as additional exit points. Knowing all exit points is 116 * useful to assist debuggers stepping over the critical section. 117 */ 118 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 119 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 120 /* 32-bit only supported on BE */ \ 121 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ 122 ".popsection\n\t" 123 124 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 125 RSEQ_INJECT_ASM(1) \ 126 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ 127 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ 128 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 129 __rseq_str(label) ":\n\t" 130 131 #endif /* #ifdef __PPC64__ */ 132 133 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 134 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 135 (post_commit_ip - start_ip), abort_ip) 136 137 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 138 RSEQ_INJECT_ASM(2) \ 139 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ 140 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ 141 "bne- cr7, " __rseq_str(label) "\n\t" 142 143 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 144 ".pushsection __rseq_failure, \"ax\"\n\t" \ 145 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 146 __rseq_str(label) ":\n\t" \ 147 "b %l[" __rseq_str(abort_label) "]\n\t" \ 148 ".popsection\n\t" 149 150 /* 151 * RSEQ_ASM_OPs: asm operations for rseq 152 * RSEQ_ASM_OP_R_*: has hard-code registers in it 153 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) 154 */ 155 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 156 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ 157 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ 158 "bne- cr7, " __rseq_str(label) "\n\t" 159 160 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \ 161 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ 162 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ 163 "beq- cr7, " __rseq_str(label) "\n\t" 164 165 #define RSEQ_ASM_OP_STORE(value, var) \ 166 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" 167 168 /* Load @var to r17 */ 169 #define RSEQ_ASM_OP_R_LOAD(var) \ 170 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" 171 172 /* Store r17 to @var */ 173 #define RSEQ_ASM_OP_R_STORE(var) \ 174 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" 175 176 /* Add @count to r17 */ 177 #define RSEQ_ASM_OP_R_ADD(count) \ 178 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t" 179 180 /* Load (r17 + voffp) to r17 */ 181 #define RSEQ_ASM_OP_R_LOADX(voffp) \ 182 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" 183 184 /* TODO: implement a faster memcpy. */ 185 #define RSEQ_ASM_OP_R_MEMCPY() \ 186 "cmpdi %%r19, 0\n\t" \ 187 "beq 333f\n\t" \ 188 "addi %%r20, %%r20, -1\n\t" \ 189 "addi %%r21, %%r21, -1\n\t" \ 190 "222:\n\t" \ 191 "lbzu %%r18, 1(%%r20)\n\t" \ 192 "stbu %%r18, 1(%%r21)\n\t" \ 193 "addi %%r19, %%r19, -1\n\t" \ 194 "cmpdi %%r19, 0\n\t" \ 195 "bne 222b\n\t" \ 196 "333:\n\t" \ 197 198 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 199 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ 200 __rseq_str(post_commit_label) ":\n\t" 201 202 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 203 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ 204 __rseq_str(post_commit_label) ":\n\t" 205 206 static inline __attribute__((always_inline)) 207 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 208 { 209 RSEQ_INJECT_C(9) 210 211 __asm__ __volatile__ goto ( 212 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 213 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 214 #ifdef RSEQ_COMPARE_TWICE 215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 216 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 217 #endif 218 /* Start rseq by storing table entry pointer into rseq_cs. */ 219 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 220 /* cmp cpuid */ 221 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 222 RSEQ_INJECT_ASM(3) 223 /* cmp @v equal to @expect */ 224 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 225 RSEQ_INJECT_ASM(4) 226 #ifdef RSEQ_COMPARE_TWICE 227 /* cmp cpuid */ 228 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 229 /* cmp @v equal to @expect */ 230 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 231 #endif 232 /* final store */ 233 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 234 RSEQ_INJECT_ASM(5) 235 RSEQ_ASM_DEFINE_ABORT(4, abort) 236 : /* gcc asm goto does not allow outputs */ 237 : [cpu_id] "r" (cpu), 238 [current_cpu_id] "m" (__rseq_abi.cpu_id), 239 [rseq_cs] "m" (__rseq_abi.rseq_cs), 240 [v] "m" (*v), 241 [expect] "r" (expect), 242 [newv] "r" (newv) 243 RSEQ_INJECT_INPUT 244 : "memory", "cc", "r17" 245 RSEQ_INJECT_CLOBBER 246 : abort, cmpfail 247 #ifdef RSEQ_COMPARE_TWICE 248 , error1, error2 249 #endif 250 ); 251 return 0; 252 abort: 253 RSEQ_INJECT_FAILED 254 return -1; 255 cmpfail: 256 return 1; 257 #ifdef RSEQ_COMPARE_TWICE 258 error1: 259 rseq_bug("cpu_id comparison failed"); 260 error2: 261 rseq_bug("expected value comparison failed"); 262 #endif 263 } 264 265 static inline __attribute__((always_inline)) 266 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 267 off_t voffp, intptr_t *load, int cpu) 268 { 269 RSEQ_INJECT_C(9) 270 271 __asm__ __volatile__ goto ( 272 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 274 #ifdef RSEQ_COMPARE_TWICE 275 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 276 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 277 #endif 278 /* Start rseq by storing table entry pointer into rseq_cs. */ 279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 280 /* cmp cpuid */ 281 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 282 RSEQ_INJECT_ASM(3) 283 /* cmp @v not equal to @expectnot */ 284 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 285 RSEQ_INJECT_ASM(4) 286 #ifdef RSEQ_COMPARE_TWICE 287 /* cmp cpuid */ 288 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 289 /* cmp @v not equal to @expectnot */ 290 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 291 #endif 292 /* load the value of @v */ 293 RSEQ_ASM_OP_R_LOAD(v) 294 /* store it in @load */ 295 RSEQ_ASM_OP_R_STORE(load) 296 /* dereference voffp(v) */ 297 RSEQ_ASM_OP_R_LOADX(voffp) 298 /* final store the value at voffp(v) */ 299 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 300 RSEQ_INJECT_ASM(5) 301 RSEQ_ASM_DEFINE_ABORT(4, abort) 302 : /* gcc asm goto does not allow outputs */ 303 : [cpu_id] "r" (cpu), 304 [current_cpu_id] "m" (__rseq_abi.cpu_id), 305 [rseq_cs] "m" (__rseq_abi.rseq_cs), 306 /* final store input */ 307 [v] "m" (*v), 308 [expectnot] "r" (expectnot), 309 [voffp] "b" (voffp), 310 [load] "m" (*load) 311 RSEQ_INJECT_INPUT 312 : "memory", "cc", "r17" 313 RSEQ_INJECT_CLOBBER 314 : abort, cmpfail 315 #ifdef RSEQ_COMPARE_TWICE 316 , error1, error2 317 #endif 318 ); 319 return 0; 320 abort: 321 RSEQ_INJECT_FAILED 322 return -1; 323 cmpfail: 324 return 1; 325 #ifdef RSEQ_COMPARE_TWICE 326 error1: 327 rseq_bug("cpu_id comparison failed"); 328 error2: 329 rseq_bug("expected value comparison failed"); 330 #endif 331 } 332 333 static inline __attribute__((always_inline)) 334 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 335 { 336 RSEQ_INJECT_C(9) 337 338 __asm__ __volatile__ goto ( 339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 340 #ifdef RSEQ_COMPARE_TWICE 341 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 342 #endif 343 /* Start rseq by storing table entry pointer into rseq_cs. */ 344 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 345 /* cmp cpuid */ 346 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 347 RSEQ_INJECT_ASM(3) 348 #ifdef RSEQ_COMPARE_TWICE 349 /* cmp cpuid */ 350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 351 #endif 352 /* load the value of @v */ 353 RSEQ_ASM_OP_R_LOAD(v) 354 /* add @count to it */ 355 RSEQ_ASM_OP_R_ADD(count) 356 /* final store */ 357 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 358 RSEQ_INJECT_ASM(4) 359 RSEQ_ASM_DEFINE_ABORT(4, abort) 360 : /* gcc asm goto does not allow outputs */ 361 : [cpu_id] "r" (cpu), 362 [current_cpu_id] "m" (__rseq_abi.cpu_id), 363 [rseq_cs] "m" (__rseq_abi.rseq_cs), 364 /* final store input */ 365 [v] "m" (*v), 366 [count] "r" (count) 367 RSEQ_INJECT_INPUT 368 : "memory", "cc", "r17" 369 RSEQ_INJECT_CLOBBER 370 : abort 371 #ifdef RSEQ_COMPARE_TWICE 372 , error1 373 #endif 374 ); 375 return 0; 376 abort: 377 RSEQ_INJECT_FAILED 378 return -1; 379 #ifdef RSEQ_COMPARE_TWICE 380 error1: 381 rseq_bug("cpu_id comparison failed"); 382 #endif 383 } 384 385 static inline __attribute__((always_inline)) 386 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 387 intptr_t *v2, intptr_t newv2, 388 intptr_t newv, int cpu) 389 { 390 RSEQ_INJECT_C(9) 391 392 __asm__ __volatile__ goto ( 393 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 394 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 395 #ifdef RSEQ_COMPARE_TWICE 396 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 397 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 398 #endif 399 /* Start rseq by storing table entry pointer into rseq_cs. */ 400 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 401 /* cmp cpuid */ 402 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 403 RSEQ_INJECT_ASM(3) 404 /* cmp @v equal to @expect */ 405 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 406 RSEQ_INJECT_ASM(4) 407 #ifdef RSEQ_COMPARE_TWICE 408 /* cmp cpuid */ 409 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 410 /* cmp @v equal to @expect */ 411 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 412 #endif 413 /* try store */ 414 RSEQ_ASM_OP_STORE(newv2, v2) 415 RSEQ_INJECT_ASM(5) 416 /* final store */ 417 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 418 RSEQ_INJECT_ASM(6) 419 RSEQ_ASM_DEFINE_ABORT(4, abort) 420 : /* gcc asm goto does not allow outputs */ 421 : [cpu_id] "r" (cpu), 422 [current_cpu_id] "m" (__rseq_abi.cpu_id), 423 [rseq_cs] "m" (__rseq_abi.rseq_cs), 424 /* try store input */ 425 [v2] "m" (*v2), 426 [newv2] "r" (newv2), 427 /* final store input */ 428 [v] "m" (*v), 429 [expect] "r" (expect), 430 [newv] "r" (newv) 431 RSEQ_INJECT_INPUT 432 : "memory", "cc", "r17" 433 RSEQ_INJECT_CLOBBER 434 : abort, cmpfail 435 #ifdef RSEQ_COMPARE_TWICE 436 , error1, error2 437 #endif 438 ); 439 return 0; 440 abort: 441 RSEQ_INJECT_FAILED 442 return -1; 443 cmpfail: 444 return 1; 445 #ifdef RSEQ_COMPARE_TWICE 446 error1: 447 rseq_bug("cpu_id comparison failed"); 448 error2: 449 rseq_bug("expected value comparison failed"); 450 #endif 451 } 452 453 static inline __attribute__((always_inline)) 454 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 455 intptr_t *v2, intptr_t newv2, 456 intptr_t newv, int cpu) 457 { 458 RSEQ_INJECT_C(9) 459 460 __asm__ __volatile__ goto ( 461 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 462 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 463 #ifdef RSEQ_COMPARE_TWICE 464 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 465 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 466 #endif 467 /* Start rseq by storing table entry pointer into rseq_cs. */ 468 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 469 /* cmp cpuid */ 470 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 471 RSEQ_INJECT_ASM(3) 472 /* cmp @v equal to @expect */ 473 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 474 RSEQ_INJECT_ASM(4) 475 #ifdef RSEQ_COMPARE_TWICE 476 /* cmp cpuid */ 477 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 478 /* cmp @v equal to @expect */ 479 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 480 #endif 481 /* try store */ 482 RSEQ_ASM_OP_STORE(newv2, v2) 483 RSEQ_INJECT_ASM(5) 484 /* for 'release' */ 485 "lwsync\n\t" 486 /* final store */ 487 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 488 RSEQ_INJECT_ASM(6) 489 RSEQ_ASM_DEFINE_ABORT(4, abort) 490 : /* gcc asm goto does not allow outputs */ 491 : [cpu_id] "r" (cpu), 492 [current_cpu_id] "m" (__rseq_abi.cpu_id), 493 [rseq_cs] "m" (__rseq_abi.rseq_cs), 494 /* try store input */ 495 [v2] "m" (*v2), 496 [newv2] "r" (newv2), 497 /* final store input */ 498 [v] "m" (*v), 499 [expect] "r" (expect), 500 [newv] "r" (newv) 501 RSEQ_INJECT_INPUT 502 : "memory", "cc", "r17" 503 RSEQ_INJECT_CLOBBER 504 : abort, cmpfail 505 #ifdef RSEQ_COMPARE_TWICE 506 , error1, error2 507 #endif 508 ); 509 return 0; 510 abort: 511 RSEQ_INJECT_FAILED 512 return -1; 513 cmpfail: 514 return 1; 515 #ifdef RSEQ_COMPARE_TWICE 516 error1: 517 rseq_bug("cpu_id comparison failed"); 518 error2: 519 rseq_bug("expected value comparison failed"); 520 #endif 521 } 522 523 static inline __attribute__((always_inline)) 524 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 525 intptr_t *v2, intptr_t expect2, 526 intptr_t newv, int cpu) 527 { 528 RSEQ_INJECT_C(9) 529 530 __asm__ __volatile__ goto ( 531 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 532 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 533 #ifdef RSEQ_COMPARE_TWICE 534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 535 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 537 #endif 538 /* Start rseq by storing table entry pointer into rseq_cs. */ 539 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 540 /* cmp cpuid */ 541 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 542 RSEQ_INJECT_ASM(3) 543 /* cmp @v equal to @expect */ 544 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 545 RSEQ_INJECT_ASM(4) 546 /* cmp @v2 equal to @expct2 */ 547 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 548 RSEQ_INJECT_ASM(5) 549 #ifdef RSEQ_COMPARE_TWICE 550 /* cmp cpuid */ 551 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 552 /* cmp @v equal to @expect */ 553 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 554 /* cmp @v2 equal to @expct2 */ 555 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 556 #endif 557 /* final store */ 558 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 559 RSEQ_INJECT_ASM(6) 560 RSEQ_ASM_DEFINE_ABORT(4, abort) 561 : /* gcc asm goto does not allow outputs */ 562 : [cpu_id] "r" (cpu), 563 [current_cpu_id] "m" (__rseq_abi.cpu_id), 564 [rseq_cs] "m" (__rseq_abi.rseq_cs), 565 /* cmp2 input */ 566 [v2] "m" (*v2), 567 [expect2] "r" (expect2), 568 /* final store input */ 569 [v] "m" (*v), 570 [expect] "r" (expect), 571 [newv] "r" (newv) 572 RSEQ_INJECT_INPUT 573 : "memory", "cc", "r17" 574 RSEQ_INJECT_CLOBBER 575 : abort, cmpfail 576 #ifdef RSEQ_COMPARE_TWICE 577 , error1, error2, error3 578 #endif 579 ); 580 return 0; 581 abort: 582 RSEQ_INJECT_FAILED 583 return -1; 584 cmpfail: 585 return 1; 586 #ifdef RSEQ_COMPARE_TWICE 587 error1: 588 rseq_bug("cpu_id comparison failed"); 589 error2: 590 rseq_bug("1st expected value comparison failed"); 591 error3: 592 rseq_bug("2nd expected value comparison failed"); 593 #endif 594 } 595 596 static inline __attribute__((always_inline)) 597 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 598 void *dst, void *src, size_t len, 599 intptr_t newv, int cpu) 600 { 601 RSEQ_INJECT_C(9) 602 603 __asm__ __volatile__ goto ( 604 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 605 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 606 #ifdef RSEQ_COMPARE_TWICE 607 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 608 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 609 #endif 610 /* setup for mempcy */ 611 "mr %%r19, %[len]\n\t" 612 "mr %%r20, %[src]\n\t" 613 "mr %%r21, %[dst]\n\t" 614 /* Start rseq by storing table entry pointer into rseq_cs. */ 615 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 616 /* cmp cpuid */ 617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 618 RSEQ_INJECT_ASM(3) 619 /* cmp @v equal to @expect */ 620 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 621 RSEQ_INJECT_ASM(4) 622 #ifdef RSEQ_COMPARE_TWICE 623 /* cmp cpuid */ 624 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 625 /* cmp @v equal to @expect */ 626 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 627 #endif 628 /* try memcpy */ 629 RSEQ_ASM_OP_R_MEMCPY() 630 RSEQ_INJECT_ASM(5) 631 /* final store */ 632 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 633 RSEQ_INJECT_ASM(6) 634 /* teardown */ 635 RSEQ_ASM_DEFINE_ABORT(4, abort) 636 : /* gcc asm goto does not allow outputs */ 637 : [cpu_id] "r" (cpu), 638 [current_cpu_id] "m" (__rseq_abi.cpu_id), 639 [rseq_cs] "m" (__rseq_abi.rseq_cs), 640 /* final store input */ 641 [v] "m" (*v), 642 [expect] "r" (expect), 643 [newv] "r" (newv), 644 /* try memcpy input */ 645 [dst] "r" (dst), 646 [src] "r" (src), 647 [len] "r" (len) 648 RSEQ_INJECT_INPUT 649 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 650 RSEQ_INJECT_CLOBBER 651 : abort, cmpfail 652 #ifdef RSEQ_COMPARE_TWICE 653 , error1, error2 654 #endif 655 ); 656 return 0; 657 abort: 658 RSEQ_INJECT_FAILED 659 return -1; 660 cmpfail: 661 return 1; 662 #ifdef RSEQ_COMPARE_TWICE 663 error1: 664 rseq_bug("cpu_id comparison failed"); 665 error2: 666 rseq_bug("expected value comparison failed"); 667 #endif 668 } 669 670 static inline __attribute__((always_inline)) 671 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 672 void *dst, void *src, size_t len, 673 intptr_t newv, int cpu) 674 { 675 RSEQ_INJECT_C(9) 676 677 __asm__ __volatile__ goto ( 678 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 679 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 680 #ifdef RSEQ_COMPARE_TWICE 681 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 682 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 683 #endif 684 /* setup for mempcy */ 685 "mr %%r19, %[len]\n\t" 686 "mr %%r20, %[src]\n\t" 687 "mr %%r21, %[dst]\n\t" 688 /* Start rseq by storing table entry pointer into rseq_cs. */ 689 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 690 /* cmp cpuid */ 691 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 692 RSEQ_INJECT_ASM(3) 693 /* cmp @v equal to @expect */ 694 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 695 RSEQ_INJECT_ASM(4) 696 #ifdef RSEQ_COMPARE_TWICE 697 /* cmp cpuid */ 698 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 699 /* cmp @v equal to @expect */ 700 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 701 #endif 702 /* try memcpy */ 703 RSEQ_ASM_OP_R_MEMCPY() 704 RSEQ_INJECT_ASM(5) 705 /* for 'release' */ 706 "lwsync\n\t" 707 /* final store */ 708 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 709 RSEQ_INJECT_ASM(6) 710 /* teardown */ 711 RSEQ_ASM_DEFINE_ABORT(4, abort) 712 : /* gcc asm goto does not allow outputs */ 713 : [cpu_id] "r" (cpu), 714 [current_cpu_id] "m" (__rseq_abi.cpu_id), 715 [rseq_cs] "m" (__rseq_abi.rseq_cs), 716 /* final store input */ 717 [v] "m" (*v), 718 [expect] "r" (expect), 719 [newv] "r" (newv), 720 /* try memcpy input */ 721 [dst] "r" (dst), 722 [src] "r" (src), 723 [len] "r" (len) 724 RSEQ_INJECT_INPUT 725 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 726 RSEQ_INJECT_CLOBBER 727 : abort, cmpfail 728 #ifdef RSEQ_COMPARE_TWICE 729 , error1, error2 730 #endif 731 ); 732 return 0; 733 abort: 734 RSEQ_INJECT_FAILED 735 return -1; 736 cmpfail: 737 return 1; 738 #ifdef RSEQ_COMPARE_TWICE 739 error1: 740 rseq_bug("cpu_id comparison failed"); 741 error2: 742 rseq_bug("expected value comparison failed"); 743 #endif 744 } 745 746 #undef STORE_WORD 747 #undef LOAD_WORD 748 #undef LOADX_WORD 749 #undef CMP_WORD 750 751 #endif /* !RSEQ_SKIP_FASTPATH */ 752