1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-x86.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #include <stdint.h> 9 10 /* 11 * RSEQ_SIG is used with the following reserved undefined instructions, which 12 * trap in user-space: 13 * 14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi 15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi 16 */ 17 #define RSEQ_SIG 0x53053053 18 19 /* 20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input 21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi 22 * address through a "r" input operand. 23 */ 24 25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */ 26 #define RSEQ_CPU_ID_OFFSET 4 27 #define RSEQ_CS_OFFSET 8 28 29 #ifdef __x86_64__ 30 31 #define rseq_smp_mb() \ 32 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") 33 #define rseq_smp_rmb() rseq_barrier() 34 #define rseq_smp_wmb() rseq_barrier() 35 36 #define rseq_smp_load_acquire(p) \ 37 __extension__ ({ \ 38 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 39 rseq_barrier(); \ 40 ____p1; \ 41 }) 42 43 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 44 45 #define rseq_smp_store_release(p, v) \ 46 do { \ 47 rseq_barrier(); \ 48 RSEQ_WRITE_ONCE(*p, v); \ 49 } while (0) 50 51 #ifdef RSEQ_SKIP_FASTPATH 52 #include "rseq-skip.h" 53 #else /* !RSEQ_SKIP_FASTPATH */ 54 55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 56 start_ip, post_commit_offset, abort_ip) \ 57 ".pushsection __rseq_cs, \"aw\"\n\t" \ 58 ".balign 32\n\t" \ 59 __rseq_str(label) ":\n\t" \ 60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 62 ".popsection\n\t" \ 63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 64 ".quad " __rseq_str(label) "b\n\t" \ 65 ".popsection\n\t" 66 67 68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 69 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 70 (post_commit_ip - start_ip), abort_ip) 71 72 /* 73 * Exit points of a rseq critical section consist of all instructions outside 74 * of the critical section where a critical section can either branch to or 75 * reach through the normal course of its execution. The abort IP and the 76 * post-commit IP are already part of the __rseq_cs section and should not be 77 * explicitly defined as additional exit points. Knowing all exit points is 78 * useful to assist debuggers stepping over the critical section. 79 */ 80 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 81 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 82 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 83 ".popsection\n\t" 84 85 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 86 RSEQ_INJECT_ASM(1) \ 87 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ 88 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ 89 __rseq_str(label) ":\n\t" 90 91 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 92 RSEQ_INJECT_ASM(2) \ 93 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 94 "jnz " __rseq_str(label) "\n\t" 95 96 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 97 ".pushsection __rseq_failure, \"ax\"\n\t" \ 98 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ 99 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 100 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 101 __rseq_str(label) ":\n\t" \ 102 teardown \ 103 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 104 ".popsection\n\t" 105 106 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 107 ".pushsection __rseq_failure, \"ax\"\n\t" \ 108 __rseq_str(label) ":\n\t" \ 109 teardown \ 110 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 111 ".popsection\n\t" 112 113 static inline __attribute__((always_inline)) 114 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 115 { 116 RSEQ_INJECT_C(9) 117 118 __asm__ __volatile__ goto ( 119 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 120 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 121 #ifdef RSEQ_COMPARE_TWICE 122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 124 #endif 125 /* Start rseq by storing table entry pointer into rseq_cs. */ 126 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 127 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 128 RSEQ_INJECT_ASM(3) 129 "cmpq %[v], %[expect]\n\t" 130 "jnz %l[cmpfail]\n\t" 131 RSEQ_INJECT_ASM(4) 132 #ifdef RSEQ_COMPARE_TWICE 133 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 134 "cmpq %[v], %[expect]\n\t" 135 "jnz %l[error2]\n\t" 136 #endif 137 /* final store */ 138 "movq %[newv], %[v]\n\t" 139 "2:\n\t" 140 RSEQ_INJECT_ASM(5) 141 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 142 : /* gcc asm goto does not allow outputs */ 143 : [cpu_id] "r" (cpu), 144 [rseq_abi] "r" (&__rseq_abi), 145 [v] "m" (*v), 146 [expect] "r" (expect), 147 [newv] "r" (newv) 148 : "memory", "cc", "rax" 149 RSEQ_INJECT_CLOBBER 150 : abort, cmpfail 151 #ifdef RSEQ_COMPARE_TWICE 152 , error1, error2 153 #endif 154 ); 155 return 0; 156 abort: 157 RSEQ_INJECT_FAILED 158 return -1; 159 cmpfail: 160 return 1; 161 #ifdef RSEQ_COMPARE_TWICE 162 error1: 163 rseq_bug("cpu_id comparison failed"); 164 error2: 165 rseq_bug("expected value comparison failed"); 166 #endif 167 } 168 169 /* 170 * Compare @v against @expectnot. When it does _not_ match, load @v 171 * into @load, and store the content of *@v + voffp into @v. 172 */ 173 static inline __attribute__((always_inline)) 174 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 175 off_t voffp, intptr_t *load, int cpu) 176 { 177 RSEQ_INJECT_C(9) 178 179 __asm__ __volatile__ goto ( 180 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 181 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 182 #ifdef RSEQ_COMPARE_TWICE 183 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 185 #endif 186 /* Start rseq by storing table entry pointer into rseq_cs. */ 187 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 188 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 189 RSEQ_INJECT_ASM(3) 190 "movq %[v], %%rbx\n\t" 191 "cmpq %%rbx, %[expectnot]\n\t" 192 "je %l[cmpfail]\n\t" 193 RSEQ_INJECT_ASM(4) 194 #ifdef RSEQ_COMPARE_TWICE 195 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 196 "movq %[v], %%rbx\n\t" 197 "cmpq %%rbx, %[expectnot]\n\t" 198 "je %l[error2]\n\t" 199 #endif 200 "movq %%rbx, %[load]\n\t" 201 "addq %[voffp], %%rbx\n\t" 202 "movq (%%rbx), %%rbx\n\t" 203 /* final store */ 204 "movq %%rbx, %[v]\n\t" 205 "2:\n\t" 206 RSEQ_INJECT_ASM(5) 207 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 208 : /* gcc asm goto does not allow outputs */ 209 : [cpu_id] "r" (cpu), 210 [rseq_abi] "r" (&__rseq_abi), 211 /* final store input */ 212 [v] "m" (*v), 213 [expectnot] "r" (expectnot), 214 [voffp] "er" (voffp), 215 [load] "m" (*load) 216 : "memory", "cc", "rax", "rbx" 217 RSEQ_INJECT_CLOBBER 218 : abort, cmpfail 219 #ifdef RSEQ_COMPARE_TWICE 220 , error1, error2 221 #endif 222 ); 223 return 0; 224 abort: 225 RSEQ_INJECT_FAILED 226 return -1; 227 cmpfail: 228 return 1; 229 #ifdef RSEQ_COMPARE_TWICE 230 error1: 231 rseq_bug("cpu_id comparison failed"); 232 error2: 233 rseq_bug("expected value comparison failed"); 234 #endif 235 } 236 237 static inline __attribute__((always_inline)) 238 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 239 { 240 RSEQ_INJECT_C(9) 241 242 __asm__ __volatile__ goto ( 243 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 244 #ifdef RSEQ_COMPARE_TWICE 245 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 246 #endif 247 /* Start rseq by storing table entry pointer into rseq_cs. */ 248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 249 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 250 RSEQ_INJECT_ASM(3) 251 #ifdef RSEQ_COMPARE_TWICE 252 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 253 #endif 254 /* final store */ 255 "addq %[count], %[v]\n\t" 256 "2:\n\t" 257 RSEQ_INJECT_ASM(4) 258 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 259 : /* gcc asm goto does not allow outputs */ 260 : [cpu_id] "r" (cpu), 261 [rseq_abi] "r" (&__rseq_abi), 262 /* final store input */ 263 [v] "m" (*v), 264 [count] "er" (count) 265 : "memory", "cc", "rax" 266 RSEQ_INJECT_CLOBBER 267 : abort 268 #ifdef RSEQ_COMPARE_TWICE 269 , error1 270 #endif 271 ); 272 return 0; 273 abort: 274 RSEQ_INJECT_FAILED 275 return -1; 276 #ifdef RSEQ_COMPARE_TWICE 277 error1: 278 rseq_bug("cpu_id comparison failed"); 279 #endif 280 } 281 282 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 283 284 /* 285 * pval = *(ptr+off) 286 * *pval += inc; 287 */ 288 static inline __attribute__((always_inline)) 289 int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu) 290 { 291 RSEQ_INJECT_C(9) 292 293 __asm__ __volatile__ goto ( 294 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 295 #ifdef RSEQ_COMPARE_TWICE 296 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 297 #endif 298 /* Start rseq by storing table entry pointer into rseq_cs. */ 299 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 300 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 301 RSEQ_INJECT_ASM(3) 302 #ifdef RSEQ_COMPARE_TWICE 303 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 304 #endif 305 /* get p+v */ 306 "movq %[ptr], %%rbx\n\t" 307 "addq %[off], %%rbx\n\t" 308 /* get pv */ 309 "movq (%%rbx), %%rcx\n\t" 310 /* *pv += inc */ 311 "addq %[inc], (%%rcx)\n\t" 312 "2:\n\t" 313 RSEQ_INJECT_ASM(4) 314 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 315 : /* gcc asm goto does not allow outputs */ 316 : [cpu_id] "r" (cpu), 317 [rseq_abi] "r" (&__rseq_abi), 318 /* final store input */ 319 [ptr] "m" (*ptr), 320 [off] "er" (off), 321 [inc] "er" (inc) 322 : "memory", "cc", "rax", "rbx", "rcx" 323 RSEQ_INJECT_CLOBBER 324 : abort 325 #ifdef RSEQ_COMPARE_TWICE 326 , error1 327 #endif 328 ); 329 return 0; 330 abort: 331 RSEQ_INJECT_FAILED 332 return -1; 333 #ifdef RSEQ_COMPARE_TWICE 334 error1: 335 rseq_bug("cpu_id comparison failed"); 336 #endif 337 } 338 339 static inline __attribute__((always_inline)) 340 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 341 intptr_t *v2, intptr_t newv2, 342 intptr_t newv, int cpu) 343 { 344 RSEQ_INJECT_C(9) 345 346 __asm__ __volatile__ goto ( 347 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 349 #ifdef RSEQ_COMPARE_TWICE 350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 351 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 352 #endif 353 /* Start rseq by storing table entry pointer into rseq_cs. */ 354 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 355 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 356 RSEQ_INJECT_ASM(3) 357 "cmpq %[v], %[expect]\n\t" 358 "jnz %l[cmpfail]\n\t" 359 RSEQ_INJECT_ASM(4) 360 #ifdef RSEQ_COMPARE_TWICE 361 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 362 "cmpq %[v], %[expect]\n\t" 363 "jnz %l[error2]\n\t" 364 #endif 365 /* try store */ 366 "movq %[newv2], %[v2]\n\t" 367 RSEQ_INJECT_ASM(5) 368 /* final store */ 369 "movq %[newv], %[v]\n\t" 370 "2:\n\t" 371 RSEQ_INJECT_ASM(6) 372 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 373 : /* gcc asm goto does not allow outputs */ 374 : [cpu_id] "r" (cpu), 375 [rseq_abi] "r" (&__rseq_abi), 376 /* try store input */ 377 [v2] "m" (*v2), 378 [newv2] "r" (newv2), 379 /* final store input */ 380 [v] "m" (*v), 381 [expect] "r" (expect), 382 [newv] "r" (newv) 383 : "memory", "cc", "rax" 384 RSEQ_INJECT_CLOBBER 385 : abort, cmpfail 386 #ifdef RSEQ_COMPARE_TWICE 387 , error1, error2 388 #endif 389 ); 390 return 0; 391 abort: 392 RSEQ_INJECT_FAILED 393 return -1; 394 cmpfail: 395 return 1; 396 #ifdef RSEQ_COMPARE_TWICE 397 error1: 398 rseq_bug("cpu_id comparison failed"); 399 error2: 400 rseq_bug("expected value comparison failed"); 401 #endif 402 } 403 404 /* x86-64 is TSO. */ 405 static inline __attribute__((always_inline)) 406 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 407 intptr_t *v2, intptr_t newv2, 408 intptr_t newv, int cpu) 409 { 410 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 411 } 412 413 static inline __attribute__((always_inline)) 414 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 415 intptr_t *v2, intptr_t expect2, 416 intptr_t newv, int cpu) 417 { 418 RSEQ_INJECT_C(9) 419 420 __asm__ __volatile__ goto ( 421 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 422 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 423 #ifdef RSEQ_COMPARE_TWICE 424 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 425 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 426 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 427 #endif 428 /* Start rseq by storing table entry pointer into rseq_cs. */ 429 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 430 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 431 RSEQ_INJECT_ASM(3) 432 "cmpq %[v], %[expect]\n\t" 433 "jnz %l[cmpfail]\n\t" 434 RSEQ_INJECT_ASM(4) 435 "cmpq %[v2], %[expect2]\n\t" 436 "jnz %l[cmpfail]\n\t" 437 RSEQ_INJECT_ASM(5) 438 #ifdef RSEQ_COMPARE_TWICE 439 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 440 "cmpq %[v], %[expect]\n\t" 441 "jnz %l[error2]\n\t" 442 "cmpq %[v2], %[expect2]\n\t" 443 "jnz %l[error3]\n\t" 444 #endif 445 /* final store */ 446 "movq %[newv], %[v]\n\t" 447 "2:\n\t" 448 RSEQ_INJECT_ASM(6) 449 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 450 : /* gcc asm goto does not allow outputs */ 451 : [cpu_id] "r" (cpu), 452 [rseq_abi] "r" (&__rseq_abi), 453 /* cmp2 input */ 454 [v2] "m" (*v2), 455 [expect2] "r" (expect2), 456 /* final store input */ 457 [v] "m" (*v), 458 [expect] "r" (expect), 459 [newv] "r" (newv) 460 : "memory", "cc", "rax" 461 RSEQ_INJECT_CLOBBER 462 : abort, cmpfail 463 #ifdef RSEQ_COMPARE_TWICE 464 , error1, error2, error3 465 #endif 466 ); 467 return 0; 468 abort: 469 RSEQ_INJECT_FAILED 470 return -1; 471 cmpfail: 472 return 1; 473 #ifdef RSEQ_COMPARE_TWICE 474 error1: 475 rseq_bug("cpu_id comparison failed"); 476 error2: 477 rseq_bug("1st expected value comparison failed"); 478 error3: 479 rseq_bug("2nd expected value comparison failed"); 480 #endif 481 } 482 483 static inline __attribute__((always_inline)) 484 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 485 void *dst, void *src, size_t len, 486 intptr_t newv, int cpu) 487 { 488 uint64_t rseq_scratch[3]; 489 490 RSEQ_INJECT_C(9) 491 492 __asm__ __volatile__ goto ( 493 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 494 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 495 #ifdef RSEQ_COMPARE_TWICE 496 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 497 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 498 #endif 499 "movq %[src], %[rseq_scratch0]\n\t" 500 "movq %[dst], %[rseq_scratch1]\n\t" 501 "movq %[len], %[rseq_scratch2]\n\t" 502 /* Start rseq by storing table entry pointer into rseq_cs. */ 503 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 504 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 505 RSEQ_INJECT_ASM(3) 506 "cmpq %[v], %[expect]\n\t" 507 "jnz 5f\n\t" 508 RSEQ_INJECT_ASM(4) 509 #ifdef RSEQ_COMPARE_TWICE 510 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 511 "cmpq %[v], %[expect]\n\t" 512 "jnz 7f\n\t" 513 #endif 514 /* try memcpy */ 515 "test %[len], %[len]\n\t" \ 516 "jz 333f\n\t" \ 517 "222:\n\t" \ 518 "movb (%[src]), %%al\n\t" \ 519 "movb %%al, (%[dst])\n\t" \ 520 "inc %[src]\n\t" \ 521 "inc %[dst]\n\t" \ 522 "dec %[len]\n\t" \ 523 "jnz 222b\n\t" \ 524 "333:\n\t" \ 525 RSEQ_INJECT_ASM(5) 526 /* final store */ 527 "movq %[newv], %[v]\n\t" 528 "2:\n\t" 529 RSEQ_INJECT_ASM(6) 530 /* teardown */ 531 "movq %[rseq_scratch2], %[len]\n\t" 532 "movq %[rseq_scratch1], %[dst]\n\t" 533 "movq %[rseq_scratch0], %[src]\n\t" 534 RSEQ_ASM_DEFINE_ABORT(4, 535 "movq %[rseq_scratch2], %[len]\n\t" 536 "movq %[rseq_scratch1], %[dst]\n\t" 537 "movq %[rseq_scratch0], %[src]\n\t", 538 abort) 539 RSEQ_ASM_DEFINE_CMPFAIL(5, 540 "movq %[rseq_scratch2], %[len]\n\t" 541 "movq %[rseq_scratch1], %[dst]\n\t" 542 "movq %[rseq_scratch0], %[src]\n\t", 543 cmpfail) 544 #ifdef RSEQ_COMPARE_TWICE 545 RSEQ_ASM_DEFINE_CMPFAIL(6, 546 "movq %[rseq_scratch2], %[len]\n\t" 547 "movq %[rseq_scratch1], %[dst]\n\t" 548 "movq %[rseq_scratch0], %[src]\n\t", 549 error1) 550 RSEQ_ASM_DEFINE_CMPFAIL(7, 551 "movq %[rseq_scratch2], %[len]\n\t" 552 "movq %[rseq_scratch1], %[dst]\n\t" 553 "movq %[rseq_scratch0], %[src]\n\t", 554 error2) 555 #endif 556 : /* gcc asm goto does not allow outputs */ 557 : [cpu_id] "r" (cpu), 558 [rseq_abi] "r" (&__rseq_abi), 559 /* final store input */ 560 [v] "m" (*v), 561 [expect] "r" (expect), 562 [newv] "r" (newv), 563 /* try memcpy input */ 564 [dst] "r" (dst), 565 [src] "r" (src), 566 [len] "r" (len), 567 [rseq_scratch0] "m" (rseq_scratch[0]), 568 [rseq_scratch1] "m" (rseq_scratch[1]), 569 [rseq_scratch2] "m" (rseq_scratch[2]) 570 : "memory", "cc", "rax" 571 RSEQ_INJECT_CLOBBER 572 : abort, cmpfail 573 #ifdef RSEQ_COMPARE_TWICE 574 , error1, error2 575 #endif 576 ); 577 return 0; 578 abort: 579 RSEQ_INJECT_FAILED 580 return -1; 581 cmpfail: 582 return 1; 583 #ifdef RSEQ_COMPARE_TWICE 584 error1: 585 rseq_bug("cpu_id comparison failed"); 586 error2: 587 rseq_bug("expected value comparison failed"); 588 #endif 589 } 590 591 /* x86-64 is TSO. */ 592 static inline __attribute__((always_inline)) 593 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 594 void *dst, void *src, size_t len, 595 intptr_t newv, int cpu) 596 { 597 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 598 newv, cpu); 599 } 600 601 #endif /* !RSEQ_SKIP_FASTPATH */ 602 603 #elif __i386__ 604 605 #define rseq_smp_mb() \ 606 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 607 #define rseq_smp_rmb() \ 608 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 609 #define rseq_smp_wmb() \ 610 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 611 612 #define rseq_smp_load_acquire(p) \ 613 __extension__ ({ \ 614 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 615 rseq_smp_mb(); \ 616 ____p1; \ 617 }) 618 619 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 620 621 #define rseq_smp_store_release(p, v) \ 622 do { \ 623 rseq_smp_mb(); \ 624 RSEQ_WRITE_ONCE(*p, v); \ 625 } while (0) 626 627 #ifdef RSEQ_SKIP_FASTPATH 628 #include "rseq-skip.h" 629 #else /* !RSEQ_SKIP_FASTPATH */ 630 631 /* 632 * Use eax as scratch register and take memory operands as input to 633 * lessen register pressure. Especially needed when compiling in O0. 634 */ 635 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 636 start_ip, post_commit_offset, abort_ip) \ 637 ".pushsection __rseq_cs, \"aw\"\n\t" \ 638 ".balign 32\n\t" \ 639 __rseq_str(label) ":\n\t" \ 640 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 641 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 642 ".popsection\n\t" \ 643 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 644 ".long " __rseq_str(label) "b, 0x0\n\t" \ 645 ".popsection\n\t" 646 647 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 648 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 649 (post_commit_ip - start_ip), abort_ip) 650 651 /* 652 * Exit points of a rseq critical section consist of all instructions outside 653 * of the critical section where a critical section can either branch to or 654 * reach through the normal course of its execution. The abort IP and the 655 * post-commit IP are already part of the __rseq_cs section and should not be 656 * explicitly defined as additional exit points. Knowing all exit points is 657 * useful to assist debuggers stepping over the critical section. 658 */ 659 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 660 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 661 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 662 ".popsection\n\t" 663 664 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 665 RSEQ_INJECT_ASM(1) \ 666 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ 667 __rseq_str(label) ":\n\t" 668 669 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 670 RSEQ_INJECT_ASM(2) \ 671 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 672 "jnz " __rseq_str(label) "\n\t" 673 674 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 675 ".pushsection __rseq_failure, \"ax\"\n\t" \ 676 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ 677 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 678 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 679 __rseq_str(label) ":\n\t" \ 680 teardown \ 681 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 682 ".popsection\n\t" 683 684 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 685 ".pushsection __rseq_failure, \"ax\"\n\t" \ 686 __rseq_str(label) ":\n\t" \ 687 teardown \ 688 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 689 ".popsection\n\t" 690 691 static inline __attribute__((always_inline)) 692 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 693 { 694 RSEQ_INJECT_C(9) 695 696 __asm__ __volatile__ goto ( 697 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 698 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 699 #ifdef RSEQ_COMPARE_TWICE 700 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 701 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 702 #endif 703 /* Start rseq by storing table entry pointer into rseq_cs. */ 704 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 705 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 706 RSEQ_INJECT_ASM(3) 707 "cmpl %[v], %[expect]\n\t" 708 "jnz %l[cmpfail]\n\t" 709 RSEQ_INJECT_ASM(4) 710 #ifdef RSEQ_COMPARE_TWICE 711 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 712 "cmpl %[v], %[expect]\n\t" 713 "jnz %l[error2]\n\t" 714 #endif 715 /* final store */ 716 "movl %[newv], %[v]\n\t" 717 "2:\n\t" 718 RSEQ_INJECT_ASM(5) 719 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 720 : /* gcc asm goto does not allow outputs */ 721 : [cpu_id] "r" (cpu), 722 [rseq_abi] "r" (&__rseq_abi), 723 [v] "m" (*v), 724 [expect] "r" (expect), 725 [newv] "r" (newv) 726 : "memory", "cc", "eax" 727 RSEQ_INJECT_CLOBBER 728 : abort, cmpfail 729 #ifdef RSEQ_COMPARE_TWICE 730 , error1, error2 731 #endif 732 ); 733 return 0; 734 abort: 735 RSEQ_INJECT_FAILED 736 return -1; 737 cmpfail: 738 return 1; 739 #ifdef RSEQ_COMPARE_TWICE 740 error1: 741 rseq_bug("cpu_id comparison failed"); 742 error2: 743 rseq_bug("expected value comparison failed"); 744 #endif 745 } 746 747 /* 748 * Compare @v against @expectnot. When it does _not_ match, load @v 749 * into @load, and store the content of *@v + voffp into @v. 750 */ 751 static inline __attribute__((always_inline)) 752 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 753 off_t voffp, intptr_t *load, int cpu) 754 { 755 RSEQ_INJECT_C(9) 756 757 __asm__ __volatile__ goto ( 758 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 759 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 760 #ifdef RSEQ_COMPARE_TWICE 761 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 762 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 763 #endif 764 /* Start rseq by storing table entry pointer into rseq_cs. */ 765 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 766 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 767 RSEQ_INJECT_ASM(3) 768 "movl %[v], %%ebx\n\t" 769 "cmpl %%ebx, %[expectnot]\n\t" 770 "je %l[cmpfail]\n\t" 771 RSEQ_INJECT_ASM(4) 772 #ifdef RSEQ_COMPARE_TWICE 773 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 774 "movl %[v], %%ebx\n\t" 775 "cmpl %%ebx, %[expectnot]\n\t" 776 "je %l[error2]\n\t" 777 #endif 778 "movl %%ebx, %[load]\n\t" 779 "addl %[voffp], %%ebx\n\t" 780 "movl (%%ebx), %%ebx\n\t" 781 /* final store */ 782 "movl %%ebx, %[v]\n\t" 783 "2:\n\t" 784 RSEQ_INJECT_ASM(5) 785 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 786 : /* gcc asm goto does not allow outputs */ 787 : [cpu_id] "r" (cpu), 788 [rseq_abi] "r" (&__rseq_abi), 789 /* final store input */ 790 [v] "m" (*v), 791 [expectnot] "r" (expectnot), 792 [voffp] "ir" (voffp), 793 [load] "m" (*load) 794 : "memory", "cc", "eax", "ebx" 795 RSEQ_INJECT_CLOBBER 796 : abort, cmpfail 797 #ifdef RSEQ_COMPARE_TWICE 798 , error1, error2 799 #endif 800 ); 801 return 0; 802 abort: 803 RSEQ_INJECT_FAILED 804 return -1; 805 cmpfail: 806 return 1; 807 #ifdef RSEQ_COMPARE_TWICE 808 error1: 809 rseq_bug("cpu_id comparison failed"); 810 error2: 811 rseq_bug("expected value comparison failed"); 812 #endif 813 } 814 815 static inline __attribute__((always_inline)) 816 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 817 { 818 RSEQ_INJECT_C(9) 819 820 __asm__ __volatile__ goto ( 821 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 822 #ifdef RSEQ_COMPARE_TWICE 823 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 824 #endif 825 /* Start rseq by storing table entry pointer into rseq_cs. */ 826 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 827 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 828 RSEQ_INJECT_ASM(3) 829 #ifdef RSEQ_COMPARE_TWICE 830 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 831 #endif 832 /* final store */ 833 "addl %[count], %[v]\n\t" 834 "2:\n\t" 835 RSEQ_INJECT_ASM(4) 836 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 837 : /* gcc asm goto does not allow outputs */ 838 : [cpu_id] "r" (cpu), 839 [rseq_abi] "r" (&__rseq_abi), 840 /* final store input */ 841 [v] "m" (*v), 842 [count] "ir" (count) 843 : "memory", "cc", "eax" 844 RSEQ_INJECT_CLOBBER 845 : abort 846 #ifdef RSEQ_COMPARE_TWICE 847 , error1 848 #endif 849 ); 850 return 0; 851 abort: 852 RSEQ_INJECT_FAILED 853 return -1; 854 #ifdef RSEQ_COMPARE_TWICE 855 error1: 856 rseq_bug("cpu_id comparison failed"); 857 #endif 858 } 859 860 static inline __attribute__((always_inline)) 861 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 862 intptr_t *v2, intptr_t newv2, 863 intptr_t newv, int cpu) 864 { 865 RSEQ_INJECT_C(9) 866 867 __asm__ __volatile__ goto ( 868 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 869 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 870 #ifdef RSEQ_COMPARE_TWICE 871 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 872 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 873 #endif 874 /* Start rseq by storing table entry pointer into rseq_cs. */ 875 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 876 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 877 RSEQ_INJECT_ASM(3) 878 "cmpl %[v], %[expect]\n\t" 879 "jnz %l[cmpfail]\n\t" 880 RSEQ_INJECT_ASM(4) 881 #ifdef RSEQ_COMPARE_TWICE 882 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 883 "cmpl %[v], %[expect]\n\t" 884 "jnz %l[error2]\n\t" 885 #endif 886 /* try store */ 887 "movl %[newv2], %%eax\n\t" 888 "movl %%eax, %[v2]\n\t" 889 RSEQ_INJECT_ASM(5) 890 /* final store */ 891 "movl %[newv], %[v]\n\t" 892 "2:\n\t" 893 RSEQ_INJECT_ASM(6) 894 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 895 : /* gcc asm goto does not allow outputs */ 896 : [cpu_id] "r" (cpu), 897 [rseq_abi] "r" (&__rseq_abi), 898 /* try store input */ 899 [v2] "m" (*v2), 900 [newv2] "m" (newv2), 901 /* final store input */ 902 [v] "m" (*v), 903 [expect] "r" (expect), 904 [newv] "r" (newv) 905 : "memory", "cc", "eax" 906 RSEQ_INJECT_CLOBBER 907 : abort, cmpfail 908 #ifdef RSEQ_COMPARE_TWICE 909 , error1, error2 910 #endif 911 ); 912 return 0; 913 abort: 914 RSEQ_INJECT_FAILED 915 return -1; 916 cmpfail: 917 return 1; 918 #ifdef RSEQ_COMPARE_TWICE 919 error1: 920 rseq_bug("cpu_id comparison failed"); 921 error2: 922 rseq_bug("expected value comparison failed"); 923 #endif 924 } 925 926 static inline __attribute__((always_inline)) 927 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 928 intptr_t *v2, intptr_t newv2, 929 intptr_t newv, int cpu) 930 { 931 RSEQ_INJECT_C(9) 932 933 __asm__ __volatile__ goto ( 934 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 935 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 936 #ifdef RSEQ_COMPARE_TWICE 937 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 938 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 939 #endif 940 /* Start rseq by storing table entry pointer into rseq_cs. */ 941 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 942 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 943 RSEQ_INJECT_ASM(3) 944 "movl %[expect], %%eax\n\t" 945 "cmpl %[v], %%eax\n\t" 946 "jnz %l[cmpfail]\n\t" 947 RSEQ_INJECT_ASM(4) 948 #ifdef RSEQ_COMPARE_TWICE 949 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 950 "movl %[expect], %%eax\n\t" 951 "cmpl %[v], %%eax\n\t" 952 "jnz %l[error2]\n\t" 953 #endif 954 /* try store */ 955 "movl %[newv2], %[v2]\n\t" 956 RSEQ_INJECT_ASM(5) 957 "lock; addl $0,-128(%%esp)\n\t" 958 /* final store */ 959 "movl %[newv], %[v]\n\t" 960 "2:\n\t" 961 RSEQ_INJECT_ASM(6) 962 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 963 : /* gcc asm goto does not allow outputs */ 964 : [cpu_id] "r" (cpu), 965 [rseq_abi] "r" (&__rseq_abi), 966 /* try store input */ 967 [v2] "m" (*v2), 968 [newv2] "r" (newv2), 969 /* final store input */ 970 [v] "m" (*v), 971 [expect] "m" (expect), 972 [newv] "r" (newv) 973 : "memory", "cc", "eax" 974 RSEQ_INJECT_CLOBBER 975 : abort, cmpfail 976 #ifdef RSEQ_COMPARE_TWICE 977 , error1, error2 978 #endif 979 ); 980 return 0; 981 abort: 982 RSEQ_INJECT_FAILED 983 return -1; 984 cmpfail: 985 return 1; 986 #ifdef RSEQ_COMPARE_TWICE 987 error1: 988 rseq_bug("cpu_id comparison failed"); 989 error2: 990 rseq_bug("expected value comparison failed"); 991 #endif 992 993 } 994 995 static inline __attribute__((always_inline)) 996 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 997 intptr_t *v2, intptr_t expect2, 998 intptr_t newv, int cpu) 999 { 1000 RSEQ_INJECT_C(9) 1001 1002 __asm__ __volatile__ goto ( 1003 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1004 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1005 #ifdef RSEQ_COMPARE_TWICE 1006 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1007 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1008 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 1009 #endif 1010 /* Start rseq by storing table entry pointer into rseq_cs. */ 1011 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1012 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1013 RSEQ_INJECT_ASM(3) 1014 "cmpl %[v], %[expect]\n\t" 1015 "jnz %l[cmpfail]\n\t" 1016 RSEQ_INJECT_ASM(4) 1017 "cmpl %[expect2], %[v2]\n\t" 1018 "jnz %l[cmpfail]\n\t" 1019 RSEQ_INJECT_ASM(5) 1020 #ifdef RSEQ_COMPARE_TWICE 1021 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 1022 "cmpl %[v], %[expect]\n\t" 1023 "jnz %l[error2]\n\t" 1024 "cmpl %[expect2], %[v2]\n\t" 1025 "jnz %l[error3]\n\t" 1026 #endif 1027 "movl %[newv], %%eax\n\t" 1028 /* final store */ 1029 "movl %%eax, %[v]\n\t" 1030 "2:\n\t" 1031 RSEQ_INJECT_ASM(6) 1032 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 1033 : /* gcc asm goto does not allow outputs */ 1034 : [cpu_id] "r" (cpu), 1035 [rseq_abi] "r" (&__rseq_abi), 1036 /* cmp2 input */ 1037 [v2] "m" (*v2), 1038 [expect2] "r" (expect2), 1039 /* final store input */ 1040 [v] "m" (*v), 1041 [expect] "r" (expect), 1042 [newv] "m" (newv) 1043 : "memory", "cc", "eax" 1044 RSEQ_INJECT_CLOBBER 1045 : abort, cmpfail 1046 #ifdef RSEQ_COMPARE_TWICE 1047 , error1, error2, error3 1048 #endif 1049 ); 1050 return 0; 1051 abort: 1052 RSEQ_INJECT_FAILED 1053 return -1; 1054 cmpfail: 1055 return 1; 1056 #ifdef RSEQ_COMPARE_TWICE 1057 error1: 1058 rseq_bug("cpu_id comparison failed"); 1059 error2: 1060 rseq_bug("1st expected value comparison failed"); 1061 error3: 1062 rseq_bug("2nd expected value comparison failed"); 1063 #endif 1064 } 1065 1066 /* TODO: implement a faster memcpy. */ 1067 static inline __attribute__((always_inline)) 1068 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 1069 void *dst, void *src, size_t len, 1070 intptr_t newv, int cpu) 1071 { 1072 uint32_t rseq_scratch[3]; 1073 1074 RSEQ_INJECT_C(9) 1075 1076 __asm__ __volatile__ goto ( 1077 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1078 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1079 #ifdef RSEQ_COMPARE_TWICE 1080 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1081 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1082 #endif 1083 "movl %[src], %[rseq_scratch0]\n\t" 1084 "movl %[dst], %[rseq_scratch1]\n\t" 1085 "movl %[len], %[rseq_scratch2]\n\t" 1086 /* Start rseq by storing table entry pointer into rseq_cs. */ 1087 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1088 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1089 RSEQ_INJECT_ASM(3) 1090 "movl %[expect], %%eax\n\t" 1091 "cmpl %%eax, %[v]\n\t" 1092 "jnz 5f\n\t" 1093 RSEQ_INJECT_ASM(4) 1094 #ifdef RSEQ_COMPARE_TWICE 1095 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 1096 "movl %[expect], %%eax\n\t" 1097 "cmpl %%eax, %[v]\n\t" 1098 "jnz 7f\n\t" 1099 #endif 1100 /* try memcpy */ 1101 "test %[len], %[len]\n\t" \ 1102 "jz 333f\n\t" \ 1103 "222:\n\t" \ 1104 "movb (%[src]), %%al\n\t" \ 1105 "movb %%al, (%[dst])\n\t" \ 1106 "inc %[src]\n\t" \ 1107 "inc %[dst]\n\t" \ 1108 "dec %[len]\n\t" \ 1109 "jnz 222b\n\t" \ 1110 "333:\n\t" \ 1111 RSEQ_INJECT_ASM(5) 1112 "movl %[newv], %%eax\n\t" 1113 /* final store */ 1114 "movl %%eax, %[v]\n\t" 1115 "2:\n\t" 1116 RSEQ_INJECT_ASM(6) 1117 /* teardown */ 1118 "movl %[rseq_scratch2], %[len]\n\t" 1119 "movl %[rseq_scratch1], %[dst]\n\t" 1120 "movl %[rseq_scratch0], %[src]\n\t" 1121 RSEQ_ASM_DEFINE_ABORT(4, 1122 "movl %[rseq_scratch2], %[len]\n\t" 1123 "movl %[rseq_scratch1], %[dst]\n\t" 1124 "movl %[rseq_scratch0], %[src]\n\t", 1125 abort) 1126 RSEQ_ASM_DEFINE_CMPFAIL(5, 1127 "movl %[rseq_scratch2], %[len]\n\t" 1128 "movl %[rseq_scratch1], %[dst]\n\t" 1129 "movl %[rseq_scratch0], %[src]\n\t", 1130 cmpfail) 1131 #ifdef RSEQ_COMPARE_TWICE 1132 RSEQ_ASM_DEFINE_CMPFAIL(6, 1133 "movl %[rseq_scratch2], %[len]\n\t" 1134 "movl %[rseq_scratch1], %[dst]\n\t" 1135 "movl %[rseq_scratch0], %[src]\n\t", 1136 error1) 1137 RSEQ_ASM_DEFINE_CMPFAIL(7, 1138 "movl %[rseq_scratch2], %[len]\n\t" 1139 "movl %[rseq_scratch1], %[dst]\n\t" 1140 "movl %[rseq_scratch0], %[src]\n\t", 1141 error2) 1142 #endif 1143 : /* gcc asm goto does not allow outputs */ 1144 : [cpu_id] "r" (cpu), 1145 [rseq_abi] "r" (&__rseq_abi), 1146 /* final store input */ 1147 [v] "m" (*v), 1148 [expect] "m" (expect), 1149 [newv] "m" (newv), 1150 /* try memcpy input */ 1151 [dst] "r" (dst), 1152 [src] "r" (src), 1153 [len] "r" (len), 1154 [rseq_scratch0] "m" (rseq_scratch[0]), 1155 [rseq_scratch1] "m" (rseq_scratch[1]), 1156 [rseq_scratch2] "m" (rseq_scratch[2]) 1157 : "memory", "cc", "eax" 1158 RSEQ_INJECT_CLOBBER 1159 : abort, cmpfail 1160 #ifdef RSEQ_COMPARE_TWICE 1161 , error1, error2 1162 #endif 1163 ); 1164 return 0; 1165 abort: 1166 RSEQ_INJECT_FAILED 1167 return -1; 1168 cmpfail: 1169 return 1; 1170 #ifdef RSEQ_COMPARE_TWICE 1171 error1: 1172 rseq_bug("cpu_id comparison failed"); 1173 error2: 1174 rseq_bug("expected value comparison failed"); 1175 #endif 1176 } 1177 1178 /* TODO: implement a faster memcpy. */ 1179 static inline __attribute__((always_inline)) 1180 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 1181 void *dst, void *src, size_t len, 1182 intptr_t newv, int cpu) 1183 { 1184 uint32_t rseq_scratch[3]; 1185 1186 RSEQ_INJECT_C(9) 1187 1188 __asm__ __volatile__ goto ( 1189 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1190 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1191 #ifdef RSEQ_COMPARE_TWICE 1192 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1193 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1194 #endif 1195 "movl %[src], %[rseq_scratch0]\n\t" 1196 "movl %[dst], %[rseq_scratch1]\n\t" 1197 "movl %[len], %[rseq_scratch2]\n\t" 1198 /* Start rseq by storing table entry pointer into rseq_cs. */ 1199 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1200 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1201 RSEQ_INJECT_ASM(3) 1202 "movl %[expect], %%eax\n\t" 1203 "cmpl %%eax, %[v]\n\t" 1204 "jnz 5f\n\t" 1205 RSEQ_INJECT_ASM(4) 1206 #ifdef RSEQ_COMPARE_TWICE 1207 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 1208 "movl %[expect], %%eax\n\t" 1209 "cmpl %%eax, %[v]\n\t" 1210 "jnz 7f\n\t" 1211 #endif 1212 /* try memcpy */ 1213 "test %[len], %[len]\n\t" \ 1214 "jz 333f\n\t" \ 1215 "222:\n\t" \ 1216 "movb (%[src]), %%al\n\t" \ 1217 "movb %%al, (%[dst])\n\t" \ 1218 "inc %[src]\n\t" \ 1219 "inc %[dst]\n\t" \ 1220 "dec %[len]\n\t" \ 1221 "jnz 222b\n\t" \ 1222 "333:\n\t" \ 1223 RSEQ_INJECT_ASM(5) 1224 "lock; addl $0,-128(%%esp)\n\t" 1225 "movl %[newv], %%eax\n\t" 1226 /* final store */ 1227 "movl %%eax, %[v]\n\t" 1228 "2:\n\t" 1229 RSEQ_INJECT_ASM(6) 1230 /* teardown */ 1231 "movl %[rseq_scratch2], %[len]\n\t" 1232 "movl %[rseq_scratch1], %[dst]\n\t" 1233 "movl %[rseq_scratch0], %[src]\n\t" 1234 RSEQ_ASM_DEFINE_ABORT(4, 1235 "movl %[rseq_scratch2], %[len]\n\t" 1236 "movl %[rseq_scratch1], %[dst]\n\t" 1237 "movl %[rseq_scratch0], %[src]\n\t", 1238 abort) 1239 RSEQ_ASM_DEFINE_CMPFAIL(5, 1240 "movl %[rseq_scratch2], %[len]\n\t" 1241 "movl %[rseq_scratch1], %[dst]\n\t" 1242 "movl %[rseq_scratch0], %[src]\n\t", 1243 cmpfail) 1244 #ifdef RSEQ_COMPARE_TWICE 1245 RSEQ_ASM_DEFINE_CMPFAIL(6, 1246 "movl %[rseq_scratch2], %[len]\n\t" 1247 "movl %[rseq_scratch1], %[dst]\n\t" 1248 "movl %[rseq_scratch0], %[src]\n\t", 1249 error1) 1250 RSEQ_ASM_DEFINE_CMPFAIL(7, 1251 "movl %[rseq_scratch2], %[len]\n\t" 1252 "movl %[rseq_scratch1], %[dst]\n\t" 1253 "movl %[rseq_scratch0], %[src]\n\t", 1254 error2) 1255 #endif 1256 : /* gcc asm goto does not allow outputs */ 1257 : [cpu_id] "r" (cpu), 1258 [rseq_abi] "r" (&__rseq_abi), 1259 /* final store input */ 1260 [v] "m" (*v), 1261 [expect] "m" (expect), 1262 [newv] "m" (newv), 1263 /* try memcpy input */ 1264 [dst] "r" (dst), 1265 [src] "r" (src), 1266 [len] "r" (len), 1267 [rseq_scratch0] "m" (rseq_scratch[0]), 1268 [rseq_scratch1] "m" (rseq_scratch[1]), 1269 [rseq_scratch2] "m" (rseq_scratch[2]) 1270 : "memory", "cc", "eax" 1271 RSEQ_INJECT_CLOBBER 1272 : abort, cmpfail 1273 #ifdef RSEQ_COMPARE_TWICE 1274 , error1, error2 1275 #endif 1276 ); 1277 return 0; 1278 abort: 1279 RSEQ_INJECT_FAILED 1280 return -1; 1281 cmpfail: 1282 return 1; 1283 #ifdef RSEQ_COMPARE_TWICE 1284 error1: 1285 rseq_bug("cpu_id comparison failed"); 1286 error2: 1287 rseq_bug("expected value comparison failed"); 1288 #endif 1289 } 1290 1291 #endif /* !RSEQ_SKIP_FASTPATH */ 1292 1293 #endif 1294