1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3 #define RSEQ_SIG 0x53053053 4 5 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory") 6 #define rseq_smp_rmb() rseq_smp_mb() 7 #define rseq_smp_wmb() rseq_smp_mb() 8 9 #define rseq_smp_load_acquire(p) \ 10 __extension__ ({ \ 11 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 12 rseq_barrier(); \ 13 ____p1; \ 14 }) 15 16 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 17 18 #define rseq_smp_store_release(p, v) \ 19 do { \ 20 rseq_barrier(); \ 21 RSEQ_WRITE_ONCE(*p, v); \ 22 } while (0) 23 24 #ifdef RSEQ_SKIP_FASTPATH 25 #include "rseq-skip.h" 26 #else /* !RSEQ_SKIP_FASTPATH */ 27 28 #ifdef __s390x__ 29 30 #define LONG_L "lg" 31 #define LONG_S "stg" 32 #define LONG_LT_R "ltgr" 33 #define LONG_CMP "cg" 34 #define LONG_CMP_R "cgr" 35 #define LONG_ADDI "aghi" 36 #define LONG_ADD_R "agr" 37 38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 39 start_ip, post_commit_offset, abort_ip) \ 40 ".pushsection __rseq_table, \"aw\"\n\t" \ 41 ".balign 32\n\t" \ 42 __rseq_str(label) ":\n\t" \ 43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 45 ".popsection\n\t" 46 47 #elif __s390__ 48 49 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 50 start_ip, post_commit_offset, abort_ip) \ 51 ".pushsection __rseq_table, \"aw\"\n\t" \ 52 ".balign 32\n\t" \ 53 __rseq_str(label) ":\n\t" \ 54 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 55 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 56 ".popsection\n\t" 57 58 #define LONG_L "l" 59 #define LONG_S "st" 60 #define LONG_LT_R "ltr" 61 #define LONG_CMP "c" 62 #define LONG_CMP_R "cr" 63 #define LONG_ADDI "ahi" 64 #define LONG_ADD_R "ar" 65 66 #endif 67 68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 69 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 70 (post_commit_ip - start_ip), abort_ip) 71 72 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 73 RSEQ_INJECT_ASM(1) \ 74 "larl %%r0, " __rseq_str(cs_label) "\n\t" \ 75 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \ 76 __rseq_str(label) ":\n\t" 77 78 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 79 RSEQ_INJECT_ASM(2) \ 80 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 81 "jnz " __rseq_str(label) "\n\t" 82 83 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 84 ".pushsection __rseq_failure, \"ax\"\n\t" \ 85 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 86 __rseq_str(label) ":\n\t" \ 87 teardown \ 88 "j %l[" __rseq_str(abort_label) "]\n\t" \ 89 ".popsection\n\t" 90 91 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 92 ".pushsection __rseq_failure, \"ax\"\n\t" \ 93 __rseq_str(label) ":\n\t" \ 94 teardown \ 95 "j %l[" __rseq_str(cmpfail_label) "]\n\t" \ 96 ".popsection\n\t" 97 98 static inline __attribute__((always_inline)) 99 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 100 { 101 RSEQ_INJECT_C(9) 102 103 __asm__ __volatile__ goto ( 104 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 105 /* Start rseq by storing table entry pointer into rseq_cs. */ 106 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 107 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 108 RSEQ_INJECT_ASM(3) 109 LONG_CMP " %[expect], %[v]\n\t" 110 "jnz %l[cmpfail]\n\t" 111 RSEQ_INJECT_ASM(4) 112 #ifdef RSEQ_COMPARE_TWICE 113 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 114 LONG_CMP " %[expect], %[v]\n\t" 115 "jnz %l[error2]\n\t" 116 #endif 117 /* final store */ 118 LONG_S " %[newv], %[v]\n\t" 119 "2:\n\t" 120 RSEQ_INJECT_ASM(5) 121 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 122 : /* gcc asm goto does not allow outputs */ 123 : [cpu_id] "r" (cpu), 124 [current_cpu_id] "m" (__rseq_abi.cpu_id), 125 [rseq_cs] "m" (__rseq_abi.rseq_cs), 126 [v] "m" (*v), 127 [expect] "r" (expect), 128 [newv] "r" (newv) 129 RSEQ_INJECT_INPUT 130 : "memory", "cc", "r0" 131 RSEQ_INJECT_CLOBBER 132 : abort, cmpfail 133 #ifdef RSEQ_COMPARE_TWICE 134 , error1, error2 135 #endif 136 ); 137 return 0; 138 abort: 139 RSEQ_INJECT_FAILED 140 return -1; 141 cmpfail: 142 return 1; 143 #ifdef RSEQ_COMPARE_TWICE 144 error1: 145 rseq_bug("cpu_id comparison failed"); 146 error2: 147 rseq_bug("expected value comparison failed"); 148 #endif 149 } 150 151 /* 152 * Compare @v against @expectnot. When it does _not_ match, load @v 153 * into @load, and store the content of *@v + voffp into @v. 154 */ 155 static inline __attribute__((always_inline)) 156 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 157 off_t voffp, intptr_t *load, int cpu) 158 { 159 RSEQ_INJECT_C(9) 160 161 __asm__ __volatile__ goto ( 162 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 163 /* Start rseq by storing table entry pointer into rseq_cs. */ 164 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 165 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 166 RSEQ_INJECT_ASM(3) 167 LONG_L " %%r1, %[v]\n\t" 168 LONG_CMP_R " %%r1, %[expectnot]\n\t" 169 "je %l[cmpfail]\n\t" 170 RSEQ_INJECT_ASM(4) 171 #ifdef RSEQ_COMPARE_TWICE 172 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 173 LONG_L " %%r1, %[v]\n\t" 174 LONG_CMP_R " %%r1, %[expectnot]\n\t" 175 "je %l[error2]\n\t" 176 #endif 177 LONG_S " %%r1, %[load]\n\t" 178 LONG_ADD_R " %%r1, %[voffp]\n\t" 179 LONG_L " %%r1, 0(%%r1)\n\t" 180 /* final store */ 181 LONG_S " %%r1, %[v]\n\t" 182 "2:\n\t" 183 RSEQ_INJECT_ASM(5) 184 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 185 : /* gcc asm goto does not allow outputs */ 186 : [cpu_id] "r" (cpu), 187 [current_cpu_id] "m" (__rseq_abi.cpu_id), 188 [rseq_cs] "m" (__rseq_abi.rseq_cs), 189 /* final store input */ 190 [v] "m" (*v), 191 [expectnot] "r" (expectnot), 192 [voffp] "r" (voffp), 193 [load] "m" (*load) 194 RSEQ_INJECT_INPUT 195 : "memory", "cc", "r0", "r1" 196 RSEQ_INJECT_CLOBBER 197 : abort, cmpfail 198 #ifdef RSEQ_COMPARE_TWICE 199 , error1, error2 200 #endif 201 ); 202 return 0; 203 abort: 204 RSEQ_INJECT_FAILED 205 return -1; 206 cmpfail: 207 return 1; 208 #ifdef RSEQ_COMPARE_TWICE 209 error1: 210 rseq_bug("cpu_id comparison failed"); 211 error2: 212 rseq_bug("expected value comparison failed"); 213 #endif 214 } 215 216 static inline __attribute__((always_inline)) 217 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 218 { 219 RSEQ_INJECT_C(9) 220 221 __asm__ __volatile__ goto ( 222 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 223 /* Start rseq by storing table entry pointer into rseq_cs. */ 224 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 226 RSEQ_INJECT_ASM(3) 227 #ifdef RSEQ_COMPARE_TWICE 228 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 229 #endif 230 LONG_L " %%r0, %[v]\n\t" 231 LONG_ADD_R " %%r0, %[count]\n\t" 232 /* final store */ 233 LONG_S " %%r0, %[v]\n\t" 234 "2:\n\t" 235 RSEQ_INJECT_ASM(4) 236 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 237 : /* gcc asm goto does not allow outputs */ 238 : [cpu_id] "r" (cpu), 239 [current_cpu_id] "m" (__rseq_abi.cpu_id), 240 [rseq_cs] "m" (__rseq_abi.rseq_cs), 241 /* final store input */ 242 [v] "m" (*v), 243 [count] "r" (count) 244 RSEQ_INJECT_INPUT 245 : "memory", "cc", "r0" 246 RSEQ_INJECT_CLOBBER 247 : abort 248 #ifdef RSEQ_COMPARE_TWICE 249 , error1 250 #endif 251 ); 252 return 0; 253 abort: 254 RSEQ_INJECT_FAILED 255 return -1; 256 #ifdef RSEQ_COMPARE_TWICE 257 error1: 258 rseq_bug("cpu_id comparison failed"); 259 #endif 260 } 261 262 static inline __attribute__((always_inline)) 263 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 264 intptr_t *v2, intptr_t newv2, 265 intptr_t newv, int cpu) 266 { 267 RSEQ_INJECT_C(9) 268 269 __asm__ __volatile__ goto ( 270 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 271 /* Start rseq by storing table entry pointer into rseq_cs. */ 272 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 273 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 274 RSEQ_INJECT_ASM(3) 275 LONG_CMP " %[expect], %[v]\n\t" 276 "jnz %l[cmpfail]\n\t" 277 RSEQ_INJECT_ASM(4) 278 #ifdef RSEQ_COMPARE_TWICE 279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 280 LONG_CMP " %[expect], %[v]\n\t" 281 "jnz %l[error2]\n\t" 282 #endif 283 /* try store */ 284 LONG_S " %[newv2], %[v2]\n\t" 285 RSEQ_INJECT_ASM(5) 286 /* final store */ 287 LONG_S " %[newv], %[v]\n\t" 288 "2:\n\t" 289 RSEQ_INJECT_ASM(6) 290 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 291 : /* gcc asm goto does not allow outputs */ 292 : [cpu_id] "r" (cpu), 293 [current_cpu_id] "m" (__rseq_abi.cpu_id), 294 [rseq_cs] "m" (__rseq_abi.rseq_cs), 295 /* try store input */ 296 [v2] "m" (*v2), 297 [newv2] "r" (newv2), 298 /* final store input */ 299 [v] "m" (*v), 300 [expect] "r" (expect), 301 [newv] "r" (newv) 302 RSEQ_INJECT_INPUT 303 : "memory", "cc", "r0" 304 RSEQ_INJECT_CLOBBER 305 : abort, cmpfail 306 #ifdef RSEQ_COMPARE_TWICE 307 , error1, error2 308 #endif 309 ); 310 return 0; 311 abort: 312 RSEQ_INJECT_FAILED 313 return -1; 314 cmpfail: 315 return 1; 316 #ifdef RSEQ_COMPARE_TWICE 317 error1: 318 rseq_bug("cpu_id comparison failed"); 319 error2: 320 rseq_bug("expected value comparison failed"); 321 #endif 322 } 323 324 /* s390 is TSO. */ 325 static inline __attribute__((always_inline)) 326 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 327 intptr_t *v2, intptr_t newv2, 328 intptr_t newv, int cpu) 329 { 330 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); 331 } 332 333 static inline __attribute__((always_inline)) 334 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 335 intptr_t *v2, intptr_t expect2, 336 intptr_t newv, int cpu) 337 { 338 RSEQ_INJECT_C(9) 339 340 __asm__ __volatile__ goto ( 341 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 342 /* Start rseq by storing table entry pointer into rseq_cs. */ 343 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 344 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 345 RSEQ_INJECT_ASM(3) 346 LONG_CMP " %[expect], %[v]\n\t" 347 "jnz %l[cmpfail]\n\t" 348 RSEQ_INJECT_ASM(4) 349 LONG_CMP " %[expect2], %[v2]\n\t" 350 "jnz %l[cmpfail]\n\t" 351 RSEQ_INJECT_ASM(5) 352 #ifdef RSEQ_COMPARE_TWICE 353 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 354 LONG_CMP " %[expect], %[v]\n\t" 355 "jnz %l[error2]\n\t" 356 LONG_CMP " %[expect2], %[v2]\n\t" 357 "jnz %l[error3]\n\t" 358 #endif 359 /* final store */ 360 LONG_S " %[newv], %[v]\n\t" 361 "2:\n\t" 362 RSEQ_INJECT_ASM(6) 363 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 364 : /* gcc asm goto does not allow outputs */ 365 : [cpu_id] "r" (cpu), 366 [current_cpu_id] "m" (__rseq_abi.cpu_id), 367 [rseq_cs] "m" (__rseq_abi.rseq_cs), 368 /* cmp2 input */ 369 [v2] "m" (*v2), 370 [expect2] "r" (expect2), 371 /* final store input */ 372 [v] "m" (*v), 373 [expect] "r" (expect), 374 [newv] "r" (newv) 375 RSEQ_INJECT_INPUT 376 : "memory", "cc", "r0" 377 RSEQ_INJECT_CLOBBER 378 : abort, cmpfail 379 #ifdef RSEQ_COMPARE_TWICE 380 , error1, error2, error3 381 #endif 382 ); 383 return 0; 384 abort: 385 RSEQ_INJECT_FAILED 386 return -1; 387 cmpfail: 388 return 1; 389 #ifdef RSEQ_COMPARE_TWICE 390 error1: 391 rseq_bug("cpu_id comparison failed"); 392 error2: 393 rseq_bug("1st expected value comparison failed"); 394 error3: 395 rseq_bug("2nd expected value comparison failed"); 396 #endif 397 } 398 399 static inline __attribute__((always_inline)) 400 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 401 void *dst, void *src, size_t len, 402 intptr_t newv, int cpu) 403 { 404 uint64_t rseq_scratch[3]; 405 406 RSEQ_INJECT_C(9) 407 408 __asm__ __volatile__ goto ( 409 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 410 LONG_S " %[src], %[rseq_scratch0]\n\t" 411 LONG_S " %[dst], %[rseq_scratch1]\n\t" 412 LONG_S " %[len], %[rseq_scratch2]\n\t" 413 /* Start rseq by storing table entry pointer into rseq_cs. */ 414 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 415 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 416 RSEQ_INJECT_ASM(3) 417 LONG_CMP " %[expect], %[v]\n\t" 418 "jnz 5f\n\t" 419 RSEQ_INJECT_ASM(4) 420 #ifdef RSEQ_COMPARE_TWICE 421 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 422 LONG_CMP " %[expect], %[v]\n\t" 423 "jnz 7f\n\t" 424 #endif 425 /* try memcpy */ 426 LONG_LT_R " %[len], %[len]\n\t" 427 "jz 333f\n\t" 428 "222:\n\t" 429 "ic %%r0,0(%[src])\n\t" 430 "stc %%r0,0(%[dst])\n\t" 431 LONG_ADDI " %[src], 1\n\t" 432 LONG_ADDI " %[dst], 1\n\t" 433 LONG_ADDI " %[len], -1\n\t" 434 "jnz 222b\n\t" 435 "333:\n\t" 436 RSEQ_INJECT_ASM(5) 437 /* final store */ 438 LONG_S " %[newv], %[v]\n\t" 439 "2:\n\t" 440 RSEQ_INJECT_ASM(6) 441 /* teardown */ 442 LONG_L " %[len], %[rseq_scratch2]\n\t" 443 LONG_L " %[dst], %[rseq_scratch1]\n\t" 444 LONG_L " %[src], %[rseq_scratch0]\n\t" 445 RSEQ_ASM_DEFINE_ABORT(4, 446 LONG_L " %[len], %[rseq_scratch2]\n\t" 447 LONG_L " %[dst], %[rseq_scratch1]\n\t" 448 LONG_L " %[src], %[rseq_scratch0]\n\t", 449 abort) 450 RSEQ_ASM_DEFINE_CMPFAIL(5, 451 LONG_L " %[len], %[rseq_scratch2]\n\t" 452 LONG_L " %[dst], %[rseq_scratch1]\n\t" 453 LONG_L " %[src], %[rseq_scratch0]\n\t", 454 cmpfail) 455 #ifdef RSEQ_COMPARE_TWICE 456 RSEQ_ASM_DEFINE_CMPFAIL(6, 457 LONG_L " %[len], %[rseq_scratch2]\n\t" 458 LONG_L " %[dst], %[rseq_scratch1]\n\t" 459 LONG_L " %[src], %[rseq_scratch0]\n\t", 460 error1) 461 RSEQ_ASM_DEFINE_CMPFAIL(7, 462 LONG_L " %[len], %[rseq_scratch2]\n\t" 463 LONG_L " %[dst], %[rseq_scratch1]\n\t" 464 LONG_L " %[src], %[rseq_scratch0]\n\t", 465 error2) 466 #endif 467 : /* gcc asm goto does not allow outputs */ 468 : [cpu_id] "r" (cpu), 469 [current_cpu_id] "m" (__rseq_abi.cpu_id), 470 [rseq_cs] "m" (__rseq_abi.rseq_cs), 471 /* final store input */ 472 [v] "m" (*v), 473 [expect] "r" (expect), 474 [newv] "r" (newv), 475 /* try memcpy input */ 476 [dst] "r" (dst), 477 [src] "r" (src), 478 [len] "r" (len), 479 [rseq_scratch0] "m" (rseq_scratch[0]), 480 [rseq_scratch1] "m" (rseq_scratch[1]), 481 [rseq_scratch2] "m" (rseq_scratch[2]) 482 RSEQ_INJECT_INPUT 483 : "memory", "cc", "r0" 484 RSEQ_INJECT_CLOBBER 485 : abort, cmpfail 486 #ifdef RSEQ_COMPARE_TWICE 487 , error1, error2 488 #endif 489 ); 490 return 0; 491 abort: 492 RSEQ_INJECT_FAILED 493 return -1; 494 cmpfail: 495 return 1; 496 #ifdef RSEQ_COMPARE_TWICE 497 error1: 498 rseq_bug("cpu_id comparison failed"); 499 error2: 500 rseq_bug("expected value comparison failed"); 501 #endif 502 } 503 504 /* s390 is TSO. */ 505 static inline __attribute__((always_inline)) 506 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 507 void *dst, void *src, size_t len, 508 intptr_t newv, int cpu) 509 { 510 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, 511 newv, cpu); 512 } 513 #endif /* !RSEQ_SKIP_FASTPATH */ 514