1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-ppc-bits.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> 7 */ 8 9 #include "rseq-bits-template.h" 10 11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 13 14 static inline __attribute__((always_inline)) 15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 16 { 17 RSEQ_INJECT_C(9) 18 19 __asm__ __volatile__ goto ( 20 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 21 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 22 #ifdef RSEQ_COMPARE_TWICE 23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 25 #endif 26 /* Start rseq by storing table entry pointer into rseq_cs. */ 27 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 28 /* cmp cpuid */ 29 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 30 RSEQ_INJECT_ASM(3) 31 /* cmp @v equal to @expect */ 32 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 33 RSEQ_INJECT_ASM(4) 34 #ifdef RSEQ_COMPARE_TWICE 35 /* cmp cpuid */ 36 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 37 /* cmp @v equal to @expect */ 38 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 39 #endif 40 /* final store */ 41 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 42 RSEQ_INJECT_ASM(5) 43 RSEQ_ASM_DEFINE_ABORT(4, abort) 44 : /* gcc asm goto does not allow outputs */ 45 : [cpu_id] "r" (cpu), 46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 48 [v] "m" (*v), 49 [expect] "r" (expect), 50 [newv] "r" (newv) 51 RSEQ_INJECT_INPUT 52 : "memory", "cc", "r17" 53 RSEQ_INJECT_CLOBBER 54 : abort, cmpfail 55 #ifdef RSEQ_COMPARE_TWICE 56 , error1, error2 57 #endif 58 ); 59 rseq_after_asm_goto(); 60 return 0; 61 abort: 62 rseq_after_asm_goto(); 63 RSEQ_INJECT_FAILED 64 return -1; 65 cmpfail: 66 rseq_after_asm_goto(); 67 return 1; 68 #ifdef RSEQ_COMPARE_TWICE 69 error1: 70 rseq_after_asm_goto(); 71 rseq_bug("cpu_id comparison failed"); 72 error2: 73 rseq_after_asm_goto(); 74 rseq_bug("expected value comparison failed"); 75 #endif 76 } 77 78 static inline __attribute__((always_inline)) 79 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 80 long voffp, intptr_t *load, int cpu) 81 { 82 RSEQ_INJECT_C(9) 83 84 __asm__ __volatile__ goto ( 85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 87 #ifdef RSEQ_COMPARE_TWICE 88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 89 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 90 #endif 91 /* Start rseq by storing table entry pointer into rseq_cs. */ 92 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 93 /* cmp cpuid */ 94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 95 RSEQ_INJECT_ASM(3) 96 /* cmp @v not equal to @expectnot */ 97 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 98 RSEQ_INJECT_ASM(4) 99 #ifdef RSEQ_COMPARE_TWICE 100 /* cmp cpuid */ 101 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 102 /* cmp @v not equal to @expectnot */ 103 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 104 #endif 105 /* load the value of @v */ 106 RSEQ_ASM_OP_R_LOAD(v) 107 /* store it in @load */ 108 RSEQ_ASM_OP_R_STORE(load) 109 /* dereference voffp(v) */ 110 RSEQ_ASM_OP_R_LOADX(voffp) 111 /* final store the value at voffp(v) */ 112 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 113 RSEQ_INJECT_ASM(5) 114 RSEQ_ASM_DEFINE_ABORT(4, abort) 115 : /* gcc asm goto does not allow outputs */ 116 : [cpu_id] "r" (cpu), 117 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 118 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 119 /* final store input */ 120 [v] "m" (*v), 121 [expectnot] "r" (expectnot), 122 [voffp] "b" (voffp), 123 [load] "m" (*load) 124 RSEQ_INJECT_INPUT 125 : "memory", "cc", "r17" 126 RSEQ_INJECT_CLOBBER 127 : abort, cmpfail 128 #ifdef RSEQ_COMPARE_TWICE 129 , error1, error2 130 #endif 131 ); 132 rseq_after_asm_goto(); 133 return 0; 134 abort: 135 rseq_after_asm_goto(); 136 RSEQ_INJECT_FAILED 137 return -1; 138 cmpfail: 139 rseq_after_asm_goto(); 140 return 1; 141 #ifdef RSEQ_COMPARE_TWICE 142 error1: 143 rseq_after_asm_goto(); 144 rseq_bug("cpu_id comparison failed"); 145 error2: 146 rseq_after_asm_goto(); 147 rseq_bug("expected value comparison failed"); 148 #endif 149 } 150 151 static inline __attribute__((always_inline)) 152 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 153 { 154 RSEQ_INJECT_C(9) 155 156 __asm__ __volatile__ goto ( 157 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 158 #ifdef RSEQ_COMPARE_TWICE 159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 160 #endif 161 /* Start rseq by storing table entry pointer into rseq_cs. */ 162 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 163 /* cmp cpuid */ 164 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 165 RSEQ_INJECT_ASM(3) 166 #ifdef RSEQ_COMPARE_TWICE 167 /* cmp cpuid */ 168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 169 #endif 170 /* load the value of @v */ 171 RSEQ_ASM_OP_R_LOAD(v) 172 /* add @count to it */ 173 RSEQ_ASM_OP_R_ADD(count) 174 /* final store */ 175 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 176 RSEQ_INJECT_ASM(4) 177 RSEQ_ASM_DEFINE_ABORT(4, abort) 178 : /* gcc asm goto does not allow outputs */ 179 : [cpu_id] "r" (cpu), 180 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 181 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 182 /* final store input */ 183 [v] "m" (*v), 184 [count] "r" (count) 185 RSEQ_INJECT_INPUT 186 : "memory", "cc", "r17" 187 RSEQ_INJECT_CLOBBER 188 : abort 189 #ifdef RSEQ_COMPARE_TWICE 190 , error1 191 #endif 192 ); 193 rseq_after_asm_goto(); 194 return 0; 195 abort: 196 rseq_after_asm_goto(); 197 RSEQ_INJECT_FAILED 198 return -1; 199 #ifdef RSEQ_COMPARE_TWICE 200 error1: 201 rseq_after_asm_goto(); 202 rseq_bug("cpu_id comparison failed"); 203 #endif 204 } 205 206 static inline __attribute__((always_inline)) 207 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 208 intptr_t *v2, intptr_t expect2, 209 intptr_t newv, int cpu) 210 { 211 RSEQ_INJECT_C(9) 212 213 __asm__ __volatile__ goto ( 214 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 216 #ifdef RSEQ_COMPARE_TWICE 217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 220 #endif 221 /* Start rseq by storing table entry pointer into rseq_cs. */ 222 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 223 /* cmp cpuid */ 224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 225 RSEQ_INJECT_ASM(3) 226 /* cmp @v equal to @expect */ 227 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 228 RSEQ_INJECT_ASM(4) 229 /* cmp @v2 equal to @expct2 */ 230 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 231 RSEQ_INJECT_ASM(5) 232 #ifdef RSEQ_COMPARE_TWICE 233 /* cmp cpuid */ 234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 235 /* cmp @v equal to @expect */ 236 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 237 /* cmp @v2 equal to @expct2 */ 238 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 239 #endif 240 /* final store */ 241 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 242 RSEQ_INJECT_ASM(6) 243 RSEQ_ASM_DEFINE_ABORT(4, abort) 244 : /* gcc asm goto does not allow outputs */ 245 : [cpu_id] "r" (cpu), 246 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 247 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 248 /* cmp2 input */ 249 [v2] "m" (*v2), 250 [expect2] "r" (expect2), 251 /* final store input */ 252 [v] "m" (*v), 253 [expect] "r" (expect), 254 [newv] "r" (newv) 255 RSEQ_INJECT_INPUT 256 : "memory", "cc", "r17" 257 RSEQ_INJECT_CLOBBER 258 : abort, cmpfail 259 #ifdef RSEQ_COMPARE_TWICE 260 , error1, error2, error3 261 #endif 262 ); 263 rseq_after_asm_goto(); 264 return 0; 265 abort: 266 rseq_after_asm_goto(); 267 RSEQ_INJECT_FAILED 268 return -1; 269 cmpfail: 270 rseq_after_asm_goto(); 271 return 1; 272 #ifdef RSEQ_COMPARE_TWICE 273 error1: 274 rseq_after_asm_goto(); 275 rseq_bug("cpu_id comparison failed"); 276 error2: 277 rseq_after_asm_goto(); 278 rseq_bug("1st expected value comparison failed"); 279 error3: 280 rseq_after_asm_goto(); 281 rseq_bug("2nd expected value comparison failed"); 282 #endif 283 } 284 285 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 286 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 287 288 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 289 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 290 291 static inline __attribute__((always_inline)) 292 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 293 intptr_t *v2, intptr_t newv2, 294 intptr_t newv, int cpu) 295 { 296 RSEQ_INJECT_C(9) 297 298 __asm__ __volatile__ goto ( 299 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 300 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 301 #ifdef RSEQ_COMPARE_TWICE 302 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 303 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 304 #endif 305 /* Start rseq by storing table entry pointer into rseq_cs. */ 306 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 307 /* cmp cpuid */ 308 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 309 RSEQ_INJECT_ASM(3) 310 /* cmp @v equal to @expect */ 311 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 312 RSEQ_INJECT_ASM(4) 313 #ifdef RSEQ_COMPARE_TWICE 314 /* cmp cpuid */ 315 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 316 /* cmp @v equal to @expect */ 317 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 318 #endif 319 /* try store */ 320 RSEQ_ASM_OP_STORE(newv2, v2) 321 RSEQ_INJECT_ASM(5) 322 #ifdef RSEQ_TEMPLATE_MO_RELEASE 323 /* for 'release' */ 324 "lwsync\n\t" 325 #endif 326 /* final store */ 327 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 328 RSEQ_INJECT_ASM(6) 329 RSEQ_ASM_DEFINE_ABORT(4, abort) 330 : /* gcc asm goto does not allow outputs */ 331 : [cpu_id] "r" (cpu), 332 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 333 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 334 /* try store input */ 335 [v2] "m" (*v2), 336 [newv2] "r" (newv2), 337 /* final store input */ 338 [v] "m" (*v), 339 [expect] "r" (expect), 340 [newv] "r" (newv) 341 RSEQ_INJECT_INPUT 342 : "memory", "cc", "r17" 343 RSEQ_INJECT_CLOBBER 344 : abort, cmpfail 345 #ifdef RSEQ_COMPARE_TWICE 346 , error1, error2 347 #endif 348 ); 349 rseq_after_asm_goto(); 350 return 0; 351 abort: 352 rseq_after_asm_goto(); 353 RSEQ_INJECT_FAILED 354 return -1; 355 cmpfail: 356 rseq_after_asm_goto(); 357 return 1; 358 #ifdef RSEQ_COMPARE_TWICE 359 error1: 360 rseq_after_asm_goto(); 361 rseq_bug("cpu_id comparison failed"); 362 error2: 363 rseq_after_asm_goto(); 364 rseq_bug("expected value comparison failed"); 365 #endif 366 } 367 368 static inline __attribute__((always_inline)) 369 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 370 void *dst, void *src, size_t len, 371 intptr_t newv, int cpu) 372 { 373 RSEQ_INJECT_C(9) 374 375 __asm__ __volatile__ goto ( 376 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 377 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 378 #ifdef RSEQ_COMPARE_TWICE 379 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 380 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 381 #endif 382 /* setup for mempcy */ 383 "mr %%r19, %[len]\n\t" 384 "mr %%r20, %[src]\n\t" 385 "mr %%r21, %[dst]\n\t" 386 /* Start rseq by storing table entry pointer into rseq_cs. */ 387 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 388 /* cmp cpuid */ 389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 390 RSEQ_INJECT_ASM(3) 391 /* cmp @v equal to @expect */ 392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 393 RSEQ_INJECT_ASM(4) 394 #ifdef RSEQ_COMPARE_TWICE 395 /* cmp cpuid */ 396 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 397 /* cmp @v equal to @expect */ 398 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 399 #endif 400 /* try memcpy */ 401 RSEQ_ASM_OP_R_MEMCPY() 402 RSEQ_INJECT_ASM(5) 403 #ifdef RSEQ_TEMPLATE_MO_RELEASE 404 /* for 'release' */ 405 "lwsync\n\t" 406 #endif 407 /* final store */ 408 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 409 RSEQ_INJECT_ASM(6) 410 /* teardown */ 411 RSEQ_ASM_DEFINE_ABORT(4, abort) 412 : /* gcc asm goto does not allow outputs */ 413 : [cpu_id] "r" (cpu), 414 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 415 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 416 /* final store input */ 417 [v] "m" (*v), 418 [expect] "r" (expect), 419 [newv] "r" (newv), 420 /* try memcpy input */ 421 [dst] "r" (dst), 422 [src] "r" (src), 423 [len] "r" (len) 424 RSEQ_INJECT_INPUT 425 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 426 RSEQ_INJECT_CLOBBER 427 : abort, cmpfail 428 #ifdef RSEQ_COMPARE_TWICE 429 , error1, error2 430 #endif 431 ); 432 rseq_after_asm_goto(); 433 return 0; 434 abort: 435 rseq_after_asm_goto(); 436 RSEQ_INJECT_FAILED 437 return -1; 438 cmpfail: 439 rseq_after_asm_goto(); 440 return 1; 441 #ifdef RSEQ_COMPARE_TWICE 442 error1: 443 rseq_after_asm_goto(); 444 rseq_bug("cpu_id comparison failed"); 445 error2: 446 rseq_after_asm_goto(); 447 rseq_bug("expected value comparison failed"); 448 #endif 449 } 450 451 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 452 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 453 454 #include "rseq-bits-reset.h" 455