1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #ifndef RSEQ_H 9 #define RSEQ_H 10 11 #include <stdint.h> 12 #include <stdbool.h> 13 #include <pthread.h> 14 #include <signal.h> 15 #include <sched.h> 16 #include <errno.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <stddef.h> 20 #include "rseq-abi.h" 21 #include "compiler.h" 22 23 #ifndef rseq_sizeof_field 24 #define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) 25 #endif 26 27 #ifndef rseq_offsetofend 28 #define rseq_offsetofend(TYPE, MEMBER) \ 29 (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER)) 30 #endif 31 32 /* 33 * Empty code injection macros, override when testing. 34 * It is important to consider that the ASM injection macros need to be 35 * fully reentrant (e.g. do not modify the stack). 36 */ 37 #ifndef RSEQ_INJECT_ASM 38 #define RSEQ_INJECT_ASM(n) 39 #endif 40 41 #ifndef RSEQ_INJECT_C 42 #define RSEQ_INJECT_C(n) 43 #endif 44 45 #ifndef RSEQ_INJECT_INPUT 46 #define RSEQ_INJECT_INPUT 47 #endif 48 49 #ifndef RSEQ_INJECT_CLOBBER 50 #define RSEQ_INJECT_CLOBBER 51 #endif 52 53 #ifndef RSEQ_INJECT_FAILED 54 #define RSEQ_INJECT_FAILED 55 #endif 56 57 #include "rseq-thread-pointer.h" 58 59 /* Offset from the thread pointer to the rseq area. */ 60 extern ptrdiff_t rseq_offset; 61 62 /* 63 * Size of the registered rseq area. 0 if the registration was 64 * unsuccessful. 65 */ 66 extern unsigned int rseq_size; 67 68 /* Flags used during rseq registration. */ 69 extern unsigned int rseq_flags; 70 71 /* 72 * rseq feature size supported by the kernel. 0 if the registration was 73 * unsuccessful. 74 */ 75 extern unsigned int rseq_feature_size; 76 77 enum rseq_mo { 78 RSEQ_MO_RELAXED = 0, 79 RSEQ_MO_CONSUME = 1, /* Unused */ 80 RSEQ_MO_ACQUIRE = 2, /* Unused */ 81 RSEQ_MO_RELEASE = 3, 82 RSEQ_MO_ACQ_REL = 4, /* Unused */ 83 RSEQ_MO_SEQ_CST = 5, /* Unused */ 84 }; 85 86 enum rseq_percpu_mode { 87 RSEQ_PERCPU_CPU_ID = 0, 88 RSEQ_PERCPU_MM_CID = 1, 89 }; 90 91 static inline struct rseq_abi *rseq_get_abi(void) 92 { 93 return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); 94 } 95 96 #define rseq_likely(x) __builtin_expect(!!(x), 1) 97 #define rseq_unlikely(x) __builtin_expect(!!(x), 0) 98 #define rseq_barrier() __asm__ __volatile__("" : : : "memory") 99 100 #define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x)) 101 #define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); }) 102 #define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x) 103 104 #define __rseq_str_1(x) #x 105 #define __rseq_str(x) __rseq_str_1(x) 106 107 #define rseq_log(fmt, args...) \ 108 fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \ 109 ## args, __func__) 110 111 #define rseq_bug(fmt, args...) \ 112 do { \ 113 rseq_log(fmt, ##args); \ 114 abort(); \ 115 } while (0) 116 117 #if defined(__x86_64__) || defined(__i386__) 118 #include <rseq-x86.h> 119 #elif defined(__ARMEL__) 120 #include <rseq-arm.h> 121 #elif defined (__AARCH64EL__) 122 #include <rseq-arm64.h> 123 #elif defined(__PPC__) 124 #include <rseq-ppc.h> 125 #elif defined(__mips__) 126 #include <rseq-mips.h> 127 #elif defined(__s390__) 128 #include <rseq-s390.h> 129 #elif defined(__riscv) 130 #include <rseq-riscv.h> 131 #else 132 #error unsupported target 133 #endif 134 135 /* 136 * Register rseq for the current thread. This needs to be called once 137 * by any thread which uses restartable sequences, before they start 138 * using restartable sequences, to ensure restartable sequences 139 * succeed. A restartable sequence executed from a non-registered 140 * thread will always fail. 141 */ 142 int rseq_register_current_thread(void); 143 144 /* 145 * Unregister rseq for current thread. 146 */ 147 int rseq_unregister_current_thread(void); 148 149 /* 150 * Restartable sequence fallback for reading the current CPU number. 151 */ 152 int32_t rseq_fallback_current_cpu(void); 153 154 /* 155 * Restartable sequence fallback for reading the current node number. 156 */ 157 int32_t rseq_fallback_current_node(void); 158 159 /* 160 * Values returned can be either the current CPU number, -1 (rseq is 161 * uninitialized), or -2 (rseq initialization has failed). 162 */ 163 static inline int32_t rseq_current_cpu_raw(void) 164 { 165 return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id); 166 } 167 168 /* 169 * Returns a possible CPU number, which is typically the current CPU. 170 * The returned CPU number can be used to prepare for an rseq critical 171 * section, which will confirm whether the cpu number is indeed the 172 * current one, and whether rseq is initialized. 173 * 174 * The CPU number returned by rseq_cpu_start should always be validated 175 * by passing it to a rseq asm sequence, or by comparing it to the 176 * return value of rseq_current_cpu_raw() if the rseq asm sequence 177 * does not need to be invoked. 178 */ 179 static inline uint32_t rseq_cpu_start(void) 180 { 181 return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start); 182 } 183 184 static inline uint32_t rseq_current_cpu(void) 185 { 186 int32_t cpu; 187 188 cpu = rseq_current_cpu_raw(); 189 if (rseq_unlikely(cpu < 0)) 190 cpu = rseq_fallback_current_cpu(); 191 return cpu; 192 } 193 194 static inline bool rseq_node_id_available(void) 195 { 196 return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, node_id); 197 } 198 199 /* 200 * Current NUMA node number. 201 */ 202 static inline uint32_t rseq_current_node_id(void) 203 { 204 assert(rseq_node_id_available()); 205 return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id); 206 } 207 208 static inline bool rseq_mm_cid_available(void) 209 { 210 return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, mm_cid); 211 } 212 213 static inline uint32_t rseq_current_mm_cid(void) 214 { 215 return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid); 216 } 217 218 static inline void rseq_clear_rseq_cs(void) 219 { 220 RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); 221 } 222 223 /* 224 * rseq_prepare_unload() should be invoked by each thread executing a rseq 225 * critical section at least once between their last critical section and 226 * library unload of the library defining the rseq critical section (struct 227 * rseq_cs) or the code referred to by the struct rseq_cs start_ip and 228 * post_commit_offset fields. This also applies to use of rseq in code 229 * generated by JIT: rseq_prepare_unload() should be invoked at least once by 230 * each thread executing a rseq critical section before reclaim of the memory 231 * holding the struct rseq_cs or reclaim of the code pointed to by struct 232 * rseq_cs start_ip and post_commit_offset fields. 233 */ 234 static inline void rseq_prepare_unload(void) 235 { 236 rseq_clear_rseq_cs(); 237 } 238 239 static inline __attribute__((always_inline)) 240 int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 241 intptr_t *v, intptr_t expect, 242 intptr_t newv, int cpu) 243 { 244 if (rseq_mo != RSEQ_MO_RELAXED) 245 return -1; 246 switch (percpu_mode) { 247 case RSEQ_PERCPU_CPU_ID: 248 return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu); 249 case RSEQ_PERCPU_MM_CID: 250 return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu); 251 } 252 return -1; 253 } 254 255 /* 256 * Compare @v against @expectnot. When it does _not_ match, load @v 257 * into @load, and store the content of *@v + voffp into @v. 258 */ 259 static inline __attribute__((always_inline)) 260 int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 261 intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, 262 int cpu) 263 { 264 if (rseq_mo != RSEQ_MO_RELAXED) 265 return -1; 266 switch (percpu_mode) { 267 case RSEQ_PERCPU_CPU_ID: 268 return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu); 269 case RSEQ_PERCPU_MM_CID: 270 return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu); 271 } 272 return -1; 273 } 274 275 static inline __attribute__((always_inline)) 276 int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 277 intptr_t *v, intptr_t count, int cpu) 278 { 279 if (rseq_mo != RSEQ_MO_RELAXED) 280 return -1; 281 switch (percpu_mode) { 282 case RSEQ_PERCPU_CPU_ID: 283 return rseq_addv_relaxed_cpu_id(v, count, cpu); 284 case RSEQ_PERCPU_MM_CID: 285 return rseq_addv_relaxed_mm_cid(v, count, cpu); 286 } 287 return -1; 288 } 289 290 #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 291 /* 292 * pval = *(ptr+off) 293 * *pval += inc; 294 */ 295 static inline __attribute__((always_inline)) 296 int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 297 intptr_t *ptr, long off, intptr_t inc, int cpu) 298 { 299 if (rseq_mo != RSEQ_MO_RELAXED) 300 return -1; 301 switch (percpu_mode) { 302 case RSEQ_PERCPU_CPU_ID: 303 return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu); 304 case RSEQ_PERCPU_MM_CID: 305 return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu); 306 } 307 return -1; 308 } 309 #endif 310 311 static inline __attribute__((always_inline)) 312 int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 313 intptr_t *v, intptr_t expect, 314 intptr_t *v2, intptr_t newv2, 315 intptr_t newv, int cpu) 316 { 317 switch (rseq_mo) { 318 case RSEQ_MO_RELAXED: 319 switch (percpu_mode) { 320 case RSEQ_PERCPU_CPU_ID: 321 return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu); 322 case RSEQ_PERCPU_MM_CID: 323 return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu); 324 } 325 return -1; 326 case RSEQ_MO_RELEASE: 327 switch (percpu_mode) { 328 case RSEQ_PERCPU_CPU_ID: 329 return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu); 330 case RSEQ_PERCPU_MM_CID: 331 return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu); 332 } 333 return -1; 334 default: 335 return -1; 336 } 337 } 338 339 static inline __attribute__((always_inline)) 340 int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 341 intptr_t *v, intptr_t expect, 342 intptr_t *v2, intptr_t expect2, 343 intptr_t newv, int cpu) 344 { 345 if (rseq_mo != RSEQ_MO_RELAXED) 346 return -1; 347 switch (percpu_mode) { 348 case RSEQ_PERCPU_CPU_ID: 349 return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu); 350 case RSEQ_PERCPU_MM_CID: 351 return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu); 352 } 353 return -1; 354 } 355 356 static inline __attribute__((always_inline)) 357 int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 358 intptr_t *v, intptr_t expect, 359 void *dst, void *src, size_t len, 360 intptr_t newv, int cpu) 361 { 362 switch (rseq_mo) { 363 case RSEQ_MO_RELAXED: 364 switch (percpu_mode) { 365 case RSEQ_PERCPU_CPU_ID: 366 return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu); 367 case RSEQ_PERCPU_MM_CID: 368 return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu); 369 } 370 return -1; 371 case RSEQ_MO_RELEASE: 372 switch (percpu_mode) { 373 case RSEQ_PERCPU_CPU_ID: 374 return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu); 375 case RSEQ_PERCPU_MM_CID: 376 return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu); 377 } 378 return -1; 379 default: 380 return -1; 381 } 382 } 383 384 #endif /* RSEQ_H_ */ 385