1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #ifndef RSEQ_H 9 #define RSEQ_H 10 11 #include <stdint.h> 12 #include <stdbool.h> 13 #include <pthread.h> 14 #include <signal.h> 15 #include <sched.h> 16 #include <errno.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <stddef.h> 20 #include "rseq-abi.h" 21 #include "compiler.h" 22 23 #ifndef rseq_sizeof_field 24 #define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) 25 #endif 26 27 #ifndef rseq_offsetofend 28 #define rseq_offsetofend(TYPE, MEMBER) \ 29 (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER)) 30 #endif 31 32 /* 33 * Empty code injection macros, override when testing. 34 * It is important to consider that the ASM injection macros need to be 35 * fully reentrant (e.g. do not modify the stack). 36 */ 37 #ifndef RSEQ_INJECT_ASM 38 #define RSEQ_INJECT_ASM(n) 39 #endif 40 41 #ifndef RSEQ_INJECT_C 42 #define RSEQ_INJECT_C(n) 43 #endif 44 45 #ifndef RSEQ_INJECT_INPUT 46 #define RSEQ_INJECT_INPUT 47 #endif 48 49 #ifndef RSEQ_INJECT_CLOBBER 50 #define RSEQ_INJECT_CLOBBER 51 #endif 52 53 #ifndef RSEQ_INJECT_FAILED 54 #define RSEQ_INJECT_FAILED 55 #endif 56 57 #include "rseq-thread-pointer.h" 58 59 /* Offset from the thread pointer to the rseq area. */ 60 extern ptrdiff_t rseq_offset; 61 62 /* 63 * The rseq ABI is composed of extensible feature fields. The extensions 64 * are done by appending additional fields at the end of the structure. 65 * The rseq_size defines the size of the active feature set which can be 66 * used by the application for the current rseq registration. Features 67 * starting at offset >= rseq_size are inactive and should not be used. 68 * 69 * The rseq_size is the intersection between the available allocation 70 * size for the rseq area and the feature size supported by the kernel. 71 * unsuccessful. 72 */ 73 extern unsigned int rseq_size; 74 75 /* Flags used during rseq registration. */ 76 extern unsigned int rseq_flags; 77 78 enum rseq_mo { 79 RSEQ_MO_RELAXED = 0, 80 RSEQ_MO_CONSUME = 1, /* Unused */ 81 RSEQ_MO_ACQUIRE = 2, /* Unused */ 82 RSEQ_MO_RELEASE = 3, 83 RSEQ_MO_ACQ_REL = 4, /* Unused */ 84 RSEQ_MO_SEQ_CST = 5, /* Unused */ 85 }; 86 87 enum rseq_percpu_mode { 88 RSEQ_PERCPU_CPU_ID = 0, 89 RSEQ_PERCPU_MM_CID = 1, 90 }; 91 92 static inline struct rseq_abi *rseq_get_abi(void) 93 { 94 return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); 95 } 96 97 #define rseq_likely(x) __builtin_expect(!!(x), 1) 98 #define rseq_unlikely(x) __builtin_expect(!!(x), 0) 99 #define rseq_barrier() __asm__ __volatile__("" : : : "memory") 100 101 #define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x)) 102 #define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); }) 103 #define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x) 104 105 #define __rseq_str_1(x) #x 106 #define __rseq_str(x) __rseq_str_1(x) 107 108 #define rseq_log(fmt, args...) \ 109 fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \ 110 ## args, __func__) 111 112 #define rseq_bug(fmt, args...) \ 113 do { \ 114 rseq_log(fmt, ##args); \ 115 abort(); \ 116 } while (0) 117 118 #if defined(__x86_64__) || defined(__i386__) 119 #include <rseq-x86.h> 120 #elif defined(__ARMEL__) 121 #include <rseq-arm.h> 122 #elif defined (__AARCH64EL__) 123 #include <rseq-arm64.h> 124 #elif defined(__PPC__) 125 #include <rseq-ppc.h> 126 #elif defined(__mips__) 127 #include <rseq-mips.h> 128 #elif defined(__s390__) 129 #include <rseq-s390.h> 130 #elif defined(__riscv) 131 #include <rseq-riscv.h> 132 #else 133 #error unsupported target 134 #endif 135 136 /* 137 * Register rseq for the current thread. This needs to be called once 138 * by any thread which uses restartable sequences, before they start 139 * using restartable sequences, to ensure restartable sequences 140 * succeed. A restartable sequence executed from a non-registered 141 * thread will always fail. 142 */ 143 int rseq_register_current_thread(void); 144 145 /* 146 * Unregister rseq for current thread. 147 */ 148 int rseq_unregister_current_thread(void); 149 150 /* 151 * Restartable sequence fallback for reading the current CPU number. 152 */ 153 int32_t rseq_fallback_current_cpu(void); 154 155 /* 156 * Restartable sequence fallback for reading the current node number. 157 */ 158 int32_t rseq_fallback_current_node(void); 159 160 /* 161 * Values returned can be either the current CPU number, -1 (rseq is 162 * uninitialized), or -2 (rseq initialization has failed). 163 */ 164 static inline int32_t rseq_current_cpu_raw(void) 165 { 166 return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id); 167 } 168 169 /* 170 * Returns a possible CPU number, which is typically the current CPU. 171 * The returned CPU number can be used to prepare for an rseq critical 172 * section, which will confirm whether the cpu number is indeed the 173 * current one, and whether rseq is initialized. 174 * 175 * The CPU number returned by rseq_cpu_start should always be validated 176 * by passing it to a rseq asm sequence, or by comparing it to the 177 * return value of rseq_current_cpu_raw() if the rseq asm sequence 178 * does not need to be invoked. 179 */ 180 static inline uint32_t rseq_cpu_start(void) 181 { 182 return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start); 183 } 184 185 static inline uint32_t rseq_current_cpu(void) 186 { 187 int32_t cpu; 188 189 cpu = rseq_current_cpu_raw(); 190 if (rseq_unlikely(cpu < 0)) 191 cpu = rseq_fallback_current_cpu(); 192 return cpu; 193 } 194 195 static inline bool rseq_node_id_available(void) 196 { 197 return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, node_id); 198 } 199 200 /* 201 * Current NUMA node number. 202 */ 203 static inline uint32_t rseq_current_node_id(void) 204 { 205 assert(rseq_node_id_available()); 206 return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id); 207 } 208 209 static inline bool rseq_mm_cid_available(void) 210 { 211 return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, mm_cid); 212 } 213 214 static inline uint32_t rseq_current_mm_cid(void) 215 { 216 return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid); 217 } 218 219 static inline void rseq_clear_rseq_cs(void) 220 { 221 RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); 222 } 223 224 /* 225 * rseq_prepare_unload() should be invoked by each thread executing a rseq 226 * critical section at least once between their last critical section and 227 * library unload of the library defining the rseq critical section (struct 228 * rseq_cs) or the code referred to by the struct rseq_cs start_ip and 229 * post_commit_offset fields. This also applies to use of rseq in code 230 * generated by JIT: rseq_prepare_unload() should be invoked at least once by 231 * each thread executing a rseq critical section before reclaim of the memory 232 * holding the struct rseq_cs or reclaim of the code pointed to by struct 233 * rseq_cs start_ip and post_commit_offset fields. 234 */ 235 static inline void rseq_prepare_unload(void) 236 { 237 rseq_clear_rseq_cs(); 238 } 239 240 static inline __attribute__((always_inline)) 241 int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 242 intptr_t *v, intptr_t expect, 243 intptr_t newv, int cpu) 244 { 245 if (rseq_mo != RSEQ_MO_RELAXED) 246 return -1; 247 switch (percpu_mode) { 248 case RSEQ_PERCPU_CPU_ID: 249 return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu); 250 case RSEQ_PERCPU_MM_CID: 251 return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu); 252 } 253 return -1; 254 } 255 256 /* 257 * Compare @v against @expectnot. When it does _not_ match, load @v 258 * into @load, and store the content of *@v + voffp into @v. 259 */ 260 static inline __attribute__((always_inline)) 261 int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 262 intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, 263 int cpu) 264 { 265 if (rseq_mo != RSEQ_MO_RELAXED) 266 return -1; 267 switch (percpu_mode) { 268 case RSEQ_PERCPU_CPU_ID: 269 return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu); 270 case RSEQ_PERCPU_MM_CID: 271 return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu); 272 } 273 return -1; 274 } 275 276 static inline __attribute__((always_inline)) 277 int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 278 intptr_t *v, intptr_t count, int cpu) 279 { 280 if (rseq_mo != RSEQ_MO_RELAXED) 281 return -1; 282 switch (percpu_mode) { 283 case RSEQ_PERCPU_CPU_ID: 284 return rseq_addv_relaxed_cpu_id(v, count, cpu); 285 case RSEQ_PERCPU_MM_CID: 286 return rseq_addv_relaxed_mm_cid(v, count, cpu); 287 } 288 return -1; 289 } 290 291 #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 292 /* 293 * pval = *(ptr+off) 294 * *pval += inc; 295 */ 296 static inline __attribute__((always_inline)) 297 int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 298 intptr_t *ptr, long off, intptr_t inc, int cpu) 299 { 300 if (rseq_mo != RSEQ_MO_RELAXED) 301 return -1; 302 switch (percpu_mode) { 303 case RSEQ_PERCPU_CPU_ID: 304 return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu); 305 case RSEQ_PERCPU_MM_CID: 306 return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu); 307 } 308 return -1; 309 } 310 #endif 311 312 static inline __attribute__((always_inline)) 313 int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 314 intptr_t *v, intptr_t expect, 315 intptr_t *v2, intptr_t newv2, 316 intptr_t newv, int cpu) 317 { 318 switch (rseq_mo) { 319 case RSEQ_MO_RELAXED: 320 switch (percpu_mode) { 321 case RSEQ_PERCPU_CPU_ID: 322 return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu); 323 case RSEQ_PERCPU_MM_CID: 324 return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu); 325 } 326 return -1; 327 case RSEQ_MO_RELEASE: 328 switch (percpu_mode) { 329 case RSEQ_PERCPU_CPU_ID: 330 return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu); 331 case RSEQ_PERCPU_MM_CID: 332 return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu); 333 } 334 return -1; 335 default: 336 return -1; 337 } 338 } 339 340 static inline __attribute__((always_inline)) 341 int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 342 intptr_t *v, intptr_t expect, 343 intptr_t *v2, intptr_t expect2, 344 intptr_t newv, int cpu) 345 { 346 if (rseq_mo != RSEQ_MO_RELAXED) 347 return -1; 348 switch (percpu_mode) { 349 case RSEQ_PERCPU_CPU_ID: 350 return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu); 351 case RSEQ_PERCPU_MM_CID: 352 return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu); 353 } 354 return -1; 355 } 356 357 static inline __attribute__((always_inline)) 358 int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 359 intptr_t *v, intptr_t expect, 360 void *dst, void *src, size_t len, 361 intptr_t newv, int cpu) 362 { 363 switch (rseq_mo) { 364 case RSEQ_MO_RELAXED: 365 switch (percpu_mode) { 366 case RSEQ_PERCPU_CPU_ID: 367 return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu); 368 case RSEQ_PERCPU_MM_CID: 369 return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu); 370 } 371 return -1; 372 case RSEQ_MO_RELEASE: 373 switch (percpu_mode) { 374 case RSEQ_PERCPU_CPU_ID: 375 return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu); 376 case RSEQ_PERCPU_MM_CID: 377 return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu); 378 } 379 return -1; 380 default: 381 return -1; 382 } 383 } 384 385 #endif /* RSEQ_H_ */ 386