1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #ifndef RSEQ_H 9 #define RSEQ_H 10 11 #include <stdint.h> 12 #include <stdbool.h> 13 #include <pthread.h> 14 #include <signal.h> 15 #include <sched.h> 16 #include <errno.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <stddef.h> 20 #include "rseq-abi.h" 21 #include "compiler.h" 22 23 #ifndef rseq_sizeof_field 24 #define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) 25 #endif 26 27 #ifndef rseq_offsetofend 28 #define rseq_offsetofend(TYPE, MEMBER) \ 29 (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER)) 30 #endif 31 32 /* 33 * Empty code injection macros, override when testing. 34 * It is important to consider that the ASM injection macros need to be 35 * fully reentrant (e.g. do not modify the stack). 36 */ 37 #ifndef RSEQ_INJECT_ASM 38 #define RSEQ_INJECT_ASM(n) 39 #endif 40 41 #ifndef RSEQ_INJECT_C 42 #define RSEQ_INJECT_C(n) 43 #endif 44 45 #ifndef RSEQ_INJECT_INPUT 46 #define RSEQ_INJECT_INPUT 47 #endif 48 49 #ifndef RSEQ_INJECT_CLOBBER 50 #define RSEQ_INJECT_CLOBBER 51 #endif 52 53 #ifndef RSEQ_INJECT_FAILED 54 #define RSEQ_INJECT_FAILED 55 #endif 56 57 #include "rseq-thread-pointer.h" 58 59 /* Offset from the thread pointer to the rseq area. */ 60 extern ptrdiff_t rseq_offset; 61 62 /* 63 * The rseq ABI is composed of extensible feature fields. The extensions 64 * are done by appending additional fields at the end of the structure. 65 * The rseq_size defines the size of the active feature set which can be 66 * used by the application for the current rseq registration. Features 67 * starting at offset >= rseq_size are inactive and should not be used. 68 * 69 * The rseq_size is the intersection between the available allocation 70 * size for the rseq area and the feature size supported by the kernel. 71 * unsuccessful. 72 */ 73 extern unsigned int rseq_size; 74 75 /* Flags used during rseq registration. */ 76 extern unsigned int rseq_flags; 77 78 enum rseq_mo { 79 RSEQ_MO_RELAXED = 0, 80 RSEQ_MO_CONSUME = 1, /* Unused */ 81 RSEQ_MO_ACQUIRE = 2, /* Unused */ 82 RSEQ_MO_RELEASE = 3, 83 RSEQ_MO_ACQ_REL = 4, /* Unused */ 84 RSEQ_MO_SEQ_CST = 5, /* Unused */ 85 }; 86 87 enum rseq_percpu_mode { 88 RSEQ_PERCPU_CPU_ID = 0, 89 RSEQ_PERCPU_MM_CID = 1, 90 }; 91 92 static inline struct rseq_abi *rseq_get_abi(void) 93 { 94 return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); 95 } 96 97 #define rseq_likely(x) __builtin_expect(!!(x), 1) 98 #define rseq_unlikely(x) __builtin_expect(!!(x), 0) 99 #define rseq_barrier() __asm__ __volatile__("" : : : "memory") 100 101 #define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x)) 102 #define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); }) 103 #define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x) 104 105 #define __rseq_str_1(x) #x 106 #define __rseq_str(x) __rseq_str_1(x) 107 108 #define rseq_log(fmt, args...) \ 109 fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \ 110 ## args, __func__) 111 112 #define rseq_bug(fmt, args...) \ 113 do { \ 114 rseq_log(fmt, ##args); \ 115 abort(); \ 116 } while (0) 117 118 #if defined(__x86_64__) || defined(__i386__) 119 #include <rseq-x86.h> 120 #elif defined(__ARMEL__) 121 #include <rseq-arm.h> 122 #elif defined (__AARCH64EL__) 123 #include <rseq-arm64.h> 124 #elif defined(__PPC__) 125 #include <rseq-ppc.h> 126 #elif defined(__mips__) 127 #include <rseq-mips.h> 128 #elif defined(__s390__) 129 #include <rseq-s390.h> 130 #elif defined(__riscv) 131 #include <rseq-riscv.h> 132 #elif defined(__or1k__) 133 #include <rseq-or1k.h> 134 #else 135 #error unsupported target 136 #endif 137 138 /* 139 * Register rseq for the current thread. This needs to be called once 140 * by any thread which uses restartable sequences, before they start 141 * using restartable sequences, to ensure restartable sequences 142 * succeed. A restartable sequence executed from a non-registered 143 * thread will always fail. 144 */ 145 int rseq_register_current_thread(void); 146 147 /* 148 * Unregister rseq for current thread. 149 */ 150 int rseq_unregister_current_thread(void); 151 152 /* 153 * Restartable sequence fallback for reading the current CPU number. 154 */ 155 int32_t rseq_fallback_current_cpu(void); 156 157 /* 158 * Restartable sequence fallback for reading the current node number. 159 */ 160 int32_t rseq_fallback_current_node(void); 161 162 /* 163 * Values returned can be either the current CPU number, -1 (rseq is 164 * uninitialized), or -2 (rseq initialization has failed). 165 */ 166 static inline int32_t rseq_current_cpu_raw(void) 167 { 168 return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id); 169 } 170 171 /* 172 * Returns a possible CPU number, which is typically the current CPU. 173 * The returned CPU number can be used to prepare for an rseq critical 174 * section, which will confirm whether the cpu number is indeed the 175 * current one, and whether rseq is initialized. 176 * 177 * The CPU number returned by rseq_cpu_start should always be validated 178 * by passing it to a rseq asm sequence, or by comparing it to the 179 * return value of rseq_current_cpu_raw() if the rseq asm sequence 180 * does not need to be invoked. 181 */ 182 static inline uint32_t rseq_cpu_start(void) 183 { 184 return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start); 185 } 186 187 static inline uint32_t rseq_current_cpu(void) 188 { 189 int32_t cpu; 190 191 cpu = rseq_current_cpu_raw(); 192 if (rseq_unlikely(cpu < 0)) 193 cpu = rseq_fallback_current_cpu(); 194 return cpu; 195 } 196 197 static inline bool rseq_node_id_available(void) 198 { 199 return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, node_id); 200 } 201 202 /* 203 * Current NUMA node number. 204 */ 205 static inline uint32_t rseq_current_node_id(void) 206 { 207 assert(rseq_node_id_available()); 208 return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id); 209 } 210 211 static inline bool rseq_mm_cid_available(void) 212 { 213 return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, mm_cid); 214 } 215 216 static inline uint32_t rseq_current_mm_cid(void) 217 { 218 return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid); 219 } 220 221 static inline void rseq_clear_rseq_cs(void) 222 { 223 RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); 224 } 225 226 /* 227 * rseq_prepare_unload() should be invoked by each thread executing a rseq 228 * critical section at least once between their last critical section and 229 * library unload of the library defining the rseq critical section (struct 230 * rseq_cs) or the code referred to by the struct rseq_cs start_ip and 231 * post_commit_offset fields. This also applies to use of rseq in code 232 * generated by JIT: rseq_prepare_unload() should be invoked at least once by 233 * each thread executing a rseq critical section before reclaim of the memory 234 * holding the struct rseq_cs or reclaim of the code pointed to by struct 235 * rseq_cs start_ip and post_commit_offset fields. 236 */ 237 static inline void rseq_prepare_unload(void) 238 { 239 rseq_clear_rseq_cs(); 240 } 241 242 static inline __attribute__((always_inline)) 243 int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 244 intptr_t *v, intptr_t expect, 245 intptr_t newv, int cpu) 246 { 247 if (rseq_mo != RSEQ_MO_RELAXED) 248 return -1; 249 switch (percpu_mode) { 250 case RSEQ_PERCPU_CPU_ID: 251 return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu); 252 case RSEQ_PERCPU_MM_CID: 253 return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu); 254 } 255 return -1; 256 } 257 258 /* 259 * Compare @v against @expectnot. When it does _not_ match, load @v 260 * into @load, and store the content of *@v + voffp into @v. 261 */ 262 static inline __attribute__((always_inline)) 263 int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 264 intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, 265 int cpu) 266 { 267 if (rseq_mo != RSEQ_MO_RELAXED) 268 return -1; 269 switch (percpu_mode) { 270 case RSEQ_PERCPU_CPU_ID: 271 return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu); 272 case RSEQ_PERCPU_MM_CID: 273 return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu); 274 } 275 return -1; 276 } 277 278 static inline __attribute__((always_inline)) 279 int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 280 intptr_t *v, intptr_t count, int cpu) 281 { 282 if (rseq_mo != RSEQ_MO_RELAXED) 283 return -1; 284 switch (percpu_mode) { 285 case RSEQ_PERCPU_CPU_ID: 286 return rseq_addv_relaxed_cpu_id(v, count, cpu); 287 case RSEQ_PERCPU_MM_CID: 288 return rseq_addv_relaxed_mm_cid(v, count, cpu); 289 } 290 return -1; 291 } 292 293 #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 294 /* 295 * pval = *(ptr+off) 296 * *pval += inc; 297 */ 298 static inline __attribute__((always_inline)) 299 int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 300 intptr_t *ptr, long off, intptr_t inc, int cpu) 301 { 302 if (rseq_mo != RSEQ_MO_RELAXED) 303 return -1; 304 switch (percpu_mode) { 305 case RSEQ_PERCPU_CPU_ID: 306 return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu); 307 case RSEQ_PERCPU_MM_CID: 308 return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu); 309 } 310 return -1; 311 } 312 #endif 313 314 static inline __attribute__((always_inline)) 315 int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 316 intptr_t *v, intptr_t expect, 317 intptr_t *v2, intptr_t newv2, 318 intptr_t newv, int cpu) 319 { 320 switch (rseq_mo) { 321 case RSEQ_MO_RELAXED: 322 switch (percpu_mode) { 323 case RSEQ_PERCPU_CPU_ID: 324 return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu); 325 case RSEQ_PERCPU_MM_CID: 326 return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu); 327 } 328 return -1; 329 case RSEQ_MO_RELEASE: 330 switch (percpu_mode) { 331 case RSEQ_PERCPU_CPU_ID: 332 return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu); 333 case RSEQ_PERCPU_MM_CID: 334 return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu); 335 } 336 return -1; 337 default: 338 return -1; 339 } 340 } 341 342 static inline __attribute__((always_inline)) 343 int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 344 intptr_t *v, intptr_t expect, 345 intptr_t *v2, intptr_t expect2, 346 intptr_t newv, int cpu) 347 { 348 if (rseq_mo != RSEQ_MO_RELAXED) 349 return -1; 350 switch (percpu_mode) { 351 case RSEQ_PERCPU_CPU_ID: 352 return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu); 353 case RSEQ_PERCPU_MM_CID: 354 return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu); 355 } 356 return -1; 357 } 358 359 static inline __attribute__((always_inline)) 360 int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, 361 intptr_t *v, intptr_t expect, 362 void *dst, void *src, size_t len, 363 intptr_t newv, int cpu) 364 { 365 switch (rseq_mo) { 366 case RSEQ_MO_RELAXED: 367 switch (percpu_mode) { 368 case RSEQ_PERCPU_CPU_ID: 369 return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu); 370 case RSEQ_PERCPU_MM_CID: 371 return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu); 372 } 373 return -1; 374 case RSEQ_MO_RELEASE: 375 switch (percpu_mode) { 376 case RSEQ_PERCPU_CPU_ID: 377 return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu); 378 case RSEQ_PERCPU_MM_CID: 379 return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu); 380 } 381 return -1; 382 default: 383 return -1; 384 } 385 } 386 387 #endif /* RSEQ_H_ */ 388