1 // SPDX-License-Identifier: LGPL-2.1 2 /* 3 * rseq.c 4 * 5 * Copyright (C) 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; only 10 * version 2.1 of the License. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 */ 17 18 #define _GNU_SOURCE 19 #include <errno.h> 20 #include <sched.h> 21 #include <stdio.h> 22 #include <stdlib.h> 23 #include <string.h> 24 #include <unistd.h> 25 #include <syscall.h> 26 #include <assert.h> 27 #include <signal.h> 28 #include <limits.h> 29 #include <dlfcn.h> 30 #include <stddef.h> 31 #include <sys/auxv.h> 32 #include <linux/auxvec.h> 33 34 #include <linux/compiler.h> 35 36 #include "../kselftest.h" 37 #include "rseq.h" 38 39 /* 40 * Define weak versions to play nice with binaries that are statically linked 41 * against a libc that doesn't support registering its own rseq. 42 */ 43 __weak ptrdiff_t __rseq_offset; 44 __weak unsigned int __rseq_size; 45 __weak unsigned int __rseq_flags; 46 47 static const ptrdiff_t *libc_rseq_offset_p = &__rseq_offset; 48 static const unsigned int *libc_rseq_size_p = &__rseq_size; 49 static const unsigned int *libc_rseq_flags_p = &__rseq_flags; 50 51 /* Offset from the thread pointer to the rseq area. */ 52 ptrdiff_t rseq_offset; 53 54 /* 55 * Size of the registered rseq area. 0 if the registration was 56 * unsuccessful. 57 */ 58 unsigned int rseq_size = -1U; 59 60 /* Flags used during rseq registration. */ 61 unsigned int rseq_flags; 62 63 /* 64 * rseq feature size supported by the kernel. 0 if the registration was 65 * unsuccessful. 66 */ 67 unsigned int rseq_feature_size = -1U; 68 69 static int rseq_ownership; 70 static int rseq_reg_success; /* At least one rseq registration has succeded. */ 71 72 /* Allocate a large area for the TLS. */ 73 #define RSEQ_THREAD_AREA_ALLOC_SIZE 1024 74 75 /* Original struct rseq feature size is 20 bytes. */ 76 #define ORIG_RSEQ_FEATURE_SIZE 20 77 78 /* Original struct rseq allocation size is 32 bytes. */ 79 #define ORIG_RSEQ_ALLOC_SIZE 32 80 81 static 82 __thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"), aligned(RSEQ_THREAD_AREA_ALLOC_SIZE))) = { 83 .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED, 84 }; 85 86 static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len, 87 int flags, uint32_t sig) 88 { 89 return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); 90 } 91 92 static int sys_getcpu(unsigned *cpu, unsigned *node) 93 { 94 return syscall(__NR_getcpu, cpu, node, NULL); 95 } 96 97 int rseq_available(void) 98 { 99 int rc; 100 101 rc = sys_rseq(NULL, 0, 0, 0); 102 if (rc != -1) 103 abort(); 104 switch (errno) { 105 case ENOSYS: 106 return 0; 107 case EINVAL: 108 return 1; 109 default: 110 abort(); 111 } 112 } 113 114 int rseq_register_current_thread(void) 115 { 116 int rc; 117 118 if (!rseq_ownership) { 119 /* Treat libc's ownership as a successful registration. */ 120 return 0; 121 } 122 rc = sys_rseq(&__rseq_abi, rseq_size, 0, RSEQ_SIG); 123 if (rc) { 124 if (RSEQ_READ_ONCE(rseq_reg_success)) { 125 /* Incoherent success/failure within process. */ 126 abort(); 127 } 128 return -1; 129 } 130 assert(rseq_current_cpu_raw() >= 0); 131 RSEQ_WRITE_ONCE(rseq_reg_success, 1); 132 return 0; 133 } 134 135 int rseq_unregister_current_thread(void) 136 { 137 int rc; 138 139 if (!rseq_ownership) { 140 /* Treat libc's ownership as a successful unregistration. */ 141 return 0; 142 } 143 rc = sys_rseq(&__rseq_abi, rseq_size, RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); 144 if (rc) 145 return -1; 146 return 0; 147 } 148 149 static 150 unsigned int get_rseq_feature_size(void) 151 { 152 unsigned long auxv_rseq_feature_size, auxv_rseq_align; 153 154 auxv_rseq_align = getauxval(AT_RSEQ_ALIGN); 155 assert(!auxv_rseq_align || auxv_rseq_align <= RSEQ_THREAD_AREA_ALLOC_SIZE); 156 157 auxv_rseq_feature_size = getauxval(AT_RSEQ_FEATURE_SIZE); 158 assert(!auxv_rseq_feature_size || auxv_rseq_feature_size <= RSEQ_THREAD_AREA_ALLOC_SIZE); 159 if (auxv_rseq_feature_size) 160 return auxv_rseq_feature_size; 161 else 162 return ORIG_RSEQ_FEATURE_SIZE; 163 } 164 165 static __attribute__((constructor)) 166 void rseq_init(void) 167 { 168 /* 169 * If the libc's registered rseq size isn't already valid, it may be 170 * because the binary is dynamically linked and not necessarily due to 171 * libc not having registered a restartable sequence. Try to find the 172 * symbols if that's the case. 173 */ 174 if (!*libc_rseq_size_p) { 175 libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset"); 176 libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size"); 177 libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags"); 178 } 179 if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p && 180 *libc_rseq_size_p != 0) { 181 /* rseq registration owned by glibc */ 182 rseq_offset = *libc_rseq_offset_p; 183 rseq_size = *libc_rseq_size_p; 184 rseq_flags = *libc_rseq_flags_p; 185 rseq_feature_size = get_rseq_feature_size(); 186 if (rseq_feature_size > rseq_size) 187 rseq_feature_size = rseq_size; 188 return; 189 } 190 rseq_ownership = 1; 191 if (!rseq_available()) { 192 rseq_size = 0; 193 rseq_feature_size = 0; 194 return; 195 } 196 rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer(); 197 rseq_flags = 0; 198 rseq_feature_size = get_rseq_feature_size(); 199 if (rseq_feature_size == ORIG_RSEQ_FEATURE_SIZE) 200 rseq_size = ORIG_RSEQ_ALLOC_SIZE; 201 else 202 rseq_size = RSEQ_THREAD_AREA_ALLOC_SIZE; 203 } 204 205 static __attribute__((destructor)) 206 void rseq_exit(void) 207 { 208 if (!rseq_ownership) 209 return; 210 rseq_offset = 0; 211 rseq_size = -1U; 212 rseq_feature_size = -1U; 213 rseq_ownership = 0; 214 } 215 216 int32_t rseq_fallback_current_cpu(void) 217 { 218 int32_t cpu; 219 220 cpu = sched_getcpu(); 221 if (cpu < 0) { 222 perror("sched_getcpu()"); 223 abort(); 224 } 225 return cpu; 226 } 227 228 int32_t rseq_fallback_current_node(void) 229 { 230 uint32_t cpu_id, node_id; 231 int ret; 232 233 ret = sys_getcpu(&cpu_id, &node_id); 234 if (ret) { 235 perror("sys_getcpu()"); 236 return ret; 237 } 238 return (int32_t) node_id; 239 } 240