1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2004 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 /* 32 * Define stubs for TLS internals so that programs and libraries can 33 * link. These functions will be replaced by functional versions at 34 * runtime from ld-elf.so.1. 35 */ 36 37 #include <sys/cdefs.h> 38 #include <sys/param.h> 39 #include <stdlib.h> 40 #include <string.h> 41 #include <elf.h> 42 #include <unistd.h> 43 44 #include "libc_private.h" 45 46 #define tls_assert(cond) ((cond) ? (void) 0 : \ 47 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 48 __XSTRING(__LINE__) "\n"), abort())) 49 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 50 51 /* Provided by jemalloc to avoid bootstrapping issues. */ 52 void *__je_bootstrap_malloc(size_t size); 53 void *__je_bootstrap_calloc(size_t num, size_t size); 54 void __je_bootstrap_free(void *ptr); 55 56 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 57 __weak_reference(__libc_free_tls, _rtld_free_tls); 58 59 #ifdef __i386__ 60 61 __weak_reference(___libc_tls_get_addr, ___tls_get_addr); 62 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 63 64 #endif 65 66 void * __libc_tls_get_addr(void *); 67 __weak_reference(__libc_tls_get_addr, __tls_get_addr); 68 69 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 70 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 71 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 72 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 73 74 #if defined(__amd64__) 75 #define TLS_TCB_ALIGN 16 76 #elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \ 77 defined(__mips__) || defined(__powerpc__) || defined(__riscv) || \ 78 defined(__sparc64__) 79 #define TLS_TCB_ALIGN sizeof(void *) 80 #else 81 #error TLS_TCB_ALIGN undefined for target architecture 82 #endif 83 84 #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \ 85 defined(__powerpc__) || defined(__riscv) 86 #define TLS_VARIANT_I 87 #endif 88 #if defined(__i386__) || defined(__amd64__) || defined(__sparc64__) 89 #define TLS_VARIANT_II 90 #endif 91 92 #ifndef PIC 93 94 static size_t tls_static_space; 95 static size_t tls_init_size; 96 static size_t tls_init_align; 97 static void *tls_init; 98 #endif 99 100 #ifdef __i386__ 101 102 /* GNU ABI */ 103 104 __attribute__((__regparm__(1))) 105 void * 106 ___libc_tls_get_addr(void *ti __unused) 107 { 108 return (0); 109 } 110 111 #endif 112 113 void * 114 __libc_tls_get_addr(void *ti __unused) 115 { 116 return (0); 117 } 118 119 #ifndef PIC 120 121 static void * 122 malloc_aligned(size_t size, size_t align) 123 { 124 void *mem, *res; 125 126 if (align < sizeof(void *)) 127 align = sizeof(void *); 128 129 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 130 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 131 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 132 return (res); 133 } 134 135 static void 136 free_aligned(void *ptr) 137 { 138 void *mem; 139 uintptr_t x; 140 141 if (ptr == NULL) 142 return; 143 144 x = (uintptr_t)ptr; 145 x -= sizeof(void *); 146 mem = *(void **)x; 147 __je_bootstrap_free(mem); 148 } 149 150 #ifdef TLS_VARIANT_I 151 152 #define TLS_TCB_SIZE (2 * sizeof(void *)) 153 154 /* 155 * Free Static TLS using the Variant I method. 156 */ 157 void 158 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign __unused) 159 { 160 Elf_Addr *dtv; 161 Elf_Addr **tls; 162 163 tls = (Elf_Addr **)tcb; 164 dtv = tls[0]; 165 __je_bootstrap_free(dtv); 166 free_aligned(tls); 167 } 168 169 /* 170 * Allocate Static TLS using the Variant I method. 171 */ 172 void * 173 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 174 { 175 Elf_Addr *dtv; 176 Elf_Addr **tls; 177 178 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 179 return (oldtcb); 180 181 tls_assert(tcbalign >= TLS_TCB_ALIGN); 182 tls_assert(tcbsize == TLS_TCB_SIZE); 183 184 tcbsize = roundup2(tcbsize, tcbalign); 185 tls = malloc_aligned(tcbsize + tls_static_space, tcbalign); 186 if (tls == NULL) { 187 tls_msg("__libc_allocate_tls: Out of memory.\n"); 188 abort(); 189 } 190 memset(tls, 0, tcbsize + tls_static_space); 191 192 if (oldtcb != NULL) { 193 memcpy(tls, oldtcb, tcbsize + tls_static_space); 194 __je_bootstrap_free(oldtcb); 195 196 /* Adjust the DTV. */ 197 dtv = tls[0]; 198 dtv[2] = (Elf_Addr)tls + tcbsize; 199 } else { 200 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 201 if (dtv == NULL) { 202 tls_msg("__libc_allocate_tls: Out of memory.\n"); 203 abort(); 204 } 205 tls[0] = dtv; 206 dtv[0] = 1; /* Generation. */ 207 dtv[1] = 1; /* Segments count. */ 208 dtv[2] = (Elf_Addr)tls + tcbsize; 209 210 if (tls_init_size > 0) 211 memcpy((void*)dtv[2], tls_init, tls_init_size); 212 } 213 214 return (tls); 215 } 216 217 #endif 218 219 #ifdef TLS_VARIANT_II 220 221 #define TLS_TCB_SIZE (3 * sizeof(Elf_Addr)) 222 223 /* 224 * Free Static TLS using the Variant II method. 225 */ 226 void 227 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 228 { 229 size_t size; 230 Elf_Addr* dtv; 231 Elf_Addr tlsstart, tlsend; 232 233 /* 234 * Figure out the size of the initial TLS block so that we can 235 * find stuff which ___tls_get_addr() allocated dynamically. 236 */ 237 size = roundup2(tls_static_space, tcbalign); 238 239 dtv = ((Elf_Addr**)tcb)[1]; 240 tlsend = (Elf_Addr) tcb; 241 tlsstart = tlsend - size; 242 free_aligned((void*)tlsstart); 243 __je_bootstrap_free(dtv); 244 } 245 246 /* 247 * Allocate Static TLS using the Variant II method. 248 */ 249 void * 250 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 251 { 252 size_t size; 253 char *tls; 254 Elf_Addr *dtv; 255 Elf_Addr segbase, oldsegbase; 256 257 size = roundup2(tls_static_space, tcbalign); 258 259 if (tcbsize < 2 * sizeof(Elf_Addr)) 260 tcbsize = 2 * sizeof(Elf_Addr); 261 tls = malloc_aligned(size + tcbsize, tcbalign); 262 if (tls == NULL) { 263 tls_msg("__libc_allocate_tls: Out of memory.\n"); 264 abort(); 265 } 266 memset(tls, 0, size + tcbsize); 267 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 268 if (dtv == NULL) { 269 tls_msg("__libc_allocate_tls: Out of memory.\n"); 270 abort(); 271 } 272 273 segbase = (Elf_Addr)(tls + size); 274 ((Elf_Addr*)segbase)[0] = segbase; 275 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 276 277 dtv[0] = 1; 278 dtv[1] = 1; 279 dtv[2] = segbase - tls_static_space; 280 281 if (oldtls) { 282 /* 283 * Copy the static TLS block over whole. 284 */ 285 oldsegbase = (Elf_Addr) oldtls; 286 memcpy((void *)(segbase - tls_static_space), 287 (const void *)(oldsegbase - tls_static_space), 288 tls_static_space); 289 290 /* 291 * We assume that this block was the one we created with 292 * allocate_initial_tls(). 293 */ 294 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 295 } else { 296 memcpy((void *)(segbase - tls_static_space), 297 tls_init, tls_init_size); 298 memset((void *)(segbase - tls_static_space + tls_init_size), 299 0, tls_static_space - tls_init_size); 300 } 301 302 return (void*) segbase; 303 } 304 305 #endif /* TLS_VARIANT_II */ 306 307 #else 308 309 void * 310 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 311 size_t tcbalign __unused) 312 { 313 return (0); 314 } 315 316 void 317 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 318 size_t tcbalign __unused) 319 { 320 } 321 322 #endif /* PIC */ 323 324 extern char **environ; 325 326 void 327 _init_tls(void) 328 { 329 #ifndef PIC 330 Elf_Addr *sp; 331 Elf_Auxinfo *aux, *auxp; 332 Elf_Phdr *phdr; 333 size_t phent, phnum; 334 int i; 335 void *tls; 336 337 sp = (Elf_Addr *) environ; 338 while (*sp++ != 0) 339 ; 340 aux = (Elf_Auxinfo *) sp; 341 phdr = NULL; 342 phent = phnum = 0; 343 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 344 switch (auxp->a_type) { 345 case AT_PHDR: 346 phdr = auxp->a_un.a_ptr; 347 break; 348 349 case AT_PHENT: 350 phent = auxp->a_un.a_val; 351 break; 352 353 case AT_PHNUM: 354 phnum = auxp->a_un.a_val; 355 break; 356 } 357 } 358 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 359 return; 360 361 for (i = 0; (unsigned) i < phnum; i++) { 362 if (phdr[i].p_type == PT_TLS) { 363 tls_static_space = roundup2(phdr[i].p_memsz, 364 phdr[i].p_align); 365 tls_init_size = phdr[i].p_filesz; 366 tls_init_align = phdr[i].p_align; 367 tls_init = (void*) phdr[i].p_vaddr; 368 break; 369 } 370 } 371 372 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, 373 MAX(TLS_TCB_ALIGN, tls_init_align)); 374 375 _set_tp(tls); 376 #endif 377 } 378