1 /*- 2 * Copyright (c) 2004 Doug Rabson 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 /* 30 * Define stubs for TLS internals so that programs and libraries can 31 * link. These functions will be replaced by functional versions at 32 * runtime from ld-elf.so.1. 33 */ 34 35 #include <sys/cdefs.h> 36 #include <sys/param.h> 37 #include <stdlib.h> 38 #include <string.h> 39 #include <elf.h> 40 #include <unistd.h> 41 42 #include "libc_private.h" 43 44 #define tls_assert(cond) ((cond) ? (void) 0 : \ 45 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 46 __XSTRING(__LINE__) "\n"), abort())) 47 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 48 49 /* Provided by jemalloc to avoid bootstrapping issues. */ 50 void *__je_bootstrap_malloc(size_t size); 51 void *__je_bootstrap_calloc(size_t num, size_t size); 52 void __je_bootstrap_free(void *ptr); 53 54 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 55 __weak_reference(__libc_free_tls, _rtld_free_tls); 56 57 #ifdef __i386__ 58 59 __weak_reference(___libc_tls_get_addr, ___tls_get_addr); 60 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 61 62 #endif 63 64 void * __libc_tls_get_addr(void *); 65 __weak_reference(__libc_tls_get_addr, __tls_get_addr); 66 67 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 68 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 69 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 70 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 71 72 #if defined(__amd64__) 73 #define TLS_TCB_ALIGN 16 74 #elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \ 75 defined(__mips__) || defined(__powerpc__) || defined(__riscv) || \ 76 defined(__sparc64__) 77 #define TLS_TCB_ALIGN sizeof(void *) 78 #else 79 #error TLS_TCB_ALIGN undefined for target architecture 80 #endif 81 82 #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \ 83 defined(__powerpc__) || defined(__riscv) 84 #define TLS_VARIANT_I 85 #endif 86 #if defined(__i386__) || defined(__amd64__) || defined(__sparc64__) 87 #define TLS_VARIANT_II 88 #endif 89 90 #ifndef PIC 91 92 static size_t tls_static_space; 93 static size_t tls_init_size; 94 static size_t tls_init_align; 95 static void *tls_init; 96 #endif 97 98 #ifdef __i386__ 99 100 /* GNU ABI */ 101 102 __attribute__((__regparm__(1))) 103 void * 104 ___libc_tls_get_addr(void *ti __unused) 105 { 106 return (0); 107 } 108 109 #endif 110 111 void * 112 __libc_tls_get_addr(void *ti __unused) 113 { 114 return (0); 115 } 116 117 #ifndef PIC 118 119 static void * 120 malloc_aligned(size_t size, size_t align) 121 { 122 void *mem, *res; 123 124 if (align < sizeof(void *)) 125 align = sizeof(void *); 126 127 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 128 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 129 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 130 return (res); 131 } 132 133 static void 134 free_aligned(void *ptr) 135 { 136 void *mem; 137 uintptr_t x; 138 139 if (ptr == NULL) 140 return; 141 142 x = (uintptr_t)ptr; 143 x -= sizeof(void *); 144 mem = *(void **)x; 145 __je_bootstrap_free(mem); 146 } 147 148 #ifdef TLS_VARIANT_I 149 150 #define TLS_TCB_SIZE (2 * sizeof(void *)) 151 152 /* 153 * Free Static TLS using the Variant I method. 154 */ 155 void 156 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign __unused) 157 { 158 Elf_Addr *dtv; 159 Elf_Addr **tls; 160 161 tls = (Elf_Addr **)tcb; 162 dtv = tls[0]; 163 __je_bootstrap_free(dtv); 164 free_aligned(tls); 165 } 166 167 /* 168 * Allocate Static TLS using the Variant I method. 169 */ 170 void * 171 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 172 { 173 Elf_Addr *dtv; 174 Elf_Addr **tls; 175 176 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 177 return (oldtcb); 178 179 tls_assert(tcbalign >= TLS_TCB_ALIGN); 180 tls_assert(tcbsize == TLS_TCB_SIZE); 181 182 tcbsize = roundup2(tcbsize, tcbalign); 183 tls = malloc_aligned(tcbsize + tls_static_space, tcbalign); 184 if (tls == NULL) { 185 tls_msg("__libc_allocate_tls: Out of memory.\n"); 186 abort(); 187 } 188 memset(tls, 0, tcbsize + tls_static_space); 189 190 if (oldtcb != NULL) { 191 memcpy(tls, oldtcb, tcbsize + tls_static_space); 192 __je_bootstrap_free(oldtcb); 193 194 /* Adjust the DTV. */ 195 dtv = tls[0]; 196 dtv[2] = (Elf_Addr)tls + tcbsize; 197 } else { 198 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 199 if (dtv == NULL) { 200 tls_msg("__libc_allocate_tls: Out of memory.\n"); 201 abort(); 202 } 203 tls[0] = dtv; 204 dtv[0] = 1; /* Generation. */ 205 dtv[1] = 1; /* Segments count. */ 206 dtv[2] = (Elf_Addr)tls + tcbsize; 207 208 if (tls_init_size > 0) 209 memcpy((void*)dtv[2], tls_init, tls_init_size); 210 } 211 212 return (tls); 213 } 214 215 #endif 216 217 #ifdef TLS_VARIANT_II 218 219 #define TLS_TCB_SIZE (3 * sizeof(Elf_Addr)) 220 221 /* 222 * Free Static TLS using the Variant II method. 223 */ 224 void 225 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 226 { 227 size_t size; 228 Elf_Addr* dtv; 229 Elf_Addr tlsstart, tlsend; 230 231 /* 232 * Figure out the size of the initial TLS block so that we can 233 * find stuff which ___tls_get_addr() allocated dynamically. 234 */ 235 size = roundup2(tls_static_space, tcbalign); 236 237 dtv = ((Elf_Addr**)tcb)[1]; 238 tlsend = (Elf_Addr) tcb; 239 tlsstart = tlsend - size; 240 free_aligned((void*)tlsstart); 241 __je_bootstrap_free(dtv); 242 } 243 244 /* 245 * Allocate Static TLS using the Variant II method. 246 */ 247 void * 248 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 249 { 250 size_t size; 251 char *tls; 252 Elf_Addr *dtv; 253 Elf_Addr segbase, oldsegbase; 254 255 size = roundup2(tls_static_space, tcbalign); 256 257 if (tcbsize < 2 * sizeof(Elf_Addr)) 258 tcbsize = 2 * sizeof(Elf_Addr); 259 tls = malloc_aligned(size + tcbsize, tcbalign); 260 if (tls == NULL) { 261 tls_msg("__libc_allocate_tls: Out of memory.\n"); 262 abort(); 263 } 264 memset(tls, 0, size + tcbsize); 265 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 266 if (dtv == NULL) { 267 tls_msg("__libc_allocate_tls: Out of memory.\n"); 268 abort(); 269 } 270 271 segbase = (Elf_Addr)(tls + size); 272 ((Elf_Addr*)segbase)[0] = segbase; 273 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 274 275 dtv[0] = 1; 276 dtv[1] = 1; 277 dtv[2] = segbase - tls_static_space; 278 279 if (oldtls) { 280 /* 281 * Copy the static TLS block over whole. 282 */ 283 oldsegbase = (Elf_Addr) oldtls; 284 memcpy((void *)(segbase - tls_static_space), 285 (const void *)(oldsegbase - tls_static_space), 286 tls_static_space); 287 288 /* 289 * We assume that this block was the one we created with 290 * allocate_initial_tls(). 291 */ 292 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 293 } else { 294 memcpy((void *)(segbase - tls_static_space), 295 tls_init, tls_init_size); 296 memset((void *)(segbase - tls_static_space + tls_init_size), 297 0, tls_static_space - tls_init_size); 298 } 299 300 return (void*) segbase; 301 } 302 303 #endif /* TLS_VARIANT_II */ 304 305 #else 306 307 void * 308 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 309 size_t tcbalign __unused) 310 { 311 return (0); 312 } 313 314 void 315 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 316 size_t tcbalign __unused) 317 { 318 } 319 320 #endif /* PIC */ 321 322 extern char **environ; 323 324 void 325 _init_tls(void) 326 { 327 #ifndef PIC 328 Elf_Addr *sp; 329 Elf_Auxinfo *aux, *auxp; 330 Elf_Phdr *phdr; 331 size_t phent, phnum; 332 int i; 333 void *tls; 334 335 sp = (Elf_Addr *) environ; 336 while (*sp++ != 0) 337 ; 338 aux = (Elf_Auxinfo *) sp; 339 phdr = NULL; 340 phent = phnum = 0; 341 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 342 switch (auxp->a_type) { 343 case AT_PHDR: 344 phdr = auxp->a_un.a_ptr; 345 break; 346 347 case AT_PHENT: 348 phent = auxp->a_un.a_val; 349 break; 350 351 case AT_PHNUM: 352 phnum = auxp->a_un.a_val; 353 break; 354 } 355 } 356 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 357 return; 358 359 for (i = 0; (unsigned) i < phnum; i++) { 360 if (phdr[i].p_type == PT_TLS) { 361 tls_static_space = roundup2(phdr[i].p_memsz, 362 phdr[i].p_align); 363 tls_init_size = phdr[i].p_filesz; 364 tls_init_align = phdr[i].p_align; 365 tls_init = (void*) phdr[i].p_vaddr; 366 break; 367 } 368 } 369 370 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, 371 MAX(TLS_TCB_ALIGN, tls_init_align)); 372 373 _set_tp(tls); 374 #endif 375 } 376