1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2004 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 /* 32 * Define stubs for TLS internals so that programs and libraries can 33 * link. These functions will be replaced by functional versions at 34 * runtime from ld-elf.so.1. 35 */ 36 37 #include <sys/cdefs.h> 38 #include <sys/param.h> 39 #include <stdlib.h> 40 #include <string.h> 41 #include <elf.h> 42 #include <unistd.h> 43 44 #include "libc_private.h" 45 46 #define tls_assert(cond) ((cond) ? (void) 0 : \ 47 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 48 __XSTRING(__LINE__) "\n"), abort())) 49 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 50 51 /* Provided by jemalloc to avoid bootstrapping issues. */ 52 void *__je_bootstrap_malloc(size_t size); 53 void *__je_bootstrap_calloc(size_t num, size_t size); 54 void __je_bootstrap_free(void *ptr); 55 56 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 57 __weak_reference(__libc_free_tls, _rtld_free_tls); 58 59 #ifdef __i386__ 60 61 __weak_reference(___libc_tls_get_addr, ___tls_get_addr); 62 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 63 64 #endif 65 66 void * __libc_tls_get_addr(void *); 67 __weak_reference(__libc_tls_get_addr, __tls_get_addr); 68 69 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 70 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 71 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 72 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 73 74 #if defined(__amd64__) 75 #define TLS_TCB_ALIGN 16 76 #elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \ 77 defined(__mips__) || defined(__powerpc__) || defined(__riscv) || \ 78 defined(__sparc64__) 79 #define TLS_TCB_ALIGN sizeof(void *) 80 #else 81 #error TLS_TCB_ALIGN undefined for target architecture 82 #endif 83 84 #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \ 85 defined(__powerpc__) || defined(__riscv) 86 #define TLS_VARIANT_I 87 #endif 88 #if defined(__i386__) || defined(__amd64__) || defined(__sparc64__) 89 #define TLS_VARIANT_II 90 #endif 91 92 #if defined(__mips__) || defined(__powerpc__) || defined(__riscv) 93 #define DTV_OFFSET 0x8000 94 #else 95 #define DTV_OFFSET 0 96 #endif 97 98 #ifndef PIC 99 100 static size_t tls_static_space; 101 static size_t tls_init_size; 102 static size_t tls_init_align; 103 static void *tls_init; 104 #endif 105 106 #ifdef __i386__ 107 108 /* GNU ABI */ 109 110 __attribute__((__regparm__(1))) 111 void * 112 ___libc_tls_get_addr(void *ti __unused) 113 { 114 return (0); 115 } 116 117 #endif 118 119 void * 120 __libc_tls_get_addr(void *ti __unused) 121 { 122 return (0); 123 } 124 125 #ifndef PIC 126 127 static void * 128 malloc_aligned(size_t size, size_t align) 129 { 130 void *mem, *res; 131 132 if (align < sizeof(void *)) 133 align = sizeof(void *); 134 135 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 136 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 137 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 138 return (res); 139 } 140 141 static void 142 free_aligned(void *ptr) 143 { 144 void *mem; 145 uintptr_t x; 146 147 if (ptr == NULL) 148 return; 149 150 x = (uintptr_t)ptr; 151 x -= sizeof(void *); 152 mem = *(void **)x; 153 __je_bootstrap_free(mem); 154 } 155 156 #ifdef TLS_VARIANT_I 157 158 /* 159 * There are two versions of variant I of TLS 160 * 161 * - ARM and aarch64 uses original variant I as is described in [1] and [2], 162 * where TP points to start of TCB followed by aligned TLS segment. 163 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0] 164 * points to DTV vector and DTV values are real addresses (without bias). 165 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to 166 * TLS variables are computed by linker, so we cannot overalign TLS section. 167 * 168 * - MIPS, PowerPC and RISC-V use modified version of variant I, 169 * described in [3] where TP points (with bias) to TLS and TCB immediately 170 * precedes TLS without any alignment gap[4]. Only TLS should be aligned. 171 * The TCB[0] points to DTV vector and DTV values are biased by constant 172 * value (0x8000) from real addresses[5]. 173 * 174 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage 175 * www.akkadia.org/drepper/tls.pdf 176 * 177 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r) 178 * Architecture 179 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf 180 * 181 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification 182 * https://members.openpowerfoundation.org/document/dl/576 183 * 184 * [4] Its unclear if "without any alignment gap" is hard ABI requirement, 185 * but we must follow this rule due to suboptimal _set_tp() 186 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but 187 * TCB as argument. 188 * 189 * [5] I'm not able to validate "values are biased" assertions. 190 */ 191 192 #define TLS_TCB_SIZE (2 * sizeof(void *)) 193 194 /* 195 * Return pointer to allocated TLS block 196 */ 197 static void * 198 get_tls_block_ptr(void *tcb, size_t tcbsize) 199 { 200 size_t extra_size, post_size, pre_size, tls_block_size; 201 202 /* Compute fragments sizes. */ 203 extra_size = tcbsize - TLS_TCB_SIZE; 204 #if defined(__aarch64__) || defined(__arm__) 205 post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE; 206 #else 207 post_size = 0; 208 #endif 209 tls_block_size = tcbsize + post_size; 210 pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size; 211 212 return ((char *)tcb - pre_size - extra_size); 213 } 214 215 /* 216 * Free Static TLS using the Variant I method. The tcbsize 217 * and tcbalign parameters must be the same as those used to allocate 218 * the block. 219 */ 220 void 221 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 222 { 223 Elf_Addr *dtv; 224 Elf_Addr **tls; 225 226 tls = (Elf_Addr **)tcb; 227 dtv = tls[0]; 228 __je_bootstrap_free(dtv); 229 free_aligned(get_tls_block_ptr(tcb, tcbsize)); 230 } 231 232 /* 233 * Allocate Static TLS using the Variant I method. 234 * 235 * To handle all above requirements, we setup the following layout for 236 * TLS block: 237 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align)) 238 * 239 * +----------+--------------+--------------+-----------+------------------+ 240 * | pre gap | extended TCB | TCB | post gap | TLS segment | 241 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space | 242 * +----------+--------------+--------------+-----------+------------------+ 243 * 244 * where: 245 * extra_size is tcbsize - TLS_TCB_SIZE 246 * post_size is used to adjust TCB to TLS aligment for first version of TLS 247 * layout and is always 0 for second version. 248 * pre_size is used to adjust TCB aligment for first version and to adjust 249 * TLS alignment for second version. 250 * 251 */ 252 void * 253 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 254 { 255 Elf_Addr *dtv, **tcb; 256 char *tls_block, *tls; 257 size_t extra_size, maxalign, post_size, pre_size, tls_block_size; 258 259 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 260 return (oldtcb); 261 262 tls_assert(tcbalign >= TLS_TCB_ALIGN); 263 maxalign = MAX(tcbalign, tls_init_align); 264 265 /* Compute fragmets sizes. */ 266 extra_size = tcbsize - TLS_TCB_SIZE; 267 #if defined(__aarch64__) || defined(__arm__) 268 post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE; 269 #else 270 post_size = 0; 271 #endif 272 tls_block_size = tcbsize + post_size; 273 pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size; 274 tls_block_size += pre_size + tls_static_space; 275 276 /* Allocate whole TLS block */ 277 tls_block = malloc_aligned(tls_block_size, maxalign); 278 if (tls_block == NULL) { 279 tls_msg("__libc_allocate_tls: Out of memory.\n"); 280 abort(); 281 } 282 memset(tls_block, 0, tls_block_size); 283 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size); 284 tls = (char *)tcb + TLS_TCB_SIZE + post_size; 285 286 if (oldtcb != NULL) { 287 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize), 288 tls_block_size); 289 free_aligned(oldtcb); 290 291 /* Adjust the DTV. */ 292 dtv = tcb[0]; 293 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET); 294 } else { 295 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 296 if (dtv == NULL) { 297 tls_msg("__libc_allocate_tls: Out of memory.\n"); 298 abort(); 299 } 300 /* Build the DTV. */ 301 tcb[0] = dtv; 302 dtv[0] = 1; /* Generation. */ 303 dtv[1] = 1; /* Segments count. */ 304 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET); 305 306 if (tls_init_size > 0) 307 memcpy(tls, tls_init, tls_init_size); 308 } 309 310 return (tcb); 311 } 312 313 #endif 314 315 #ifdef TLS_VARIANT_II 316 317 #define TLS_TCB_SIZE (3 * sizeof(Elf_Addr)) 318 319 /* 320 * Free Static TLS using the Variant II method. 321 */ 322 void 323 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 324 { 325 size_t size; 326 Elf_Addr* dtv; 327 Elf_Addr tlsstart, tlsend; 328 329 /* 330 * Figure out the size of the initial TLS block so that we can 331 * find stuff which ___tls_get_addr() allocated dynamically. 332 */ 333 tcbalign = MAX(tcbalign, tls_init_align); 334 size = roundup2(tls_static_space, tcbalign); 335 336 dtv = ((Elf_Addr**)tcb)[1]; 337 tlsend = (Elf_Addr) tcb; 338 tlsstart = tlsend - size; 339 free_aligned((void*)tlsstart); 340 __je_bootstrap_free(dtv); 341 } 342 343 /* 344 * Allocate Static TLS using the Variant II method. 345 */ 346 void * 347 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 348 { 349 size_t size; 350 char *tls; 351 Elf_Addr *dtv; 352 Elf_Addr segbase, oldsegbase; 353 354 tcbalign = MAX(tcbalign, tls_init_align); 355 size = roundup2(tls_static_space, tcbalign); 356 357 if (tcbsize < 2 * sizeof(Elf_Addr)) 358 tcbsize = 2 * sizeof(Elf_Addr); 359 tls = malloc_aligned(size + tcbsize, tcbalign); 360 if (tls == NULL) { 361 tls_msg("__libc_allocate_tls: Out of memory.\n"); 362 abort(); 363 } 364 memset(tls, 0, size + tcbsize); 365 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 366 if (dtv == NULL) { 367 tls_msg("__libc_allocate_tls: Out of memory.\n"); 368 abort(); 369 } 370 371 segbase = (Elf_Addr)(tls + size); 372 ((Elf_Addr*)segbase)[0] = segbase; 373 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 374 375 dtv[0] = 1; 376 dtv[1] = 1; 377 dtv[2] = segbase - tls_static_space; 378 379 if (oldtls) { 380 /* 381 * Copy the static TLS block over whole. 382 */ 383 oldsegbase = (Elf_Addr) oldtls; 384 memcpy((void *)(segbase - tls_static_space), 385 (const void *)(oldsegbase - tls_static_space), 386 tls_static_space); 387 388 /* 389 * We assume that this block was the one we created with 390 * allocate_initial_tls(). 391 */ 392 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 393 } else { 394 memcpy((void *)(segbase - tls_static_space), 395 tls_init, tls_init_size); 396 memset((void *)(segbase - tls_static_space + tls_init_size), 397 0, tls_static_space - tls_init_size); 398 } 399 400 return (void*) segbase; 401 } 402 403 #endif /* TLS_VARIANT_II */ 404 405 #else 406 407 void * 408 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 409 size_t tcbalign __unused) 410 { 411 return (0); 412 } 413 414 void 415 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 416 size_t tcbalign __unused) 417 { 418 } 419 420 #endif /* PIC */ 421 422 extern char **environ; 423 424 void 425 _init_tls(void) 426 { 427 #ifndef PIC 428 Elf_Addr *sp; 429 Elf_Auxinfo *aux, *auxp; 430 Elf_Phdr *phdr; 431 size_t phent, phnum; 432 int i; 433 void *tls; 434 435 sp = (Elf_Addr *) environ; 436 while (*sp++ != 0) 437 ; 438 aux = (Elf_Auxinfo *) sp; 439 phdr = NULL; 440 phent = phnum = 0; 441 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 442 switch (auxp->a_type) { 443 case AT_PHDR: 444 phdr = auxp->a_un.a_ptr; 445 break; 446 447 case AT_PHENT: 448 phent = auxp->a_un.a_val; 449 break; 450 451 case AT_PHNUM: 452 phnum = auxp->a_un.a_val; 453 break; 454 } 455 } 456 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 457 return; 458 459 for (i = 0; (unsigned) i < phnum; i++) { 460 if (phdr[i].p_type == PT_TLS) { 461 tls_static_space = roundup2(phdr[i].p_memsz, 462 phdr[i].p_align); 463 tls_init_size = phdr[i].p_filesz; 464 tls_init_align = phdr[i].p_align; 465 tls_init = (void*) phdr[i].p_vaddr; 466 break; 467 } 468 } 469 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 470 471 _set_tp(tls); 472 #endif 473 } 474