1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2004 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 /* 32 * Define stubs for TLS internals so that programs and libraries can 33 * link. These functions will be replaced by functional versions at 34 * runtime from ld-elf.so.1. 35 */ 36 37 #include <sys/cdefs.h> 38 #include <sys/param.h> 39 #include <stdlib.h> 40 #include <string.h> 41 #include <elf.h> 42 #include <unistd.h> 43 44 #include "rtld.h" 45 #include "libc_private.h" 46 47 #define tls_assert(cond) ((cond) ? (void) 0 : \ 48 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 49 __XSTRING(__LINE__) "\n"), abort())) 50 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 51 52 /* Provided by jemalloc to avoid bootstrapping issues. */ 53 void *__je_bootstrap_malloc(size_t size); 54 void *__je_bootstrap_calloc(size_t num, size_t size); 55 void __je_bootstrap_free(void *ptr); 56 57 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 58 __weak_reference(__libc_free_tls, _rtld_free_tls); 59 60 #ifdef __i386__ 61 62 __weak_reference(___libc_tls_get_addr, ___tls_get_addr); 63 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 64 65 #endif 66 67 void * __libc_tls_get_addr(void *); 68 __weak_reference(__libc_tls_get_addr, __tls_get_addr); 69 70 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 71 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 72 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 73 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 74 75 #if defined(__amd64__) || defined(__aarch64__) || defined(__riscv) 76 #define TLS_TCB_ALIGN 16 77 #elif defined(__arm__) || defined(__mips__) 78 #define TLS_TCB_ALIGN 8 79 #elif defined(__powerpc__) 80 #define TLS_TCB_ALIGN TLS_TCB_SIZE 81 #elif defined(__i386__) 82 #define TLS_TCB_ALIGN 4 83 #else 84 #error TLS_TCB_ALIGN undefined for target architecture 85 #endif 86 87 #ifndef PIC 88 89 static size_t libc_tls_static_space; 90 static size_t libc_tls_init_size; 91 static size_t libc_tls_init_align; 92 static void *libc_tls_init; 93 #endif 94 95 void * 96 __libc_tls_get_addr(void *vti) 97 { 98 Elf_Addr **dtvp, *dtv; 99 tls_index *ti; 100 101 dtvp = _get_tp(); 102 dtv = *dtvp; 103 ti = vti; 104 return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) + 105 TLS_DTV_OFFSET); 106 } 107 108 #ifdef __i386__ 109 110 /* GNU ABI */ 111 112 __attribute__((__regparm__(1))) 113 void * 114 ___libc_tls_get_addr(void *vti) 115 { 116 return (__libc_tls_get_addr(vti)); 117 } 118 119 #endif 120 121 #ifndef PIC 122 123 static void * 124 libc_malloc_aligned(size_t size, size_t align) 125 { 126 void *mem, *res; 127 128 if (align < sizeof(void *)) 129 align = sizeof(void *); 130 131 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 132 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 133 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 134 return (res); 135 } 136 137 static void 138 libc_free_aligned(void *ptr) 139 { 140 void *mem; 141 uintptr_t x; 142 143 if (ptr == NULL) 144 return; 145 146 x = (uintptr_t)ptr; 147 x -= sizeof(void *); 148 mem = *(void **)x; 149 __je_bootstrap_free(mem); 150 } 151 152 #ifdef TLS_VARIANT_I 153 154 /* 155 * There are two versions of variant I of TLS 156 * 157 * - ARM and aarch64 uses original variant I as is described in [1] and [2], 158 * where TP points to start of TCB followed by aligned TLS segment. 159 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0] 160 * points to DTV vector and DTV values are real addresses (without bias). 161 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to 162 * TLS variables are computed by linker, so we cannot overalign TLS section. 163 * 164 * - MIPS, PowerPC and RISC-V use modified version of variant I, 165 * described in [3] where TP points (with bias) to TLS and TCB immediately 166 * precedes TLS without any alignment gap[4]. Only TLS should be aligned. 167 * The TCB[0] points to DTV vector and DTV values are biased by constant 168 * value (0x8000) from real addresses[5]. 169 * 170 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage 171 * www.akkadia.org/drepper/tls.pdf 172 * 173 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r) 174 * Architecture 175 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf 176 * 177 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification 178 * https://members.openpowerfoundation.org/document/dl/576 179 * 180 * [4] Its unclear if "without any alignment gap" is hard ABI requirement, 181 * but we must follow this rule due to suboptimal _set_tp() 182 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but 183 * TCB as argument. 184 * 185 * [5] I'm not able to validate "values are biased" assertions. 186 */ 187 188 /* 189 * Return pointer to allocated TLS block 190 */ 191 static void * 192 get_tls_block_ptr(void *tcb, size_t tcbsize) 193 { 194 size_t extra_size, post_size, pre_size, tls_block_size; 195 196 /* Compute fragments sizes. */ 197 extra_size = tcbsize - TLS_TCB_SIZE; 198 #if defined(__aarch64__) || defined(__arm__) 199 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 200 #else 201 post_size = 0; 202 #endif 203 tls_block_size = tcbsize + post_size; 204 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 205 tls_block_size; 206 207 return ((char *)tcb - pre_size - extra_size); 208 } 209 210 /* 211 * Free Static TLS using the Variant I method. The tcbsize 212 * and tcbalign parameters must be the same as those used to allocate 213 * the block. 214 */ 215 void 216 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 217 { 218 Elf_Addr *dtv; 219 Elf_Addr **tls; 220 221 tls = (Elf_Addr **)tcb; 222 dtv = tls[0]; 223 __je_bootstrap_free(dtv); 224 libc_free_aligned(get_tls_block_ptr(tcb, tcbsize)); 225 } 226 227 /* 228 * Allocate Static TLS using the Variant I method. 229 * 230 * To handle all above requirements, we setup the following layout for 231 * TLS block: 232 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align)) 233 * 234 * +----------+--------------+--------------+-----------+------------------+ 235 * | pre gap | extended TCB | TCB | post gap | TLS segment | 236 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space | 237 * +----------+--------------+--------------+-----------+------------------+ 238 * 239 * where: 240 * extra_size is tcbsize - TLS_TCB_SIZE 241 * post_size is used to adjust TCB to TLS alignment for first version of TLS 242 * layout and is always 0 for second version. 243 * pre_size is used to adjust TCB alignment for first version and to adjust 244 * TLS alignment for second version. 245 * 246 */ 247 void * 248 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 249 { 250 Elf_Addr *dtv, **tcb; 251 char *tls_block, *tls; 252 size_t extra_size, maxalign, post_size, pre_size, tls_block_size; 253 254 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 255 return (oldtcb); 256 257 tls_assert(tcbalign >= TLS_TCB_ALIGN); 258 maxalign = MAX(tcbalign, libc_tls_init_align); 259 260 /* Compute fragmets sizes. */ 261 extra_size = tcbsize - TLS_TCB_SIZE; 262 #if defined(__aarch64__) || defined(__arm__) 263 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 264 #else 265 post_size = 0; 266 #endif 267 tls_block_size = tcbsize + post_size; 268 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 269 tls_block_size; 270 tls_block_size += pre_size + libc_tls_static_space; 271 272 /* Allocate whole TLS block */ 273 tls_block = libc_malloc_aligned(tls_block_size, maxalign); 274 if (tls_block == NULL) { 275 tls_msg("__libc_allocate_tls: Out of memory.\n"); 276 abort(); 277 } 278 memset(tls_block, 0, tls_block_size); 279 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size); 280 tls = (char *)tcb + TLS_TCB_SIZE + post_size; 281 282 if (oldtcb != NULL) { 283 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize), 284 tls_block_size); 285 libc_free_aligned(oldtcb); 286 287 /* Adjust the DTV. */ 288 dtv = tcb[0]; 289 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 290 } else { 291 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 292 if (dtv == NULL) { 293 tls_msg("__libc_allocate_tls: Out of memory.\n"); 294 abort(); 295 } 296 /* Build the DTV. */ 297 tcb[0] = dtv; 298 dtv[0] = 1; /* Generation. */ 299 dtv[1] = 1; /* Segments count. */ 300 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 301 302 if (libc_tls_init_size > 0) 303 memcpy(tls, libc_tls_init, libc_tls_init_size); 304 } 305 306 return (tcb); 307 } 308 309 #endif 310 311 #ifdef TLS_VARIANT_II 312 313 #define TLS_TCB_SIZE (3 * sizeof(Elf_Addr)) 314 315 /* 316 * Free Static TLS using the Variant II method. 317 */ 318 void 319 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 320 { 321 size_t size; 322 Elf_Addr* dtv; 323 Elf_Addr tlsstart, tlsend; 324 325 /* 326 * Figure out the size of the initial TLS block so that we can 327 * find stuff which ___tls_get_addr() allocated dynamically. 328 */ 329 tcbalign = MAX(tcbalign, libc_tls_init_align); 330 size = roundup2(libc_tls_static_space, tcbalign); 331 332 dtv = ((Elf_Addr**)tcb)[1]; 333 tlsend = (Elf_Addr) tcb; 334 tlsstart = tlsend - size; 335 libc_free_aligned((void*)tlsstart); 336 __je_bootstrap_free(dtv); 337 } 338 339 /* 340 * Allocate Static TLS using the Variant II method. 341 */ 342 void * 343 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 344 { 345 size_t size; 346 char *tls; 347 Elf_Addr *dtv; 348 Elf_Addr segbase, oldsegbase; 349 350 tcbalign = MAX(tcbalign, libc_tls_init_align); 351 size = roundup2(libc_tls_static_space, tcbalign); 352 353 if (tcbsize < 2 * sizeof(Elf_Addr)) 354 tcbsize = 2 * sizeof(Elf_Addr); 355 tls = libc_malloc_aligned(size + tcbsize, tcbalign); 356 if (tls == NULL) { 357 tls_msg("__libc_allocate_tls: Out of memory.\n"); 358 abort(); 359 } 360 memset(tls, 0, size + tcbsize); 361 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 362 if (dtv == NULL) { 363 tls_msg("__libc_allocate_tls: Out of memory.\n"); 364 abort(); 365 } 366 367 segbase = (Elf_Addr)(tls + size); 368 ((Elf_Addr*)segbase)[0] = segbase; 369 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 370 371 dtv[0] = 1; 372 dtv[1] = 1; 373 dtv[2] = segbase - libc_tls_static_space; 374 375 if (oldtls) { 376 /* 377 * Copy the static TLS block over whole. 378 */ 379 oldsegbase = (Elf_Addr) oldtls; 380 memcpy((void *)(segbase - libc_tls_static_space), 381 (const void *)(oldsegbase - libc_tls_static_space), 382 libc_tls_static_space); 383 384 /* 385 * We assume that this block was the one we created with 386 * allocate_initial_tls(). 387 */ 388 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 389 } else { 390 memcpy((void *)(segbase - libc_tls_static_space), 391 libc_tls_init, libc_tls_init_size); 392 memset((void *)(segbase - libc_tls_static_space + 393 libc_tls_init_size), 0, 394 libc_tls_static_space - libc_tls_init_size); 395 } 396 397 return (void*) segbase; 398 } 399 400 #endif /* TLS_VARIANT_II */ 401 402 #else 403 404 void * 405 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 406 size_t tcbalign __unused) 407 { 408 return (0); 409 } 410 411 void 412 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 413 size_t tcbalign __unused) 414 { 415 } 416 417 #endif /* PIC */ 418 419 extern char **environ; 420 421 void 422 _init_tls(void) 423 { 424 #ifndef PIC 425 Elf_Addr *sp; 426 Elf_Auxinfo *aux, *auxp; 427 Elf_Phdr *phdr; 428 size_t phent, phnum; 429 int i; 430 void *tls; 431 432 sp = (Elf_Addr *) environ; 433 while (*sp++ != 0) 434 ; 435 aux = (Elf_Auxinfo *) sp; 436 phdr = NULL; 437 phent = phnum = 0; 438 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 439 switch (auxp->a_type) { 440 case AT_PHDR: 441 phdr = auxp->a_un.a_ptr; 442 break; 443 444 case AT_PHENT: 445 phent = auxp->a_un.a_val; 446 break; 447 448 case AT_PHNUM: 449 phnum = auxp->a_un.a_val; 450 break; 451 } 452 } 453 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 454 return; 455 456 for (i = 0; (unsigned) i < phnum; i++) { 457 if (phdr[i].p_type == PT_TLS) { 458 libc_tls_static_space = roundup2(phdr[i].p_memsz, 459 phdr[i].p_align); 460 libc_tls_init_size = phdr[i].p_filesz; 461 libc_tls_init_align = phdr[i].p_align; 462 libc_tls_init = (void *)phdr[i].p_vaddr; 463 break; 464 } 465 } 466 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 467 468 _set_tp(tls); 469 #endif 470 } 471