1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2004 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 /* 32 * Define stubs for TLS internals so that programs and libraries can 33 * link. These functions will be replaced by functional versions at 34 * runtime from ld-elf.so.1. 35 */ 36 37 #include <sys/cdefs.h> 38 #include <sys/param.h> 39 #include <stdlib.h> 40 #include <string.h> 41 #include <elf.h> 42 #include <unistd.h> 43 44 #include "rtld.h" 45 #include "libc_private.h" 46 47 #define tls_assert(cond) ((cond) ? (void) 0 : \ 48 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 49 __XSTRING(__LINE__) "\n"), abort())) 50 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 51 52 /* Provided by jemalloc to avoid bootstrapping issues. */ 53 void *__je_bootstrap_malloc(size_t size); 54 void *__je_bootstrap_calloc(size_t num, size_t size); 55 void __je_bootstrap_free(void *ptr); 56 57 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 58 __weak_reference(__libc_free_tls, _rtld_free_tls); 59 60 #ifdef __i386__ 61 62 __weak_reference(___libc_tls_get_addr, ___tls_get_addr); 63 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 64 65 #endif 66 67 void * __libc_tls_get_addr(void *); 68 __weak_reference(__libc_tls_get_addr, __tls_get_addr); 69 70 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 71 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 72 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 73 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 74 75 #ifndef PIC 76 77 static size_t libc_tls_static_space; 78 static size_t libc_tls_init_size; 79 static size_t libc_tls_init_align; 80 static void *libc_tls_init; 81 #endif 82 83 void * 84 __libc_tls_get_addr(void *vti) 85 { 86 uintptr_t *dtv; 87 tls_index *ti; 88 89 dtv = _tcb_get()->tcb_dtv; 90 ti = vti; 91 return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) + 92 TLS_DTV_OFFSET); 93 } 94 95 #ifdef __i386__ 96 97 /* GNU ABI */ 98 99 __attribute__((__regparm__(1))) 100 void * 101 ___libc_tls_get_addr(void *vti) 102 { 103 return (__libc_tls_get_addr(vti)); 104 } 105 106 #endif 107 108 #ifndef PIC 109 110 static void * 111 libc_malloc_aligned(size_t size, size_t align) 112 { 113 void *mem, *res; 114 115 if (align < sizeof(void *)) 116 align = sizeof(void *); 117 118 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 119 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 120 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 121 return (res); 122 } 123 124 static void 125 libc_free_aligned(void *ptr) 126 { 127 void *mem; 128 uintptr_t x; 129 130 if (ptr == NULL) 131 return; 132 133 x = (uintptr_t)ptr; 134 x -= sizeof(void *); 135 mem = *(void **)x; 136 __je_bootstrap_free(mem); 137 } 138 139 #ifdef TLS_VARIANT_I 140 141 /* 142 * There are two versions of variant I of TLS 143 * 144 * - ARM and aarch64 uses original variant I as is described in [1] and [2], 145 * where TP points to start of TCB followed by aligned TLS segment. 146 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0] 147 * points to DTV vector and DTV values are real addresses (without bias). 148 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to 149 * TLS variables are computed by linker, so we cannot overalign TLS section. 150 * 151 * - PowerPC and RISC-V use modified version of variant I, described in [3] 152 * where TP points (with bias) to TLS and TCB immediately precedes TLS without 153 * any alignment gap[4]. Only TLS should be aligned. The TCB[0] points to DTV 154 * vector and DTV values are biased by constant value (TLS_DTV_OFFSET) from 155 * real addresses[5]. 156 * 157 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage 158 * www.akkadia.org/drepper/tls.pdf 159 * 160 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r) 161 * Architecture 162 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf 163 * 164 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification 165 * https://members.openpowerfoundation.org/document/dl/576 166 * 167 * [4] Its unclear if "without any alignment gap" is hard ABI requirement, 168 * but we must follow this rule due to suboptimal _tcb_set() 169 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but 170 * TCB as argument. 171 * 172 * [5] I'm not able to validate "values are biased" assertions. 173 */ 174 175 /* 176 * Return pointer to allocated TLS block 177 */ 178 static void * 179 get_tls_block_ptr(void *tcb, size_t tcbsize) 180 { 181 size_t extra_size, post_size, pre_size, tls_block_size; 182 183 /* Compute fragments sizes. */ 184 extra_size = tcbsize - TLS_TCB_SIZE; 185 #if defined(__aarch64__) || defined(__arm__) 186 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 187 #else 188 post_size = 0; 189 #endif 190 tls_block_size = tcbsize + post_size; 191 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 192 tls_block_size; 193 194 return ((char *)tcb - pre_size - extra_size); 195 } 196 197 /* 198 * Free Static TLS using the Variant I method. The tcbsize 199 * and tcbalign parameters must be the same as those used to allocate 200 * the block. 201 */ 202 void 203 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 204 { 205 Elf_Addr *dtv; 206 Elf_Addr **tls; 207 208 tls = (Elf_Addr **)tcb; 209 dtv = tls[0]; 210 __je_bootstrap_free(dtv); 211 libc_free_aligned(get_tls_block_ptr(tcb, tcbsize)); 212 } 213 214 /* 215 * Allocate Static TLS using the Variant I method. 216 * 217 * To handle all above requirements, we setup the following layout for 218 * TLS block: 219 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align)) 220 * 221 * +----------+--------------+--------------+-----------+------------------+ 222 * | pre gap | extended TCB | TCB | post gap | TLS segment | 223 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space | 224 * +----------+--------------+--------------+-----------+------------------+ 225 * 226 * where: 227 * extra_size is tcbsize - TLS_TCB_SIZE 228 * post_size is used to adjust TCB to TLS alignment for first version of TLS 229 * layout and is always 0 for second version. 230 * pre_size is used to adjust TCB alignment for first version and to adjust 231 * TLS alignment for second version. 232 * 233 */ 234 void * 235 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 236 { 237 Elf_Addr *dtv, **tcb; 238 char *tls_block, *tls; 239 size_t extra_size, maxalign, post_size, pre_size, tls_block_size; 240 241 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 242 return (oldtcb); 243 244 tls_assert(tcbalign >= TLS_TCB_ALIGN); 245 maxalign = MAX(tcbalign, libc_tls_init_align); 246 247 /* Compute fragmets sizes. */ 248 extra_size = tcbsize - TLS_TCB_SIZE; 249 #if defined(__aarch64__) || defined(__arm__) 250 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 251 #else 252 post_size = 0; 253 #endif 254 tls_block_size = tcbsize + post_size; 255 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 256 tls_block_size; 257 tls_block_size += pre_size + libc_tls_static_space; 258 259 /* Allocate whole TLS block */ 260 tls_block = libc_malloc_aligned(tls_block_size, maxalign); 261 if (tls_block == NULL) { 262 tls_msg("__libc_allocate_tls: Out of memory.\n"); 263 abort(); 264 } 265 memset(tls_block, 0, tls_block_size); 266 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size); 267 tls = (char *)tcb + TLS_TCB_SIZE + post_size; 268 269 if (oldtcb != NULL) { 270 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize), 271 tls_block_size); 272 libc_free_aligned(oldtcb); 273 274 /* Adjust the DTV. */ 275 dtv = tcb[0]; 276 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 277 } else { 278 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 279 if (dtv == NULL) { 280 tls_msg("__libc_allocate_tls: Out of memory.\n"); 281 abort(); 282 } 283 /* Build the DTV. */ 284 tcb[0] = dtv; 285 dtv[0] = 1; /* Generation. */ 286 dtv[1] = 1; /* Segments count. */ 287 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 288 289 if (libc_tls_init_size > 0) 290 memcpy(tls, libc_tls_init, libc_tls_init_size); 291 } 292 293 return (tcb); 294 } 295 296 #endif 297 298 #ifdef TLS_VARIANT_II 299 300 /* 301 * Free Static TLS using the Variant II method. 302 */ 303 void 304 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 305 { 306 size_t size; 307 Elf_Addr* dtv; 308 Elf_Addr tlsstart, tlsend; 309 310 /* 311 * Figure out the size of the initial TLS block so that we can 312 * find stuff which ___tls_get_addr() allocated dynamically. 313 */ 314 tcbalign = MAX(tcbalign, libc_tls_init_align); 315 size = roundup2(libc_tls_static_space, tcbalign); 316 317 dtv = ((Elf_Addr**)tcb)[1]; 318 tlsend = (Elf_Addr) tcb; 319 tlsstart = tlsend - size; 320 libc_free_aligned((void*)tlsstart); 321 __je_bootstrap_free(dtv); 322 } 323 324 /* 325 * Allocate Static TLS using the Variant II method. 326 */ 327 void * 328 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 329 { 330 size_t size; 331 char *tls; 332 Elf_Addr *dtv; 333 Elf_Addr segbase, oldsegbase; 334 335 tcbalign = MAX(tcbalign, libc_tls_init_align); 336 size = roundup2(libc_tls_static_space, tcbalign); 337 338 if (tcbsize < 2 * sizeof(Elf_Addr)) 339 tcbsize = 2 * sizeof(Elf_Addr); 340 tls = libc_malloc_aligned(size + tcbsize, tcbalign); 341 if (tls == NULL) { 342 tls_msg("__libc_allocate_tls: Out of memory.\n"); 343 abort(); 344 } 345 memset(tls, 0, size + tcbsize); 346 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 347 if (dtv == NULL) { 348 tls_msg("__libc_allocate_tls: Out of memory.\n"); 349 abort(); 350 } 351 352 segbase = (Elf_Addr)(tls + size); 353 ((Elf_Addr*)segbase)[0] = segbase; 354 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 355 356 dtv[0] = 1; 357 dtv[1] = 1; 358 dtv[2] = segbase - libc_tls_static_space; 359 360 if (oldtls) { 361 /* 362 * Copy the static TLS block over whole. 363 */ 364 oldsegbase = (Elf_Addr) oldtls; 365 memcpy((void *)(segbase - libc_tls_static_space), 366 (const void *)(oldsegbase - libc_tls_static_space), 367 libc_tls_static_space); 368 369 /* 370 * We assume that this block was the one we created with 371 * allocate_initial_tls(). 372 */ 373 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 374 } else { 375 memcpy((void *)(segbase - libc_tls_static_space), 376 libc_tls_init, libc_tls_init_size); 377 memset((void *)(segbase - libc_tls_static_space + 378 libc_tls_init_size), 0, 379 libc_tls_static_space - libc_tls_init_size); 380 } 381 382 return (void*) segbase; 383 } 384 385 #endif /* TLS_VARIANT_II */ 386 387 #else 388 389 void * 390 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 391 size_t tcbalign __unused) 392 { 393 return (0); 394 } 395 396 void 397 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 398 size_t tcbalign __unused) 399 { 400 } 401 402 #endif /* PIC */ 403 404 void 405 _init_tls(void) 406 { 407 #ifndef PIC 408 Elf_Addr *sp; 409 Elf_Auxinfo *aux, *auxp; 410 Elf_Phdr *phdr; 411 size_t phent, phnum; 412 int i; 413 void *tls; 414 415 sp = (Elf_Addr *) environ; 416 while (*sp++ != 0) 417 ; 418 aux = (Elf_Auxinfo *) sp; 419 phdr = NULL; 420 phent = phnum = 0; 421 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 422 switch (auxp->a_type) { 423 case AT_PHDR: 424 phdr = auxp->a_un.a_ptr; 425 break; 426 427 case AT_PHENT: 428 phent = auxp->a_un.a_val; 429 break; 430 431 case AT_PHNUM: 432 phnum = auxp->a_un.a_val; 433 break; 434 } 435 } 436 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 437 return; 438 439 for (i = 0; (unsigned) i < phnum; i++) { 440 if (phdr[i].p_type == PT_TLS) { 441 libc_tls_static_space = roundup2(phdr[i].p_memsz, 442 phdr[i].p_align); 443 libc_tls_init_size = phdr[i].p_filesz; 444 libc_tls_init_align = phdr[i].p_align; 445 libc_tls_init = (void *)phdr[i].p_vaddr; 446 break; 447 } 448 } 449 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 450 451 _tcb_set(tls); 452 #endif 453 } 454