1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2004 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Define stubs for TLS internals so that programs and libraries can 31 * link. These functions will be replaced by functional versions at 32 * runtime from ld-elf.so.1. 33 */ 34 35 #include <sys/param.h> 36 #include <stdlib.h> 37 #include <string.h> 38 #include <elf.h> 39 #include <unistd.h> 40 41 #include "rtld.h" 42 #include "libc_private.h" 43 44 #define tls_assert(cond) ((cond) ? (void) 0 : \ 45 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 46 __XSTRING(__LINE__) "\n"), abort())) 47 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 48 49 /* Provided by jemalloc to avoid bootstrapping issues. */ 50 void *__je_bootstrap_malloc(size_t size); 51 void *__je_bootstrap_calloc(size_t num, size_t size); 52 void __je_bootstrap_free(void *ptr); 53 54 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 55 __weak_reference(__libc_free_tls, _rtld_free_tls); 56 57 #ifdef __i386__ 58 59 __weak_reference(___libc_tls_get_addr, ___tls_get_addr); 60 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 61 62 #endif 63 64 void * __libc_tls_get_addr(void *); 65 __weak_reference(__libc_tls_get_addr, __tls_get_addr); 66 67 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 68 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 69 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 70 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 71 72 #ifndef PIC 73 74 static size_t libc_tls_static_space; 75 static size_t libc_tls_init_size; 76 static size_t libc_tls_init_align; 77 static void *libc_tls_init; 78 #endif 79 80 void * 81 __libc_tls_get_addr(void *vti) 82 { 83 uintptr_t *dtv; 84 tls_index *ti; 85 86 dtv = _tcb_get()->tcb_dtv; 87 ti = vti; 88 return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) + 89 TLS_DTV_OFFSET); 90 } 91 92 #ifdef __i386__ 93 94 /* GNU ABI */ 95 96 __attribute__((__regparm__(1))) 97 void * 98 ___libc_tls_get_addr(void *vti) 99 { 100 return (__libc_tls_get_addr(vti)); 101 } 102 103 #endif 104 105 #ifndef PIC 106 107 static void * 108 libc_malloc_aligned(size_t size, size_t align) 109 { 110 void *mem, *res; 111 112 if (align < sizeof(void *)) 113 align = sizeof(void *); 114 115 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 116 if (mem == NULL) 117 return (NULL); 118 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 119 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 120 return (res); 121 } 122 123 static void 124 libc_free_aligned(void *ptr) 125 { 126 void *mem; 127 uintptr_t x; 128 129 if (ptr == NULL) 130 return; 131 132 x = (uintptr_t)ptr; 133 x -= sizeof(void *); 134 mem = *(void **)x; 135 __je_bootstrap_free(mem); 136 } 137 138 #ifdef TLS_VARIANT_I 139 140 /* 141 * There are two versions of variant I of TLS 142 * 143 * - ARM and aarch64 uses original variant I as is described in [1] and [2], 144 * where TP points to start of TCB followed by aligned TLS segment. 145 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0] 146 * points to DTV vector and DTV values are real addresses (without bias). 147 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to 148 * TLS variables are computed by linker, so we cannot overalign TLS section. 149 * 150 * - PowerPC and RISC-V use modified version of variant I, described in [3] 151 * where TP points (with bias) to TLS and TCB immediately precedes TLS without 152 * any alignment gap[4]. Only TLS should be aligned. The TCB[0] points to DTV 153 * vector and DTV values are biased by constant value (TLS_DTV_OFFSET) from 154 * real addresses[5]. 155 * 156 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage 157 * www.akkadia.org/drepper/tls.pdf 158 * 159 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r) 160 * Architecture 161 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf 162 * 163 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification 164 * https://members.openpowerfoundation.org/document/dl/576 165 * 166 * [4] Its unclear if "without any alignment gap" is hard ABI requirement, 167 * but we must follow this rule due to suboptimal _tcb_set() 168 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but 169 * TCB as argument. 170 * 171 * [5] I'm not able to validate "values are biased" assertions. 172 */ 173 174 /* 175 * Return pointer to allocated TLS block 176 */ 177 static void * 178 get_tls_block_ptr(void *tcb, size_t tcbsize) 179 { 180 size_t extra_size, post_size, pre_size, tls_block_size; 181 182 /* Compute fragments sizes. */ 183 extra_size = tcbsize - TLS_TCB_SIZE; 184 #if defined(__aarch64__) || defined(__arm__) 185 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 186 #else 187 post_size = 0; 188 #endif 189 tls_block_size = tcbsize + post_size; 190 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 191 tls_block_size; 192 193 return ((char *)tcb - pre_size - extra_size); 194 } 195 196 /* 197 * Free Static TLS using the Variant I method. The tcbsize 198 * and tcbalign parameters must be the same as those used to allocate 199 * the block. 200 */ 201 void 202 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 203 { 204 Elf_Addr *dtv; 205 Elf_Addr **tls; 206 207 tls = (Elf_Addr **)tcb; 208 dtv = tls[0]; 209 __je_bootstrap_free(dtv); 210 libc_free_aligned(get_tls_block_ptr(tcb, tcbsize)); 211 } 212 213 /* 214 * Allocate Static TLS using the Variant I method. 215 * 216 * To handle all above requirements, we setup the following layout for 217 * TLS block: 218 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align)) 219 * 220 * +----------+--------------+--------------+-----------+------------------+ 221 * | pre gap | extended TCB | TCB | post gap | TLS segment | 222 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space | 223 * +----------+--------------+--------------+-----------+------------------+ 224 * 225 * where: 226 * extra_size is tcbsize - TLS_TCB_SIZE 227 * post_size is used to adjust TCB to TLS alignment for first version of TLS 228 * layout and is always 0 for second version. 229 * pre_size is used to adjust TCB alignment for first version and to adjust 230 * TLS alignment for second version. 231 * 232 */ 233 void * 234 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 235 { 236 Elf_Addr *dtv, **tcb; 237 char *tls_block, *tls; 238 size_t extra_size, maxalign, post_size, pre_size, tls_block_size; 239 240 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 241 return (oldtcb); 242 243 tls_assert(tcbalign >= TLS_TCB_ALIGN); 244 maxalign = MAX(tcbalign, libc_tls_init_align); 245 246 /* Compute fragmets sizes. */ 247 extra_size = tcbsize - TLS_TCB_SIZE; 248 #if defined(__aarch64__) || defined(__arm__) 249 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 250 #else 251 post_size = 0; 252 #endif 253 tls_block_size = tcbsize + post_size; 254 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 255 tls_block_size; 256 tls_block_size += pre_size + libc_tls_static_space; 257 258 /* Allocate whole TLS block */ 259 tls_block = libc_malloc_aligned(tls_block_size, maxalign); 260 if (tls_block == NULL) { 261 tls_msg("__libc_allocate_tls: Out of memory.\n"); 262 abort(); 263 } 264 memset(tls_block, 0, tls_block_size); 265 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size); 266 tls = (char *)tcb + TLS_TCB_SIZE + post_size; 267 268 if (oldtcb != NULL) { 269 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize), 270 tls_block_size); 271 libc_free_aligned(oldtcb); 272 273 /* Adjust the DTV. */ 274 dtv = tcb[0]; 275 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 276 } else { 277 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 278 if (dtv == NULL) { 279 tls_msg("__libc_allocate_tls: Out of memory.\n"); 280 abort(); 281 } 282 /* Build the DTV. */ 283 tcb[0] = dtv; 284 dtv[0] = 1; /* Generation. */ 285 dtv[1] = 1; /* Segments count. */ 286 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 287 288 if (libc_tls_init_size > 0) 289 memcpy(tls, libc_tls_init, libc_tls_init_size); 290 } 291 292 return (tcb); 293 } 294 295 #endif 296 297 #ifdef TLS_VARIANT_II 298 299 /* 300 * Free Static TLS using the Variant II method. 301 */ 302 void 303 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 304 { 305 size_t size; 306 Elf_Addr* dtv; 307 Elf_Addr tlsstart, tlsend; 308 309 /* 310 * Figure out the size of the initial TLS block so that we can 311 * find stuff which ___tls_get_addr() allocated dynamically. 312 */ 313 tcbalign = MAX(tcbalign, libc_tls_init_align); 314 size = roundup2(libc_tls_static_space, tcbalign); 315 316 dtv = ((Elf_Addr**)tcb)[1]; 317 tlsend = (Elf_Addr) tcb; 318 tlsstart = tlsend - size; 319 libc_free_aligned((void*)tlsstart); 320 __je_bootstrap_free(dtv); 321 } 322 323 /* 324 * Allocate Static TLS using the Variant II method. 325 */ 326 void * 327 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 328 { 329 size_t size; 330 char *tls; 331 Elf_Addr *dtv; 332 Elf_Addr segbase, oldsegbase; 333 334 tcbalign = MAX(tcbalign, libc_tls_init_align); 335 size = roundup2(libc_tls_static_space, tcbalign); 336 337 if (tcbsize < 2 * sizeof(Elf_Addr)) 338 tcbsize = 2 * sizeof(Elf_Addr); 339 tls = libc_malloc_aligned(size + tcbsize, tcbalign); 340 if (tls == NULL) { 341 tls_msg("__libc_allocate_tls: Out of memory.\n"); 342 abort(); 343 } 344 memset(tls, 0, size + tcbsize); 345 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 346 if (dtv == NULL) { 347 tls_msg("__libc_allocate_tls: Out of memory.\n"); 348 abort(); 349 } 350 351 segbase = (Elf_Addr)(tls + size); 352 ((Elf_Addr*)segbase)[0] = segbase; 353 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 354 355 dtv[0] = 1; 356 dtv[1] = 1; 357 dtv[2] = segbase - libc_tls_static_space; 358 359 if (oldtls) { 360 /* 361 * Copy the static TLS block over whole. 362 */ 363 oldsegbase = (Elf_Addr) oldtls; 364 memcpy((void *)(segbase - libc_tls_static_space), 365 (const void *)(oldsegbase - libc_tls_static_space), 366 libc_tls_static_space); 367 368 /* 369 * We assume that this block was the one we created with 370 * allocate_initial_tls(). 371 */ 372 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 373 } else { 374 memcpy((void *)(segbase - libc_tls_static_space), 375 libc_tls_init, libc_tls_init_size); 376 memset((void *)(segbase - libc_tls_static_space + 377 libc_tls_init_size), 0, 378 libc_tls_static_space - libc_tls_init_size); 379 } 380 381 return (void*) segbase; 382 } 383 384 #endif /* TLS_VARIANT_II */ 385 386 #else 387 388 void * 389 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 390 size_t tcbalign __unused) 391 { 392 return (0); 393 } 394 395 void 396 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 397 size_t tcbalign __unused) 398 { 399 } 400 401 #endif /* PIC */ 402 403 void 404 _init_tls(void) 405 { 406 #ifndef PIC 407 Elf_Addr *sp; 408 Elf_Auxinfo *aux, *auxp; 409 Elf_Phdr *phdr; 410 size_t phent, phnum; 411 int i; 412 void *tls; 413 414 sp = (Elf_Addr *) environ; 415 while (*sp++ != 0) 416 ; 417 aux = (Elf_Auxinfo *) sp; 418 phdr = NULL; 419 phent = phnum = 0; 420 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 421 switch (auxp->a_type) { 422 case AT_PHDR: 423 phdr = auxp->a_un.a_ptr; 424 break; 425 426 case AT_PHENT: 427 phent = auxp->a_un.a_val; 428 break; 429 430 case AT_PHNUM: 431 phnum = auxp->a_un.a_val; 432 break; 433 } 434 } 435 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 436 return; 437 438 for (i = 0; (unsigned) i < phnum; i++) { 439 if (phdr[i].p_type == PT_TLS) { 440 libc_tls_static_space = roundup2(phdr[i].p_memsz, 441 phdr[i].p_align); 442 libc_tls_init_size = phdr[i].p_filesz; 443 libc_tls_init_align = phdr[i].p_align; 444 libc_tls_init = (void *)phdr[i].p_vaddr; 445 break; 446 } 447 } 448 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 449 450 _tcb_set(tls); 451 #endif 452 } 453