1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2004 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Define stubs for TLS internals so that programs and libraries can 31 * link. These functions will be replaced by functional versions at 32 * runtime from ld-elf.so.1. 33 */ 34 35 #include <sys/cdefs.h> 36 #include <sys/param.h> 37 #include <stdlib.h> 38 #include <string.h> 39 #include <elf.h> 40 #include <unistd.h> 41 42 #include "rtld.h" 43 #include "libc_private.h" 44 45 #define tls_assert(cond) ((cond) ? (void) 0 : \ 46 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 47 __XSTRING(__LINE__) "\n"), abort())) 48 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 49 50 /* Provided by jemalloc to avoid bootstrapping issues. */ 51 void *__je_bootstrap_malloc(size_t size); 52 void *__je_bootstrap_calloc(size_t num, size_t size); 53 void __je_bootstrap_free(void *ptr); 54 55 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 56 __weak_reference(__libc_free_tls, _rtld_free_tls); 57 58 #ifdef __i386__ 59 60 __weak_reference(___libc_tls_get_addr, ___tls_get_addr); 61 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 62 63 #endif 64 65 void * __libc_tls_get_addr(void *); 66 __weak_reference(__libc_tls_get_addr, __tls_get_addr); 67 68 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 69 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 70 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 71 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 72 73 #ifndef PIC 74 75 static size_t libc_tls_static_space; 76 static size_t libc_tls_init_size; 77 static size_t libc_tls_init_align; 78 static void *libc_tls_init; 79 #endif 80 81 void * 82 __libc_tls_get_addr(void *vti) 83 { 84 uintptr_t *dtv; 85 tls_index *ti; 86 87 dtv = _tcb_get()->tcb_dtv; 88 ti = vti; 89 return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) + 90 TLS_DTV_OFFSET); 91 } 92 93 #ifdef __i386__ 94 95 /* GNU ABI */ 96 97 __attribute__((__regparm__(1))) 98 void * 99 ___libc_tls_get_addr(void *vti) 100 { 101 return (__libc_tls_get_addr(vti)); 102 } 103 104 #endif 105 106 #ifndef PIC 107 108 static void * 109 libc_malloc_aligned(size_t size, size_t align) 110 { 111 void *mem, *res; 112 113 if (align < sizeof(void *)) 114 align = sizeof(void *); 115 116 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 117 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 118 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 119 return (res); 120 } 121 122 static void 123 libc_free_aligned(void *ptr) 124 { 125 void *mem; 126 uintptr_t x; 127 128 if (ptr == NULL) 129 return; 130 131 x = (uintptr_t)ptr; 132 x -= sizeof(void *); 133 mem = *(void **)x; 134 __je_bootstrap_free(mem); 135 } 136 137 #ifdef TLS_VARIANT_I 138 139 /* 140 * There are two versions of variant I of TLS 141 * 142 * - ARM and aarch64 uses original variant I as is described in [1] and [2], 143 * where TP points to start of TCB followed by aligned TLS segment. 144 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0] 145 * points to DTV vector and DTV values are real addresses (without bias). 146 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to 147 * TLS variables are computed by linker, so we cannot overalign TLS section. 148 * 149 * - PowerPC and RISC-V use modified version of variant I, described in [3] 150 * where TP points (with bias) to TLS and TCB immediately precedes TLS without 151 * any alignment gap[4]. Only TLS should be aligned. The TCB[0] points to DTV 152 * vector and DTV values are biased by constant value (TLS_DTV_OFFSET) from 153 * real addresses[5]. 154 * 155 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage 156 * www.akkadia.org/drepper/tls.pdf 157 * 158 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r) 159 * Architecture 160 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf 161 * 162 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification 163 * https://members.openpowerfoundation.org/document/dl/576 164 * 165 * [4] Its unclear if "without any alignment gap" is hard ABI requirement, 166 * but we must follow this rule due to suboptimal _tcb_set() 167 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but 168 * TCB as argument. 169 * 170 * [5] I'm not able to validate "values are biased" assertions. 171 */ 172 173 /* 174 * Return pointer to allocated TLS block 175 */ 176 static void * 177 get_tls_block_ptr(void *tcb, size_t tcbsize) 178 { 179 size_t extra_size, post_size, pre_size, tls_block_size; 180 181 /* Compute fragments sizes. */ 182 extra_size = tcbsize - TLS_TCB_SIZE; 183 #if defined(__aarch64__) || defined(__arm__) 184 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 185 #else 186 post_size = 0; 187 #endif 188 tls_block_size = tcbsize + post_size; 189 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 190 tls_block_size; 191 192 return ((char *)tcb - pre_size - extra_size); 193 } 194 195 /* 196 * Free Static TLS using the Variant I method. The tcbsize 197 * and tcbalign parameters must be the same as those used to allocate 198 * the block. 199 */ 200 void 201 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 202 { 203 Elf_Addr *dtv; 204 Elf_Addr **tls; 205 206 tls = (Elf_Addr **)tcb; 207 dtv = tls[0]; 208 __je_bootstrap_free(dtv); 209 libc_free_aligned(get_tls_block_ptr(tcb, tcbsize)); 210 } 211 212 /* 213 * Allocate Static TLS using the Variant I method. 214 * 215 * To handle all above requirements, we setup the following layout for 216 * TLS block: 217 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align)) 218 * 219 * +----------+--------------+--------------+-----------+------------------+ 220 * | pre gap | extended TCB | TCB | post gap | TLS segment | 221 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space | 222 * +----------+--------------+--------------+-----------+------------------+ 223 * 224 * where: 225 * extra_size is tcbsize - TLS_TCB_SIZE 226 * post_size is used to adjust TCB to TLS alignment for first version of TLS 227 * layout and is always 0 for second version. 228 * pre_size is used to adjust TCB alignment for first version and to adjust 229 * TLS alignment for second version. 230 * 231 */ 232 void * 233 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 234 { 235 Elf_Addr *dtv, **tcb; 236 char *tls_block, *tls; 237 size_t extra_size, maxalign, post_size, pre_size, tls_block_size; 238 239 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 240 return (oldtcb); 241 242 tls_assert(tcbalign >= TLS_TCB_ALIGN); 243 maxalign = MAX(tcbalign, libc_tls_init_align); 244 245 /* Compute fragmets sizes. */ 246 extra_size = tcbsize - TLS_TCB_SIZE; 247 #if defined(__aarch64__) || defined(__arm__) 248 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 249 #else 250 post_size = 0; 251 #endif 252 tls_block_size = tcbsize + post_size; 253 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 254 tls_block_size; 255 tls_block_size += pre_size + libc_tls_static_space; 256 257 /* Allocate whole TLS block */ 258 tls_block = libc_malloc_aligned(tls_block_size, maxalign); 259 if (tls_block == NULL) { 260 tls_msg("__libc_allocate_tls: Out of memory.\n"); 261 abort(); 262 } 263 memset(tls_block, 0, tls_block_size); 264 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size); 265 tls = (char *)tcb + TLS_TCB_SIZE + post_size; 266 267 if (oldtcb != NULL) { 268 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize), 269 tls_block_size); 270 libc_free_aligned(oldtcb); 271 272 /* Adjust the DTV. */ 273 dtv = tcb[0]; 274 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 275 } else { 276 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 277 if (dtv == NULL) { 278 tls_msg("__libc_allocate_tls: Out of memory.\n"); 279 abort(); 280 } 281 /* Build the DTV. */ 282 tcb[0] = dtv; 283 dtv[0] = 1; /* Generation. */ 284 dtv[1] = 1; /* Segments count. */ 285 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET); 286 287 if (libc_tls_init_size > 0) 288 memcpy(tls, libc_tls_init, libc_tls_init_size); 289 } 290 291 return (tcb); 292 } 293 294 #endif 295 296 #ifdef TLS_VARIANT_II 297 298 /* 299 * Free Static TLS using the Variant II method. 300 */ 301 void 302 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 303 { 304 size_t size; 305 Elf_Addr* dtv; 306 Elf_Addr tlsstart, tlsend; 307 308 /* 309 * Figure out the size of the initial TLS block so that we can 310 * find stuff which ___tls_get_addr() allocated dynamically. 311 */ 312 tcbalign = MAX(tcbalign, libc_tls_init_align); 313 size = roundup2(libc_tls_static_space, tcbalign); 314 315 dtv = ((Elf_Addr**)tcb)[1]; 316 tlsend = (Elf_Addr) tcb; 317 tlsstart = tlsend - size; 318 libc_free_aligned((void*)tlsstart); 319 __je_bootstrap_free(dtv); 320 } 321 322 /* 323 * Allocate Static TLS using the Variant II method. 324 */ 325 void * 326 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 327 { 328 size_t size; 329 char *tls; 330 Elf_Addr *dtv; 331 Elf_Addr segbase, oldsegbase; 332 333 tcbalign = MAX(tcbalign, libc_tls_init_align); 334 size = roundup2(libc_tls_static_space, tcbalign); 335 336 if (tcbsize < 2 * sizeof(Elf_Addr)) 337 tcbsize = 2 * sizeof(Elf_Addr); 338 tls = libc_malloc_aligned(size + tcbsize, tcbalign); 339 if (tls == NULL) { 340 tls_msg("__libc_allocate_tls: Out of memory.\n"); 341 abort(); 342 } 343 memset(tls, 0, size + tcbsize); 344 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 345 if (dtv == NULL) { 346 tls_msg("__libc_allocate_tls: Out of memory.\n"); 347 abort(); 348 } 349 350 segbase = (Elf_Addr)(tls + size); 351 ((Elf_Addr*)segbase)[0] = segbase; 352 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 353 354 dtv[0] = 1; 355 dtv[1] = 1; 356 dtv[2] = segbase - libc_tls_static_space; 357 358 if (oldtls) { 359 /* 360 * Copy the static TLS block over whole. 361 */ 362 oldsegbase = (Elf_Addr) oldtls; 363 memcpy((void *)(segbase - libc_tls_static_space), 364 (const void *)(oldsegbase - libc_tls_static_space), 365 libc_tls_static_space); 366 367 /* 368 * We assume that this block was the one we created with 369 * allocate_initial_tls(). 370 */ 371 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 372 } else { 373 memcpy((void *)(segbase - libc_tls_static_space), 374 libc_tls_init, libc_tls_init_size); 375 memset((void *)(segbase - libc_tls_static_space + 376 libc_tls_init_size), 0, 377 libc_tls_static_space - libc_tls_init_size); 378 } 379 380 return (void*) segbase; 381 } 382 383 #endif /* TLS_VARIANT_II */ 384 385 #else 386 387 void * 388 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 389 size_t tcbalign __unused) 390 { 391 return (0); 392 } 393 394 void 395 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 396 size_t tcbalign __unused) 397 { 398 } 399 400 #endif /* PIC */ 401 402 void 403 _init_tls(void) 404 { 405 #ifndef PIC 406 Elf_Addr *sp; 407 Elf_Auxinfo *aux, *auxp; 408 Elf_Phdr *phdr; 409 size_t phent, phnum; 410 int i; 411 void *tls; 412 413 sp = (Elf_Addr *) environ; 414 while (*sp++ != 0) 415 ; 416 aux = (Elf_Auxinfo *) sp; 417 phdr = NULL; 418 phent = phnum = 0; 419 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 420 switch (auxp->a_type) { 421 case AT_PHDR: 422 phdr = auxp->a_un.a_ptr; 423 break; 424 425 case AT_PHENT: 426 phent = auxp->a_un.a_val; 427 break; 428 429 case AT_PHNUM: 430 phnum = auxp->a_un.a_val; 431 break; 432 } 433 } 434 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 435 return; 436 437 for (i = 0; (unsigned) i < phnum; i++) { 438 if (phdr[i].p_type == PT_TLS) { 439 libc_tls_static_space = roundup2(phdr[i].p_memsz, 440 phdr[i].p_align); 441 libc_tls_init_size = phdr[i].p_filesz; 442 libc_tls_init_align = phdr[i].p_align; 443 libc_tls_init = (void *)phdr[i].p_vaddr; 444 break; 445 } 446 } 447 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 448 449 _tcb_set(tls); 450 #endif 451 } 452