1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2004 Doug Rabson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * Define stubs for TLS internals so that programs and libraries can
31 * link. These functions will be replaced by functional versions at
32 * runtime from ld-elf.so.1.
33 */
34
35 #include <sys/param.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #include <elf.h>
39 #include <unistd.h>
40
41 #include "rtld.h"
42 #include "libc_private.h"
43
44 #define tls_assert(cond) ((cond) ? (void) 0 : \
45 (tls_msg(#cond ": assert failed: " __FILE__ ":" \
46 __XSTRING(__LINE__) "\n"), abort()))
47 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s))
48
49 /* Provided by jemalloc to avoid bootstrapping issues. */
50 void *__je_bootstrap_malloc(size_t size);
51 void *__je_bootstrap_calloc(size_t num, size_t size);
52 void __je_bootstrap_free(void *ptr);
53
54 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls);
55 __weak_reference(__libc_free_tls, _rtld_free_tls);
56
57 #ifdef __i386__
58
59 __weak_reference(___libc_tls_get_addr, ___tls_get_addr);
60 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *);
61
62 #endif
63
64 void * __libc_tls_get_addr(void *);
65 __weak_reference(__libc_tls_get_addr, __tls_get_addr);
66
67 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
68 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
69 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
70 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
71
72 #ifndef PIC
73
74 static size_t libc_tls_static_space;
75 static size_t libc_tls_init_size;
76 static size_t libc_tls_init_align;
77 static void *libc_tls_init;
78 #endif
79
80 void *
__libc_tls_get_addr(void * vti)81 __libc_tls_get_addr(void *vti)
82 {
83 uintptr_t *dtv;
84 tls_index *ti;
85
86 dtv = _tcb_get()->tcb_dtv;
87 ti = vti;
88 return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) +
89 TLS_DTV_OFFSET);
90 }
91
92 #ifdef __i386__
93
94 /* GNU ABI */
95
96 __attribute__((__regparm__(1)))
97 void *
___libc_tls_get_addr(void * vti)98 ___libc_tls_get_addr(void *vti)
99 {
100 return (__libc_tls_get_addr(vti));
101 }
102
103 #endif
104
105 #ifndef PIC
106
107 static void *
libc_malloc_aligned(size_t size,size_t align)108 libc_malloc_aligned(size_t size, size_t align)
109 {
110 void *mem, *res;
111
112 if (align < sizeof(void *))
113 align = sizeof(void *);
114
115 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1);
116 if (mem == NULL)
117 return (NULL);
118 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align);
119 *(void **)((uintptr_t)res - sizeof(void *)) = mem;
120 return (res);
121 }
122
123 static void
libc_free_aligned(void * ptr)124 libc_free_aligned(void *ptr)
125 {
126 void *mem;
127 uintptr_t x;
128
129 if (ptr == NULL)
130 return;
131
132 x = (uintptr_t)ptr;
133 x -= sizeof(void *);
134 mem = *(void **)x;
135 __je_bootstrap_free(mem);
136 }
137
138 #ifdef TLS_VARIANT_I
139
140 /*
141 * There are two versions of variant I of TLS
142 *
143 * - ARM and aarch64 uses original variant I as is described in [1] and [2],
144 * where TP points to start of TCB followed by aligned TLS segment.
145 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0]
146 * points to DTV vector and DTV values are real addresses (without bias).
147 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to
148 * TLS variables are computed by linker, so we cannot overalign TLS section.
149 *
150 * - PowerPC and RISC-V use modified version of variant I, described in [3]
151 * where TP points (with bias) to TLS and TCB immediately precedes TLS without
152 * any alignment gap[4]. Only TLS should be aligned. The TCB[0] points to DTV
153 * vector and DTV values are biased by constant value (TLS_DTV_OFFSET) from
154 * real addresses[5].
155 *
156 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage
157 * www.akkadia.org/drepper/tls.pdf
158 *
159 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r)
160 * Architecture
161 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf
162 *
163 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification
164 * https://members.openpowerfoundation.org/document/dl/576
165 *
166 * [4] Its unclear if "without any alignment gap" is hard ABI requirement,
167 * but we must follow this rule due to suboptimal _tcb_set()
168 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but
169 * TCB as argument.
170 *
171 * [5] I'm not able to validate "values are biased" assertions.
172 */
173
174 /*
175 * Return pointer to allocated TLS block
176 */
177 static void *
get_tls_block_ptr(void * tcb,size_t tcbsize)178 get_tls_block_ptr(void *tcb, size_t tcbsize)
179 {
180 size_t extra_size, post_size, pre_size, tls_block_size;
181
182 /* Compute fragments sizes. */
183 extra_size = tcbsize - TLS_TCB_SIZE;
184 #if defined(__aarch64__) || defined(__arm__)
185 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE;
186 #else
187 post_size = 0;
188 #endif
189 tls_block_size = tcbsize + post_size;
190 pre_size = roundup2(tls_block_size, libc_tls_init_align) -
191 tls_block_size;
192
193 return ((char *)tcb - pre_size - extra_size);
194 }
195
196 /*
197 * Free Static TLS using the Variant I method. The tcbsize
198 * and tcbalign parameters must be the same as those used to allocate
199 * the block.
200 */
201 void
__libc_free_tls(void * tcb,size_t tcbsize,size_t tcbalign __unused)202 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused)
203 {
204 Elf_Addr *dtv;
205 Elf_Addr **tls;
206
207 tls = (Elf_Addr **)tcb;
208 dtv = tls[0];
209 __je_bootstrap_free(dtv);
210 libc_free_aligned(get_tls_block_ptr(tcb, tcbsize));
211 }
212
213 /*
214 * Allocate Static TLS using the Variant I method.
215 *
216 * To handle all above requirements, we setup the following layout for
217 * TLS block:
218 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align))
219 *
220 * +----------+--------------+--------------+-----------+------------------+
221 * | pre gap | extended TCB | TCB | post gap | TLS segment |
222 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space |
223 * +----------+--------------+--------------+-----------+------------------+
224 *
225 * where:
226 * extra_size is tcbsize - TLS_TCB_SIZE
227 * post_size is used to adjust TCB to TLS alignment for first version of TLS
228 * layout and is always 0 for second version.
229 * pre_size is used to adjust TCB alignment for first version and to adjust
230 * TLS alignment for second version.
231 *
232 */
233 void *
__libc_allocate_tls(void * oldtcb,size_t tcbsize,size_t tcbalign)234 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign)
235 {
236 Elf_Addr *dtv, **tcb;
237 char *tls_block, *tls;
238 size_t extra_size, maxalign, post_size, pre_size, tls_block_size;
239
240 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE)
241 return (oldtcb);
242
243 tls_assert(tcbalign >= TLS_TCB_ALIGN);
244 maxalign = MAX(tcbalign, libc_tls_init_align);
245
246 /* Compute fragmets sizes. */
247 extra_size = tcbsize - TLS_TCB_SIZE;
248 #if defined(__aarch64__) || defined(__arm__)
249 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE;
250 #else
251 post_size = 0;
252 #endif
253 tls_block_size = tcbsize + post_size;
254 pre_size = roundup2(tls_block_size, libc_tls_init_align) -
255 tls_block_size;
256 tls_block_size += pre_size + libc_tls_static_space;
257
258 /* Allocate whole TLS block */
259 tls_block = libc_malloc_aligned(tls_block_size, maxalign);
260 if (tls_block == NULL) {
261 tls_msg("__libc_allocate_tls: Out of memory.\n");
262 abort();
263 }
264 memset(tls_block, 0, tls_block_size);
265 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size);
266 tls = (char *)tcb + TLS_TCB_SIZE + post_size;
267
268 if (oldtcb != NULL) {
269 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize),
270 tls_block_size);
271 libc_free_aligned(oldtcb);
272
273 /* Adjust the DTV. */
274 dtv = tcb[0];
275 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET);
276 } else {
277 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
278 if (dtv == NULL) {
279 tls_msg("__libc_allocate_tls: Out of memory.\n");
280 abort();
281 }
282 /* Build the DTV. */
283 tcb[0] = dtv;
284 dtv[0] = 1; /* Generation. */
285 dtv[1] = 1; /* Segments count. */
286 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET);
287
288 if (libc_tls_init_size > 0)
289 memcpy(tls, libc_tls_init, libc_tls_init_size);
290 }
291
292 return (tcb);
293 }
294
295 #endif
296
297 #ifdef TLS_VARIANT_II
298
299 /*
300 * Free Static TLS using the Variant II method.
301 */
302 void
__libc_free_tls(void * tcb,size_t tcbsize __unused,size_t tcbalign)303 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign)
304 {
305 size_t size;
306 Elf_Addr* dtv;
307 Elf_Addr tlsstart, tlsend;
308
309 /*
310 * Figure out the size of the initial TLS block so that we can
311 * find stuff which ___tls_get_addr() allocated dynamically.
312 */
313 tcbalign = MAX(tcbalign, libc_tls_init_align);
314 size = roundup2(libc_tls_static_space, tcbalign);
315
316 dtv = ((Elf_Addr**)tcb)[1];
317 tlsend = (Elf_Addr) tcb;
318 tlsstart = tlsend - size;
319 libc_free_aligned((void*)tlsstart);
320 __je_bootstrap_free(dtv);
321 }
322
323 /*
324 * Allocate Static TLS using the Variant II method.
325 */
326 void *
__libc_allocate_tls(void * oldtls,size_t tcbsize,size_t tcbalign)327 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign)
328 {
329 size_t size;
330 char *tls;
331 Elf_Addr *dtv;
332 Elf_Addr segbase, oldsegbase;
333
334 tcbalign = MAX(tcbalign, libc_tls_init_align);
335 size = roundup2(libc_tls_static_space, tcbalign);
336
337 if (tcbsize < 2 * sizeof(Elf_Addr))
338 tcbsize = 2 * sizeof(Elf_Addr);
339 tls = libc_malloc_aligned(size + tcbsize, tcbalign);
340 if (tls == NULL) {
341 tls_msg("__libc_allocate_tls: Out of memory.\n");
342 abort();
343 }
344 memset(tls, 0, size + tcbsize);
345 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
346 if (dtv == NULL) {
347 tls_msg("__libc_allocate_tls: Out of memory.\n");
348 abort();
349 }
350
351 segbase = (Elf_Addr)(tls + size);
352 ((Elf_Addr*)segbase)[0] = segbase;
353 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv;
354
355 dtv[0] = 1;
356 dtv[1] = 1;
357 dtv[2] = segbase - libc_tls_static_space;
358
359 if (oldtls) {
360 /*
361 * Copy the static TLS block over whole.
362 */
363 oldsegbase = (Elf_Addr) oldtls;
364 memcpy((void *)(segbase - libc_tls_static_space),
365 (const void *)(oldsegbase - libc_tls_static_space),
366 libc_tls_static_space);
367
368 /*
369 * We assume that this block was the one we created with
370 * allocate_initial_tls().
371 */
372 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr));
373 } else {
374 memcpy((void *)(segbase - libc_tls_static_space),
375 libc_tls_init, libc_tls_init_size);
376 memset((void *)(segbase - libc_tls_static_space +
377 libc_tls_init_size), 0,
378 libc_tls_static_space - libc_tls_init_size);
379 }
380
381 return (void*) segbase;
382 }
383
384 #endif /* TLS_VARIANT_II */
385
386 #else
387
388 void *
__libc_allocate_tls(void * oldtls __unused,size_t tcbsize __unused,size_t tcbalign __unused)389 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused,
390 size_t tcbalign __unused)
391 {
392 return (0);
393 }
394
395 void
__libc_free_tls(void * tcb __unused,size_t tcbsize __unused,size_t tcbalign __unused)396 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused,
397 size_t tcbalign __unused)
398 {
399 }
400
401 #endif /* PIC */
402
403 void
_init_tls(void)404 _init_tls(void)
405 {
406 #ifndef PIC
407 Elf_Addr *sp;
408 Elf_Auxinfo *aux, *auxp;
409 Elf_Phdr *phdr;
410 size_t phent, phnum;
411 int i;
412 void *tls;
413
414 sp = (Elf_Addr *) environ;
415 while (*sp++ != 0)
416 ;
417 aux = (Elf_Auxinfo *) sp;
418 phdr = NULL;
419 phent = phnum = 0;
420 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) {
421 switch (auxp->a_type) {
422 case AT_PHDR:
423 phdr = auxp->a_un.a_ptr;
424 break;
425
426 case AT_PHENT:
427 phent = auxp->a_un.a_val;
428 break;
429
430 case AT_PHNUM:
431 phnum = auxp->a_un.a_val;
432 break;
433 }
434 }
435 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0)
436 return;
437
438 for (i = 0; (unsigned) i < phnum; i++) {
439 if (phdr[i].p_type == PT_TLS) {
440 libc_tls_static_space = roundup2(phdr[i].p_memsz,
441 phdr[i].p_align);
442 libc_tls_init_size = phdr[i].p_filesz;
443 libc_tls_init_align = phdr[i].p_align;
444 libc_tls_init = (void *)phdr[i].p_vaddr;
445 break;
446 }
447 }
448 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN);
449
450 _tcb_set(tls);
451 #endif
452 }
453