xref: /freebsd/lib/libc/gen/tls.c (revision 996e4759621dd7150cec5ad240e252b35a4d9db4)
1 /*-
2  * Copyright (c) 2004 Doug Rabson
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  *	$FreeBSD$
27  */
28 
29 /*
30  * Define stubs for TLS internals so that programs and libraries can
31  * link. These functions will be replaced by functional versions at
32  * runtime from ld-elf.so.1.
33  */
34 
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <elf.h>
40 #include <unistd.h>
41 
42 #include "libc_private.h"
43 
44 #define	tls_assert(cond)	((cond) ? (void) 0 :			\
45     (tls_msg(#cond ": assert failed: " __FILE__ ":"			\
46       __XSTRING(__LINE__) "\n"), abort()))
47 #define	tls_msg(s)		write(STDOUT_FILENO, s, strlen(s))
48 
49 /* Provided by jemalloc to avoid bootstrapping issues. */
50 void	*__je_bootstrap_malloc(size_t size);
51 void	*__je_bootstrap_calloc(size_t num, size_t size);
52 void	__je_bootstrap_free(void *ptr);
53 
54 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls);
55 __weak_reference(__libc_free_tls, _rtld_free_tls);
56 
57 #ifdef __i386__
58 
59 __weak_reference(___libc_tls_get_addr, ___tls_get_addr);
60 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *);
61 
62 #endif
63 
64 void * __libc_tls_get_addr(void *);
65 __weak_reference(__libc_tls_get_addr, __tls_get_addr);
66 
67 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
68 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
69 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
70 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
71 
72 #if defined(__amd64__)
73 #define TLS_TCB_ALIGN 16
74 #elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \
75     defined(__mips__) || defined(__powerpc__) || defined(__riscv) || \
76     defined(__sparc64__)
77 #define TLS_TCB_ALIGN sizeof(void *)
78 #else
79 #error TLS_TCB_ALIGN undefined for target architecture
80 #endif
81 
82 #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \
83     defined(__powerpc__) || defined(__riscv)
84 #define TLS_VARIANT_I
85 #endif
86 #if defined(__i386__) || defined(__amd64__) || defined(__sparc64__)
87 #define TLS_VARIANT_II
88 #endif
89 
90 #ifndef PIC
91 
92 static size_t tls_static_space;
93 static size_t tls_init_size;
94 static size_t tls_init_align;
95 static void *tls_init;
96 #endif
97 
98 #ifdef __i386__
99 
100 /* GNU ABI */
101 
102 __attribute__((__regparm__(1)))
103 void *
104 ___libc_tls_get_addr(void *ti __unused)
105 {
106 	return (0);
107 }
108 
109 #endif
110 
111 void *
112 __libc_tls_get_addr(void *ti __unused)
113 {
114 	return (0);
115 }
116 
117 #ifndef PIC
118 
119 static void *
120 malloc_aligned(size_t size, size_t align)
121 {
122 	void *mem, *res;
123 
124 	if (align < sizeof(void *))
125 		align = sizeof(void *);
126 
127 	mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1);
128 	res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align);
129 	*(void **)((uintptr_t)res - sizeof(void *)) = mem;
130 	return (res);
131 }
132 
133 static void
134 free_aligned(void *ptr)
135 {
136 	void *mem;
137 	uintptr_t x;
138 
139 	if (ptr == NULL)
140 		return;
141 
142 	x = (uintptr_t)ptr;
143 	x -= sizeof(void *);
144 	mem = *(void **)x;
145 	__je_bootstrap_free(mem);
146 }
147 
148 #ifdef TLS_VARIANT_I
149 
150 #define	TLS_TCB_SIZE	(2 * sizeof(void *))
151 
152 /*
153  * Free Static TLS using the Variant I method.
154  */
155 void
156 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign __unused)
157 {
158 	Elf_Addr *dtv;
159 	Elf_Addr **tls;
160 
161 	tls = (Elf_Addr **)tcb;
162 	dtv = tls[0];
163 	__je_bootstrap_free(dtv);
164 	free_aligned(tls);
165 }
166 
167 /*
168  * Allocate Static TLS using the Variant I method.
169  */
170 void *
171 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign)
172 {
173 	Elf_Addr *dtv;
174 	Elf_Addr **tls;
175 
176 	if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE)
177 		return (oldtcb);
178 
179 	tls_assert(tcbalign >= TLS_TCB_ALIGN);
180 	tls_assert(tcbsize == TLS_TCB_SIZE);
181 
182 	tcbsize = roundup2(tcbsize, tcbalign);
183 	tls = malloc_aligned(tcbsize + tls_static_space, tcbalign);
184 	if (tls == NULL) {
185 		tls_msg("__libc_allocate_tls: Out of memory.\n");
186 		abort();
187 	}
188 	memset(tls, 0, tcbsize + tls_static_space);
189 
190 	if (oldtcb != NULL) {
191 		memcpy(tls, oldtcb, tcbsize + tls_static_space);
192 		__je_bootstrap_free(oldtcb);
193 
194 		/* Adjust the DTV. */
195 		dtv = tls[0];
196 		dtv[2] = (Elf_Addr)tls + tcbsize;
197 	} else {
198 		dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
199 		if (dtv == NULL) {
200 			tls_msg("__libc_allocate_tls: Out of memory.\n");
201 			abort();
202 		}
203 		tls[0] = dtv;
204 		dtv[0] = 1;		/* Generation. */
205 		dtv[1] = 1;		/* Segments count. */
206 		dtv[2] = (Elf_Addr)tls + tcbsize;
207 
208 		if (tls_init_size > 0)
209 			memcpy((void*)dtv[2], tls_init, tls_init_size);
210 	}
211 
212 	return (tls);
213 }
214 
215 #endif
216 
217 #ifdef TLS_VARIANT_II
218 
219 #define	TLS_TCB_SIZE	(3 * sizeof(Elf_Addr))
220 
221 /*
222  * Free Static TLS using the Variant II method.
223  */
224 void
225 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign)
226 {
227 	size_t size;
228 	Elf_Addr* dtv;
229 	Elf_Addr tlsstart, tlsend;
230 
231 	/*
232 	 * Figure out the size of the initial TLS block so that we can
233 	 * find stuff which ___tls_get_addr() allocated dynamically.
234 	 */
235 	size = roundup2(tls_static_space, tcbalign);
236 
237 	dtv = ((Elf_Addr**)tcb)[1];
238 	tlsend = (Elf_Addr) tcb;
239 	tlsstart = tlsend - size;
240 	free_aligned((void*)tlsstart);
241 	__je_bootstrap_free(dtv);
242 }
243 
244 /*
245  * Allocate Static TLS using the Variant II method.
246  */
247 void *
248 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign)
249 {
250 	size_t size;
251 	char *tls;
252 	Elf_Addr *dtv;
253 	Elf_Addr segbase, oldsegbase;
254 
255 	size = roundup2(tls_static_space, tcbalign);
256 
257 	if (tcbsize < 2 * sizeof(Elf_Addr))
258 		tcbsize = 2 * sizeof(Elf_Addr);
259 	tls = malloc_aligned(size + tcbsize, tcbalign);
260 	if (tls == NULL) {
261 		tls_msg("__libc_allocate_tls: Out of memory.\n");
262 		abort();
263 	}
264 	memset(tls, 0, size + tcbsize);
265 	dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
266 	if (dtv == NULL) {
267 		tls_msg("__libc_allocate_tls: Out of memory.\n");
268 		abort();
269 	}
270 
271 	segbase = (Elf_Addr)(tls + size);
272 	((Elf_Addr*)segbase)[0] = segbase;
273 	((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv;
274 
275 	dtv[0] = 1;
276 	dtv[1] = 1;
277 	dtv[2] = segbase - tls_static_space;
278 
279 	if (oldtls) {
280 		/*
281 		 * Copy the static TLS block over whole.
282 		 */
283 		oldsegbase = (Elf_Addr) oldtls;
284 		memcpy((void *)(segbase - tls_static_space),
285 		    (const void *)(oldsegbase - tls_static_space),
286 		    tls_static_space);
287 
288 		/*
289 		 * We assume that this block was the one we created with
290 		 * allocate_initial_tls().
291 		 */
292 		_rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr));
293 	} else {
294 		memcpy((void *)(segbase - tls_static_space),
295 		    tls_init, tls_init_size);
296 		memset((void *)(segbase - tls_static_space + tls_init_size),
297 		    0, tls_static_space - tls_init_size);
298 	}
299 
300 	return (void*) segbase;
301 }
302 
303 #endif /* TLS_VARIANT_II */
304 
305 #else
306 
307 void *
308 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused,
309 	size_t tcbalign __unused)
310 {
311 	return (0);
312 }
313 
314 void
315 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused,
316 	size_t tcbalign __unused)
317 {
318 }
319 
320 #endif /* PIC */
321 
322 extern char **environ;
323 
324 void
325 _init_tls(void)
326 {
327 #ifndef PIC
328 	Elf_Addr *sp;
329 	Elf_Auxinfo *aux, *auxp;
330 	Elf_Phdr *phdr;
331 	size_t phent, phnum;
332 	int i;
333 	void *tls;
334 
335 	sp = (Elf_Addr *) environ;
336 	while (*sp++ != 0)
337 		;
338 	aux = (Elf_Auxinfo *) sp;
339 	phdr = NULL;
340 	phent = phnum = 0;
341 	for (auxp = aux; auxp->a_type != AT_NULL; auxp++) {
342 		switch (auxp->a_type) {
343 		case AT_PHDR:
344 			phdr = auxp->a_un.a_ptr;
345 			break;
346 
347 		case AT_PHENT:
348 			phent = auxp->a_un.a_val;
349 			break;
350 
351 		case AT_PHNUM:
352 			phnum = auxp->a_un.a_val;
353 			break;
354 		}
355 	}
356 	if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0)
357 		return;
358 
359 	for (i = 0; (unsigned) i < phnum; i++) {
360 		if (phdr[i].p_type == PT_TLS) {
361 			tls_static_space = roundup2(phdr[i].p_memsz,
362 			    phdr[i].p_align);
363 			tls_init_size = phdr[i].p_filesz;
364 			tls_init_align = phdr[i].p_align;
365 			tls_init = (void*) phdr[i].p_vaddr;
366 			break;
367 		}
368 	}
369 
370 	tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE,
371 	    MAX(TLS_TCB_ALIGN, tls_init_align));
372 
373 	_set_tp(tls);
374 #endif
375 }
376