1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright 1999, 2000 John D. Polstra.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 * from: FreeBSD: src/libexec/rtld-elf/sparc64/lockdflt.c,v 1.3 2002/10/09
28 */
29
30 /*
31 * Thread locking implementation for the dynamic linker.
32 *
33 * We use the "simple, non-scalable reader-preference lock" from:
34 *
35 * J. M. Mellor-Crummey and M. L. Scott. "Scalable Reader-Writer
36 * Synchronization for Shared-Memory Multiprocessors." 3rd ACM Symp. on
37 * Principles and Practice of Parallel Programming, April 1991.
38 *
39 * In this algorithm the lock is a single word. Its low-order bit is
40 * set when a writer holds the lock. The remaining high-order bits
41 * contain a count of readers desiring the lock. The algorithm requires
42 * atomic "compare_and_store" and "add" operations, which we take
43 * from machine/atomic.h.
44 */
45
46 #include <sys/param.h>
47 #include <sys/signalvar.h>
48 #include <signal.h>
49 #include <stdlib.h>
50 #include <time.h>
51
52 #include "debug.h"
53 #include "rtld.h"
54 #include "rtld_machdep.h"
55 #include "rtld_libc.h"
56
57 void _rtld_thread_init(struct RtldLockInfo *) __exported;
58 void _rtld_atfork_pre(int *) __exported;
59 void _rtld_atfork_post(int *) __exported;
60
61 static char def_dlerror_msg[512];
62 static int def_dlerror_seen_val = 1;
63
64 static char *
def_dlerror_loc(void)65 def_dlerror_loc(void)
66 {
67 return (def_dlerror_msg);
68 }
69
70 static int *
def_dlerror_seen(void)71 def_dlerror_seen(void)
72 {
73 return (&def_dlerror_seen_val);
74 }
75
76 #define WAFLAG 0x1 /* A writer holds the lock */
77 #define RC_INCR 0x2 /* Adjusts count of readers desiring lock */
78
79 typedef struct Struct_Lock {
80 volatile u_int lock;
81 void *base;
82 } Lock;
83
84 static sigset_t fullsigmask, oldsigmask;
85 static int thread_flag, wnested;
86 static uint32_t fsigblock;
87
88 static void *
def_lock_create(void)89 def_lock_create(void)
90 {
91 void *base;
92 char *p;
93 uintptr_t r;
94 Lock *l;
95
96 /*
97 * Arrange for the lock to occupy its own cache line. First, we
98 * optimistically allocate just a cache line, hoping that malloc
99 * will give us a well-aligned block of memory. If that doesn't
100 * work, we allocate a larger block and take a well-aligned cache
101 * line from it.
102 */
103 base = xmalloc(CACHE_LINE_SIZE);
104 p = base;
105 if ((uintptr_t)p % CACHE_LINE_SIZE != 0) {
106 free(base);
107 base = xmalloc(2 * CACHE_LINE_SIZE);
108 p = base;
109 if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0)
110 p += CACHE_LINE_SIZE - r;
111 }
112 l = (Lock *)p;
113 l->base = base;
114 l->lock = 0;
115 return (l);
116 }
117
118 static void
def_lock_destroy(void * lock)119 def_lock_destroy(void *lock)
120 {
121 Lock *l = lock;
122
123 free(l->base);
124 }
125
126 static void
sig_fastunblock(void)127 sig_fastunblock(void)
128 {
129 uint32_t oldval;
130
131 assert((fsigblock & ~SIGFASTBLOCK_FLAGS) >= SIGFASTBLOCK_INC);
132 oldval = atomic_fetchadd_32(&fsigblock, -SIGFASTBLOCK_INC);
133 if (oldval == (SIGFASTBLOCK_PEND | SIGFASTBLOCK_INC))
134 __sys_sigfastblock(SIGFASTBLOCK_UNBLOCK, NULL);
135 }
136
137 static bool
def_lock_acquire_set(Lock * l,bool wlock)138 def_lock_acquire_set(Lock *l, bool wlock)
139 {
140 if (wlock) {
141 if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG))
142 return (true);
143 } else {
144 atomic_add_acq_int(&l->lock, RC_INCR);
145 if ((l->lock & WAFLAG) == 0)
146 return (true);
147 atomic_add_int(&l->lock, -RC_INCR);
148 }
149 return (false);
150 }
151
152 static void
def_lock_acquire(Lock * l,bool wlock)153 def_lock_acquire(Lock *l, bool wlock)
154 {
155 sigset_t tmp_oldsigmask;
156
157 if (ld_fast_sigblock) {
158 for (;;) {
159 atomic_add_32(&fsigblock, SIGFASTBLOCK_INC);
160 if (def_lock_acquire_set(l, wlock))
161 break;
162 sig_fastunblock();
163 }
164 } else {
165 for (;;) {
166 sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask);
167 if (def_lock_acquire_set(l, wlock))
168 break;
169 sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
170 }
171 if (atomic_fetchadd_int(&wnested, 1) == 0)
172 oldsigmask = tmp_oldsigmask;
173 }
174 }
175
176 static void
def_rlock_acquire(void * lock)177 def_rlock_acquire(void *lock)
178 {
179 def_lock_acquire(lock, false);
180 }
181
182 static void
def_wlock_acquire(void * lock)183 def_wlock_acquire(void *lock)
184 {
185 def_lock_acquire(lock, true);
186 }
187
188 static void
def_lock_release(void * lock)189 def_lock_release(void *lock)
190 {
191 Lock *l = lock;
192
193 atomic_add_rel_int(&l->lock, -((l->lock & WAFLAG) == 0 ?
194 RC_INCR : WAFLAG));
195 if (ld_fast_sigblock)
196 sig_fastunblock();
197 else if (atomic_fetchadd_int(&wnested, -1) == 1)
198 sigprocmask(SIG_SETMASK, &oldsigmask, NULL);
199 }
200
201 static int
def_thread_set_flag(int mask)202 def_thread_set_flag(int mask)
203 {
204 int old_val = thread_flag;
205
206 thread_flag |= mask;
207 return (old_val);
208 }
209
210 static int
def_thread_clr_flag(int mask)211 def_thread_clr_flag(int mask)
212 {
213 int old_val = thread_flag;
214
215 thread_flag &= ~mask;
216 return (old_val);
217 }
218
219 /*
220 * Public interface exposed to the rest of the dynamic linker.
221 */
222 struct RtldLockInfo lockinfo;
223 static struct RtldLockInfo deflockinfo;
224
225 static __inline int
thread_mask_set(int mask)226 thread_mask_set(int mask)
227 {
228 return (lockinfo.thread_set_flag(mask));
229 }
230
231 static __inline void
thread_mask_clear(int mask)232 thread_mask_clear(int mask)
233 {
234 lockinfo.thread_clr_flag(mask);
235 }
236
237 #define RTLD_LOCK_CNT 3
238 static struct rtld_lock {
239 void *handle;
240 int mask;
241 } rtld_locks[RTLD_LOCK_CNT];
242
243 rtld_lock_t rtld_bind_lock = &rtld_locks[0];
244 rtld_lock_t rtld_libc_lock = &rtld_locks[1];
245 rtld_lock_t rtld_phdr_lock = &rtld_locks[2];
246
247 void
rlock_acquire(rtld_lock_t lock,RtldLockState * lockstate)248 rlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
249 {
250
251 if (lockstate == NULL)
252 return;
253
254 if ((thread_mask_set(lock->mask) & lock->mask) != 0) {
255 dbg("rlock_acquire: recursed");
256 lockstate->lockstate = RTLD_LOCK_UNLOCKED;
257 return;
258 }
259 lockinfo.rlock_acquire(lock->handle);
260 lockstate->lockstate = RTLD_LOCK_RLOCKED;
261 }
262
263 void
wlock_acquire(rtld_lock_t lock,RtldLockState * lockstate)264 wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
265 {
266
267 if (lockstate == NULL)
268 return;
269
270 if ((thread_mask_set(lock->mask) & lock->mask) != 0) {
271 dbg("wlock_acquire: recursed");
272 lockstate->lockstate = RTLD_LOCK_UNLOCKED;
273 return;
274 }
275 lockinfo.wlock_acquire(lock->handle);
276 lockstate->lockstate = RTLD_LOCK_WLOCKED;
277 }
278
279 void
lock_release(rtld_lock_t lock,RtldLockState * lockstate)280 lock_release(rtld_lock_t lock, RtldLockState *lockstate)
281 {
282
283 if (lockstate == NULL)
284 return;
285
286 switch (lockstate->lockstate) {
287 case RTLD_LOCK_UNLOCKED:
288 break;
289 case RTLD_LOCK_RLOCKED:
290 case RTLD_LOCK_WLOCKED:
291 thread_mask_clear(lock->mask);
292 lockinfo.lock_release(lock->handle);
293 break;
294 default:
295 assert(0);
296 }
297 }
298
299 void
lock_upgrade(rtld_lock_t lock,RtldLockState * lockstate)300 lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate)
301 {
302
303 if (lockstate == NULL)
304 return;
305
306 lock_release(lock, lockstate);
307 wlock_acquire(lock, lockstate);
308 }
309
310 void
lock_restart_for_upgrade(RtldLockState * lockstate)311 lock_restart_for_upgrade(RtldLockState *lockstate)
312 {
313
314 if (lockstate == NULL)
315 return;
316
317 switch (lockstate->lockstate) {
318 case RTLD_LOCK_UNLOCKED:
319 case RTLD_LOCK_WLOCKED:
320 break;
321 case RTLD_LOCK_RLOCKED:
322 siglongjmp(lockstate->env, 1);
323 break;
324 default:
325 assert(0);
326 }
327 }
328
329 bool
lockstate_wlocked(const RtldLockState * lockstate)330 lockstate_wlocked(const RtldLockState *lockstate)
331 {
332 return (lockstate->lockstate == RTLD_LOCK_WLOCKED);
333 }
334
335 void
dlerror_dflt_init(void)336 dlerror_dflt_init(void)
337 {
338 lockinfo.dlerror_loc = def_dlerror_loc;
339 lockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg);
340 lockinfo.dlerror_seen = def_dlerror_seen;
341 }
342
343 void
lockdflt_init(void)344 lockdflt_init(void)
345 {
346 int i;
347
348 deflockinfo.rtli_version = RTLI_VERSION;
349 deflockinfo.lock_create = def_lock_create;
350 deflockinfo.lock_destroy = def_lock_destroy;
351 deflockinfo.rlock_acquire = def_rlock_acquire;
352 deflockinfo.wlock_acquire = def_wlock_acquire;
353 deflockinfo.lock_release = def_lock_release;
354 deflockinfo.thread_set_flag = def_thread_set_flag;
355 deflockinfo.thread_clr_flag = def_thread_clr_flag;
356 deflockinfo.at_fork = NULL;
357 deflockinfo.dlerror_loc = def_dlerror_loc;
358 deflockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg);
359 deflockinfo.dlerror_seen = def_dlerror_seen;
360
361 for (i = 0; i < RTLD_LOCK_CNT; i++) {
362 rtld_locks[i].mask = (1 << i);
363 rtld_locks[i].handle = NULL;
364 }
365
366 memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo));
367 _rtld_thread_init(NULL);
368 if (ld_fast_sigblock) {
369 __sys_sigfastblock(SIGFASTBLOCK_SETPTR, &fsigblock);
370 } else {
371 /*
372 * Construct a mask to block all signals. Note that
373 * blocked traps mean that the process is terminated
374 * if trap occurs while we are in locked section, with
375 * the default settings for kern.forcesigexit.
376 */
377 sigfillset(&fullsigmask);
378 }
379 }
380
381 /*
382 * Callback function to allow threads implementation to
383 * register their own locking primitives if the default
384 * one is not suitable.
385 * The current context should be the only context
386 * executing at the invocation time.
387 */
388 void
_rtld_thread_init(struct RtldLockInfo * pli)389 _rtld_thread_init(struct RtldLockInfo *pli)
390 {
391 const Obj_Entry *obj;
392 SymLook req;
393 void *locks[RTLD_LOCK_CNT];
394 int flags, i, res;
395
396 if (pli == NULL) {
397 lockinfo.rtli_version = RTLI_VERSION;
398 } else {
399 lockinfo.rtli_version = RTLI_VERSION_ONE;
400 obj = obj_from_addr(pli->lock_create);
401 if (obj != NULL) {
402 symlook_init(&req, "_pli_rtli_version");
403 res = symlook_obj(&req, obj);
404 if (res == 0)
405 lockinfo.rtli_version = pli->rtli_version;
406 }
407 }
408
409 /* disable all locking while this function is running */
410 flags = thread_mask_set(~0);
411
412 if (pli == NULL)
413 pli = &deflockinfo;
414 else if (ld_fast_sigblock) {
415 fsigblock = 0;
416 __sys_sigfastblock(SIGFASTBLOCK_UNSETPTR, NULL);
417 }
418
419 for (i = 0; i < RTLD_LOCK_CNT; i++)
420 if ((locks[i] = pli->lock_create()) == NULL)
421 break;
422
423 if (i < RTLD_LOCK_CNT) {
424 while (--i >= 0)
425 pli->lock_destroy(locks[i]);
426 abort();
427 }
428
429 for (i = 0; i < RTLD_LOCK_CNT; i++) {
430 if (rtld_locks[i].handle == NULL)
431 continue;
432 if (flags & rtld_locks[i].mask)
433 lockinfo.lock_release(rtld_locks[i].handle);
434 lockinfo.lock_destroy(rtld_locks[i].handle);
435 }
436
437 for (i = 0; i < RTLD_LOCK_CNT; i++) {
438 rtld_locks[i].handle = locks[i];
439 if (flags & rtld_locks[i].mask)
440 pli->wlock_acquire(rtld_locks[i].handle);
441 }
442
443 lockinfo.lock_create = pli->lock_create;
444 lockinfo.lock_destroy = pli->lock_destroy;
445 lockinfo.rlock_acquire = pli->rlock_acquire;
446 lockinfo.wlock_acquire = pli->wlock_acquire;
447 lockinfo.lock_release = pli->lock_release;
448 lockinfo.thread_set_flag = pli->thread_set_flag;
449 lockinfo.thread_clr_flag = pli->thread_clr_flag;
450 lockinfo.at_fork = pli->at_fork;
451 if (lockinfo.rtli_version > RTLI_VERSION_ONE && pli != NULL) {
452 strlcpy(pli->dlerror_loc(), lockinfo.dlerror_loc(),
453 lockinfo.dlerror_loc_sz);
454 lockinfo.dlerror_loc = pli->dlerror_loc;
455 lockinfo.dlerror_loc_sz = pli->dlerror_loc_sz;
456 lockinfo.dlerror_seen = pli->dlerror_seen;
457 }
458
459 /* restore thread locking state, this time with new locks */
460 thread_mask_clear(~0);
461 thread_mask_set(flags);
462 dbg("_rtld_thread_init: done");
463 }
464
465 void
_rtld_atfork_pre(int * locks)466 _rtld_atfork_pre(int *locks)
467 {
468 RtldLockState ls[2];
469
470 if (locks == NULL)
471 return;
472 bzero(ls, sizeof(ls));
473
474 /*
475 * Warning: this did not worked well with the rtld compat
476 * locks above, when the thread signal mask was corrupted (set
477 * to all signals blocked) if two locks were taken
478 * simultaneously in the write mode. The caller of the
479 * _rtld_atfork_pre() must provide the working implementation
480 * of the locks anyway, and libthr locks are fine.
481 */
482 if (ld_get_env_var(LD_NO_DL_ITERATE_PHDR_AFTER_FORK) == NULL)
483 wlock_acquire(rtld_phdr_lock, &ls[0]);
484 wlock_acquire(rtld_bind_lock, &ls[1]);
485
486 /* XXXKIB: I am really sorry for this. */
487 locks[0] = ls[1].lockstate;
488 locks[2] = ls[0].lockstate;
489 }
490
491 void
_rtld_atfork_post(int * locks)492 _rtld_atfork_post(int *locks)
493 {
494 RtldLockState ls[2];
495
496 if (locks == NULL)
497 return;
498
499 bzero(ls, sizeof(ls));
500 ls[0].lockstate = locks[2];
501 ls[1].lockstate = locks[0];
502 lock_release(rtld_bind_lock, &ls[1]);
503 if (ld_get_env_var(LD_NO_DL_ITERATE_PHDR_AFTER_FORK) == NULL)
504 lock_release(rtld_phdr_lock, &ls[0]);
505 }
506