1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2006, David Xu <davidxu@freebsd.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * A lockless rwlock for rtld.
31 */
32 #include <sys/mman.h>
33 #include <sys/syscall.h>
34 #include <link.h>
35 #include <stdlib.h>
36 #include <string.h>
37
38 #include "libc_private.h"
39 #include "rtld_lock.h"
40 #include "thr_private.h"
41
42 extern int __libsys_errno;
43
44 static int _thr_rtld_clr_flag(int);
45 static void *_thr_rtld_lock_create(void);
46 static void _thr_rtld_lock_destroy(void *);
47 static void _thr_rtld_lock_release(void *);
48 static void _thr_rtld_rlock_acquire(void *);
49 static int _thr_rtld_set_flag(int);
50 static void _thr_rtld_wlock_acquire(void *);
51
52 struct rtld_lock {
53 struct urwlock lock;
54 struct pthread *wowner;
55 u_int rlocks;
56 char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock) -
57 sizeof(struct pthread *) - sizeof(u_int)];
58 };
59
60 static struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE);
61 static int busy_places;
62
63 static void *
_thr_rtld_lock_create(void)64 _thr_rtld_lock_create(void)
65 {
66 int locki;
67 struct rtld_lock *l;
68 static const char fail[] = "_thr_rtld_lock_create failed\n";
69
70 for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) {
71 if ((busy_places & (1 << locki)) == 0)
72 break;
73 }
74 if (locki == MAX_RTLD_LOCKS) {
75 write(2, fail, sizeof(fail) - 1);
76 return (NULL);
77 }
78 busy_places |= (1 << locki);
79
80 l = &lock_place[locki];
81 l->lock.rw_flags = URWLOCK_PREFER_READER;
82 return (l);
83 }
84
85 static void
_thr_rtld_lock_destroy(void * lock)86 _thr_rtld_lock_destroy(void *lock)
87 {
88 int locki;
89 size_t i;
90
91 locki = (struct rtld_lock *)lock - &lock_place[0];
92 for (i = 0; i < sizeof(struct rtld_lock); ++i)
93 ((char *)lock)[i] = 0;
94 busy_places &= ~(1 << locki);
95 }
96
97 #define SAVE_ERRNO() { \
98 if (curthread != _thr_initial) \
99 errsave = curthread->error; \
100 else \
101 errsave = __libsys_errno; \
102 }
103
104 #define RESTORE_ERRNO() { \
105 if (curthread != _thr_initial) \
106 curthread->error = errsave; \
107 else \
108 __libsys_errno = errsave; \
109 }
110
111 static void
_thr_rtld_rlock_acquire(void * lock)112 _thr_rtld_rlock_acquire(void *lock)
113 {
114 struct pthread *curthread;
115 struct rtld_lock *l;
116 int errsave;
117
118 curthread = _get_curthread();
119 SAVE_ERRNO();
120 l = (struct rtld_lock *)lock;
121
122 if (l->wowner == curthread) {
123 l->rlocks++;
124 } else {
125 THR_CRITICAL_ENTER(curthread);
126 while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0)
127 ;
128 }
129 curthread->rdlock_count++;
130 RESTORE_ERRNO();
131 }
132
133 static void
_thr_rtld_wlock_acquire(void * lock)134 _thr_rtld_wlock_acquire(void *lock)
135 {
136 struct pthread *curthread;
137 struct rtld_lock *l;
138 int errsave;
139
140 curthread = _get_curthread();
141 SAVE_ERRNO();
142 l = (struct rtld_lock *)lock;
143
144 THR_CRITICAL_ENTER(curthread);
145 while (_thr_rwlock_wrlock(&l->lock, NULL) != 0)
146 ;
147 l->wowner = curthread;
148 RESTORE_ERRNO();
149 }
150
151 static void
_thr_rtld_lock_release(void * lock)152 _thr_rtld_lock_release(void *lock)
153 {
154 struct pthread *curthread;
155 struct rtld_lock *l;
156 int32_t state;
157 int errsave;
158
159 curthread = _get_curthread();
160 SAVE_ERRNO();
161 l = (struct rtld_lock *)lock;
162
163 state = l->lock.rw_state;
164 if (__predict_false(_thr_after_fork)) {
165 /*
166 * After fork, only this thread is running, there is no
167 * waiters. Keeping waiters recorded in rwlock breaks
168 * wake logic.
169 */
170 atomic_clear_int(&l->lock.rw_state,
171 URWLOCK_WRITE_WAITERS | URWLOCK_READ_WAITERS);
172 l->lock.rw_blocked_readers = 0;
173 l->lock.rw_blocked_writers = 0;
174 }
175 if ((state & URWLOCK_WRITE_OWNER) != 0) {
176 if (l->rlocks > 0) {
177 l->rlocks--;
178 return;
179 } else {
180 l->wowner = NULL;
181 }
182 }
183 if (_thr_rwlock_unlock(&l->lock) == 0) {
184 if ((state & URWLOCK_WRITE_OWNER) == 0)
185 curthread->rdlock_count--;
186 THR_CRITICAL_LEAVE(curthread);
187 }
188 RESTORE_ERRNO();
189 }
190
191 static int
_thr_rtld_set_flag(int mask __unused)192 _thr_rtld_set_flag(int mask __unused)
193 {
194 /*
195 * The caller's code in rtld-elf is broken, it is not signal safe,
196 * just return zero to fool it.
197 */
198 return (0);
199 }
200
201 static int
_thr_rtld_clr_flag(int mask __unused)202 _thr_rtld_clr_flag(int mask __unused)
203 {
204 return (0);
205 }
206
207 /*
208 * ABI bug workaround: This symbol must be present for rtld to accept
209 * RTLI_VERSION from RtldLockInfo
210 */
211 extern char _pli_rtli_version;
212 char _pli_rtli_version;
213
214 static char *
_thr_dlerror_loc(void)215 _thr_dlerror_loc(void)
216 {
217 struct pthread *curthread;
218
219 curthread = _get_curthread();
220 return (curthread->dlerror_msg);
221 }
222
223 static int *
_thr_dlerror_seen(void)224 _thr_dlerror_seen(void)
225 {
226 struct pthread *curthread;
227
228 curthread = _get_curthread();
229 return (&curthread->dlerror_seen);
230 }
231
232 void
_thr_rtld_init(void)233 _thr_rtld_init(void)
234 {
235 struct RtldLockInfo li;
236 struct pthread *curthread;
237 ucontext_t *uc;
238 int uc_len;
239 char dummy[2] = {};
240
241 curthread = _get_curthread();
242
243 /* force to resolve _umtx_op PLT */
244 _umtx_op_err(&dummy, UMTX_OP_WAKE, 1, 0, 0);
245
246 /* force to resolve errno() PLT */
247 __error();
248
249 /* force to resolve memcpy PLT */
250 memcpy(&dummy[0], &dummy[1], 1);
251
252 mprotect(NULL, 0, 0);
253 _rtld_get_stack_prot();
254 thr_wake(-1);
255
256 li.rtli_version = RTLI_VERSION;
257 li.lock_create = _thr_rtld_lock_create;
258 li.lock_destroy = _thr_rtld_lock_destroy;
259 li.rlock_acquire = _thr_rtld_rlock_acquire;
260 li.wlock_acquire = _thr_rtld_wlock_acquire;
261 li.lock_release = _thr_rtld_lock_release;
262 li.thread_set_flag = _thr_rtld_set_flag;
263 li.thread_clr_flag = _thr_rtld_clr_flag;
264 li.at_fork = NULL;
265 li.dlerror_loc = _thr_dlerror_loc;
266 li.dlerror_loc_sz = sizeof(curthread->dlerror_msg);
267 li.dlerror_seen = _thr_dlerror_seen;
268
269 /*
270 * Preresolve the symbols needed for the fork interposer. We
271 * call _rtld_atfork_pre() and _rtld_atfork_post() with NULL
272 * argument to indicate that no actual locking inside the
273 * functions should happen. Neither rtld compat locks nor
274 * libthr rtld locks cannot work there:
275 * - compat locks do not handle the case of two locks taken
276 * in write mode (the signal mask for the thread is corrupted);
277 * - libthr locks would work, but locked rtld_bind_lock prevents
278 * symbol resolution for _rtld_atfork_post.
279 */
280 _rtld_atfork_pre(NULL);
281 _rtld_atfork_post(NULL);
282 _malloc_prefork();
283 _malloc_postfork();
284 getpid();
285 syscall(SYS_getpid);
286
287 /* mask signals, also force to resolve __sys_sigprocmask PLT */
288 _thr_signal_block(curthread);
289 _rtld_thread_init(&li);
290 _thr_signal_unblock(curthread);
291 _thr_signal_block_check_fast();
292 _thr_signal_block_setup(curthread);
293
294 /* resolve machine depended functions, if any */
295 _thr_resolve_machdep();
296
297 uc_len = __getcontextx_size();
298 uc = alloca(uc_len);
299 getcontext(uc);
300 __fillcontextx2((char *)uc);
301 }
302