1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2006, David Xu <davidxu@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * A lockless rwlock for rtld. 31 */ 32 #include <sys/mman.h> 33 #include <sys/syscall.h> 34 #include <link.h> 35 #include <stdlib.h> 36 #include <string.h> 37 38 #include "libc_private.h" 39 #include "rtld_lock.h" 40 #include "thr_private.h" 41 42 extern int __libsys_errno; 43 44 static int _thr_rtld_clr_flag(int); 45 static void *_thr_rtld_lock_create(void); 46 static void _thr_rtld_lock_destroy(void *); 47 static void _thr_rtld_lock_release(void *); 48 static void _thr_rtld_rlock_acquire(void *); 49 static int _thr_rtld_set_flag(int); 50 static void _thr_rtld_wlock_acquire(void *); 51 52 struct rtld_lock { 53 struct urwlock lock; 54 char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)]; 55 }; 56 57 static struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE); 58 static int busy_places; 59 60 static void * 61 _thr_rtld_lock_create(void) 62 { 63 int locki; 64 struct rtld_lock *l; 65 static const char fail[] = "_thr_rtld_lock_create failed\n"; 66 67 for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) { 68 if ((busy_places & (1 << locki)) == 0) 69 break; 70 } 71 if (locki == MAX_RTLD_LOCKS) { 72 write(2, fail, sizeof(fail) - 1); 73 return (NULL); 74 } 75 busy_places |= (1 << locki); 76 77 l = &lock_place[locki]; 78 l->lock.rw_flags = URWLOCK_PREFER_READER; 79 return (l); 80 } 81 82 static void 83 _thr_rtld_lock_destroy(void *lock) 84 { 85 int locki; 86 size_t i; 87 88 locki = (struct rtld_lock *)lock - &lock_place[0]; 89 for (i = 0; i < sizeof(struct rtld_lock); ++i) 90 ((char *)lock)[i] = 0; 91 busy_places &= ~(1 << locki); 92 } 93 94 #define SAVE_ERRNO() { \ 95 if (curthread != _thr_initial) \ 96 errsave = curthread->error; \ 97 else \ 98 errsave = __libsys_errno; \ 99 } 100 101 #define RESTORE_ERRNO() { \ 102 if (curthread != _thr_initial) \ 103 curthread->error = errsave; \ 104 else \ 105 __libsys_errno = errsave; \ 106 } 107 108 static void 109 _thr_rtld_rlock_acquire(void *lock) 110 { 111 struct pthread *curthread; 112 struct rtld_lock *l; 113 int errsave; 114 115 curthread = _get_curthread(); 116 SAVE_ERRNO(); 117 l = (struct rtld_lock *)lock; 118 119 THR_CRITICAL_ENTER(curthread); 120 while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0) 121 ; 122 curthread->rdlock_count++; 123 RESTORE_ERRNO(); 124 } 125 126 static void 127 _thr_rtld_wlock_acquire(void *lock) 128 { 129 struct pthread *curthread; 130 struct rtld_lock *l; 131 int errsave; 132 133 curthread = _get_curthread(); 134 SAVE_ERRNO(); 135 l = (struct rtld_lock *)lock; 136 137 THR_CRITICAL_ENTER(curthread); 138 while (_thr_rwlock_wrlock(&l->lock, NULL) != 0) 139 ; 140 RESTORE_ERRNO(); 141 } 142 143 static void 144 _thr_rtld_lock_release(void *lock) 145 { 146 struct pthread *curthread; 147 struct rtld_lock *l; 148 int32_t state; 149 int errsave; 150 151 curthread = _get_curthread(); 152 SAVE_ERRNO(); 153 l = (struct rtld_lock *)lock; 154 155 state = l->lock.rw_state; 156 if (__predict_false(_thr_after_fork)) { 157 /* 158 * After fork, only this thread is running, there is no 159 * waiters. Keeping waiters recorded in rwlock breaks 160 * wake logic. 161 */ 162 atomic_clear_int(&l->lock.rw_state, 163 URWLOCK_WRITE_WAITERS | URWLOCK_READ_WAITERS); 164 l->lock.rw_blocked_readers = 0; 165 l->lock.rw_blocked_writers = 0; 166 } 167 if (_thr_rwlock_unlock(&l->lock) == 0) { 168 if ((state & URWLOCK_WRITE_OWNER) == 0) 169 curthread->rdlock_count--; 170 THR_CRITICAL_LEAVE(curthread); 171 } 172 RESTORE_ERRNO(); 173 } 174 175 static int 176 _thr_rtld_set_flag(int mask __unused) 177 { 178 /* 179 * The caller's code in rtld-elf is broken, it is not signal safe, 180 * just return zero to fool it. 181 */ 182 return (0); 183 } 184 185 static int 186 _thr_rtld_clr_flag(int mask __unused) 187 { 188 return (0); 189 } 190 191 /* 192 * ABI bug workaround: This symbol must be present for rtld to accept 193 * RTLI_VERSION from RtldLockInfo 194 */ 195 extern char _pli_rtli_version; 196 char _pli_rtli_version; 197 198 static char * 199 _thr_dlerror_loc(void) 200 { 201 struct pthread *curthread; 202 203 curthread = _get_curthread(); 204 return (curthread->dlerror_msg); 205 } 206 207 static int * 208 _thr_dlerror_seen(void) 209 { 210 struct pthread *curthread; 211 212 curthread = _get_curthread(); 213 return (&curthread->dlerror_seen); 214 } 215 216 void 217 _thr_rtld_init(void) 218 { 219 struct RtldLockInfo li; 220 struct pthread *curthread; 221 ucontext_t *uc; 222 int uc_len; 223 char dummy[2] = {}; 224 225 curthread = _get_curthread(); 226 227 /* force to resolve _umtx_op PLT */ 228 _umtx_op_err(&dummy, UMTX_OP_WAKE, 1, 0, 0); 229 230 /* force to resolve errno() PLT */ 231 __error(); 232 233 /* force to resolve memcpy PLT */ 234 memcpy(&dummy[0], &dummy[1], 1); 235 236 mprotect(NULL, 0, 0); 237 _rtld_get_stack_prot(); 238 thr_wake(-1); 239 240 li.rtli_version = RTLI_VERSION; 241 li.lock_create = _thr_rtld_lock_create; 242 li.lock_destroy = _thr_rtld_lock_destroy; 243 li.rlock_acquire = _thr_rtld_rlock_acquire; 244 li.wlock_acquire = _thr_rtld_wlock_acquire; 245 li.lock_release = _thr_rtld_lock_release; 246 li.thread_set_flag = _thr_rtld_set_flag; 247 li.thread_clr_flag = _thr_rtld_clr_flag; 248 li.at_fork = NULL; 249 li.dlerror_loc = _thr_dlerror_loc; 250 li.dlerror_loc_sz = sizeof(curthread->dlerror_msg); 251 li.dlerror_seen = _thr_dlerror_seen; 252 253 /* 254 * Preresolve the symbols needed for the fork interposer. We 255 * call _rtld_atfork_pre() and _rtld_atfork_post() with NULL 256 * argument to indicate that no actual locking inside the 257 * functions should happen. Neither rtld compat locks nor 258 * libthr rtld locks cannot work there: 259 * - compat locks do not handle the case of two locks taken 260 * in write mode (the signal mask for the thread is corrupted); 261 * - libthr locks would work, but locked rtld_bind_lock prevents 262 * symbol resolution for _rtld_atfork_post. 263 */ 264 _rtld_atfork_pre(NULL); 265 _rtld_atfork_post(NULL); 266 _malloc_prefork(); 267 _malloc_postfork(); 268 getpid(); 269 syscall(SYS_getpid); 270 271 /* mask signals, also force to resolve __sys_sigprocmask PLT */ 272 _thr_signal_block(curthread); 273 _rtld_thread_init(&li); 274 _thr_signal_unblock(curthread); 275 _thr_signal_block_check_fast(); 276 _thr_signal_block_setup(curthread); 277 278 /* resolve machine depended functions, if any */ 279 _thr_resolve_machdep(); 280 281 uc_len = __getcontextx_size(); 282 uc = alloca(uc_len); 283 getcontext(uc); 284 __fillcontextx2((char *)uc); 285 } 286