1 /* 2 * Copyright (c) 2006, David Xu <davidxu@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 30 /* 31 * A lockless rwlock for rtld. 32 */ 33 #include <sys/cdefs.h> 34 #include <stdlib.h> 35 36 #include "rtld_lock.h" 37 #include "thr_private.h" 38 39 #undef errno 40 extern int errno; 41 42 #define CACHE_LINE_SIZE 64 43 #define WAFLAG 0x1 44 #define RC_INCR 0x2 45 46 static int _thr_rtld_clr_flag(int); 47 static void *_thr_rtld_lock_create(void); 48 static void _thr_rtld_lock_destroy(void *); 49 static void _thr_rtld_lock_release(void *); 50 static void _thr_rtld_rlock_acquire(void *); 51 static int _thr_rtld_set_flag(int); 52 static void _thr_rtld_wlock_acquire(void *); 53 54 struct rtld_lock { 55 volatile int lock; 56 volatile int rd_waiters; 57 volatile int wr_waiters; 58 volatile long rd_cv; 59 volatile long wr_cv; 60 void *base; 61 }; 62 63 static void * 64 _thr_rtld_lock_create(void) 65 { 66 void *base; 67 char *p; 68 uintptr_t r; 69 struct rtld_lock *l; 70 71 THR_ASSERT(sizeof(struct rtld_lock) <= CACHE_LINE_SIZE, 72 "rtld_lock too large"); 73 base = calloc(1, CACHE_LINE_SIZE); 74 p = (char *)base; 75 if ((uintptr_t)p % CACHE_LINE_SIZE != 0) { 76 free(base); 77 base = calloc(1, 2 * CACHE_LINE_SIZE); 78 p = (char *)base; 79 if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0) 80 p += CACHE_LINE_SIZE - r; 81 } 82 l = (struct rtld_lock *)p; 83 l->base = base; 84 return (l); 85 } 86 87 static void 88 _thr_rtld_lock_destroy(void *lock) 89 { 90 struct rtld_lock *l = (struct rtld_lock *)lock; 91 free(l->base); 92 } 93 94 #define SAVE_ERRNO() { \ 95 if (curthread != _thr_initial) \ 96 errsave = curthread->error; \ 97 else \ 98 errsave = errno; \ 99 } 100 101 #define RESTORE_ERRNO() { \ 102 if (curthread != _thr_initial) \ 103 curthread->error = errsave; \ 104 else \ 105 errno = errsave; \ 106 } 107 108 static void 109 _thr_rtld_rlock_acquire(void *lock) 110 { 111 struct pthread *curthread; 112 struct rtld_lock *l; 113 long v; 114 int errsave; 115 116 curthread = _get_curthread(); 117 SAVE_ERRNO(); 118 l = (struct rtld_lock *)lock; 119 120 THR_CRITICAL_ENTER(curthread); 121 atomic_add_acq_int(&l->lock, RC_INCR); 122 if (!(l->lock & WAFLAG)) { 123 RESTORE_ERRNO(); 124 return; 125 } 126 v = l->rd_cv; 127 atomic_add_int(&l->rd_waiters, 1); 128 while (l->lock & WAFLAG) { 129 _thr_umtx_wait(&l->rd_cv, v, NULL); 130 v = l->rd_cv; 131 } 132 atomic_add_int(&l->rd_waiters, -1); 133 RESTORE_ERRNO(); 134 } 135 136 static void 137 _thr_rtld_wlock_acquire(void *lock) 138 { 139 struct pthread *curthread; 140 struct rtld_lock *l; 141 long v; 142 int errsave; 143 144 curthread = _get_curthread(); 145 SAVE_ERRNO(); 146 l = (struct rtld_lock *)lock; 147 148 _thr_signal_block(curthread); 149 for (;;) { 150 if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) { 151 RESTORE_ERRNO(); 152 return; 153 } 154 v = l->wr_cv; 155 atomic_add_int(&l->wr_waiters, 1); 156 while (l->lock != 0) { 157 _thr_umtx_wait(&l->wr_cv, v, NULL); 158 v = l->wr_cv; 159 } 160 atomic_add_int(&l->wr_waiters, -1); 161 } 162 } 163 164 static void 165 _thr_rtld_lock_release(void *lock) 166 { 167 struct pthread *curthread; 168 struct rtld_lock *l; 169 int errsave; 170 171 curthread = _get_curthread(); 172 SAVE_ERRNO(); 173 l = (struct rtld_lock *)lock; 174 175 if ((l->lock & WAFLAG) == 0) { 176 atomic_add_rel_int(&l->lock, -RC_INCR); 177 if (l->lock == 0 && l->wr_waiters) { 178 atomic_add_long(&l->wr_cv, 1); 179 _thr_umtx_wake(&l->wr_cv, l->wr_waiters); 180 } 181 THR_CRITICAL_LEAVE(curthread); 182 } else { 183 atomic_add_rel_int(&l->lock, -WAFLAG); 184 if (l->lock == 0 && l->wr_waiters) { 185 atomic_add_long(&l->wr_cv, 1); 186 _thr_umtx_wake(&l->wr_cv, l->wr_waiters); 187 } else if (l->rd_waiters) { 188 atomic_add_long(&l->rd_cv, 1); 189 _thr_umtx_wake(&l->rd_cv, l->rd_waiters); 190 } 191 _thr_signal_unblock(curthread); 192 } 193 RESTORE_ERRNO(); 194 } 195 196 static int 197 _thr_rtld_set_flag(int mask __unused) 198 { 199 /* 200 * The caller's code in rtld-elf is broken, it is not signal safe, 201 * just return zero to fool it. 202 */ 203 return (0); 204 } 205 206 static int 207 _thr_rtld_clr_flag(int mask __unused) 208 { 209 return (0); 210 } 211 212 void 213 _thr_rtld_init(void) 214 { 215 struct RtldLockInfo li; 216 struct pthread *curthread; 217 long dummy; 218 219 curthread = _get_curthread(); 220 221 /* force to resolve _umtx_op PLT */ 222 _umtx_op((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0); 223 224 /* force to resolve errno() PLT */ 225 __error(); 226 227 li.lock_create = _thr_rtld_lock_create; 228 li.lock_destroy = _thr_rtld_lock_destroy; 229 li.rlock_acquire = _thr_rtld_rlock_acquire; 230 li.wlock_acquire = _thr_rtld_wlock_acquire; 231 li.lock_release = _thr_rtld_lock_release; 232 li.thread_set_flag = _thr_rtld_set_flag; 233 li.thread_clr_flag = _thr_rtld_clr_flag; 234 li.at_fork = NULL; 235 236 /* mask signals, also force to resolve __sys_sigprocmask PLT */ 237 _thr_signal_block(curthread); 238 _rtld_thread_init(&li); 239 _thr_signal_unblock(curthread); 240 } 241 242 void 243 _thr_rtld_fini(void) 244 { 245 struct pthread *curthread; 246 247 curthread = _get_curthread(); 248 _thr_signal_block(curthread); 249 _rtld_thread_init(NULL); 250 _thr_signal_unblock(curthread); 251 } 252