1 /* 2 * Copyright (c) 2006, David Xu <davidxu@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 30 /* 31 * A lockless rwlock for rtld. 32 */ 33 #include <sys/cdefs.h> 34 #include <stdlib.h> 35 36 #include "rtld_lock.h" 37 #include "thr_private.h" 38 39 #define CACHE_LINE_SIZE 64 40 #define WAFLAG 0x1 41 #define RC_INCR 0x2 42 43 static int _thr_rtld_clr_flag(int); 44 static void *_thr_rtld_lock_create(void); 45 static void _thr_rtld_lock_destroy(void *); 46 static void _thr_rtld_lock_release(void *); 47 static void _thr_rtld_rlock_acquire(void *); 48 static int _thr_rtld_set_flag(int); 49 static void _thr_rtld_wlock_acquire(void *); 50 51 struct rtld_lock { 52 volatile int lock; 53 volatile int rd_waiters; 54 volatile int wr_waiters; 55 volatile long rd_cv; 56 volatile long wr_cv; 57 void *base; 58 }; 59 60 static void * 61 _thr_rtld_lock_create(void) 62 { 63 void *base; 64 char *p; 65 uintptr_t r; 66 struct rtld_lock *l; 67 68 THR_ASSERT(sizeof(struct rtld_lock) <= CACHE_LINE_SIZE, 69 "rtld_lock too large"); 70 base = calloc(1, CACHE_LINE_SIZE); 71 p = (char *)base; 72 if ((uintptr_t)p % CACHE_LINE_SIZE != 0) { 73 free(base); 74 base = calloc(1, 2 * CACHE_LINE_SIZE); 75 p = (char *)base; 76 if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0) 77 p += CACHE_LINE_SIZE - r; 78 } 79 l = (struct rtld_lock *)p; 80 l->base = base; 81 return (l); 82 } 83 84 static void 85 _thr_rtld_lock_destroy(void *lock) 86 { 87 struct rtld_lock *l = (struct rtld_lock *)lock; 88 free(l->base); 89 } 90 91 static void 92 _thr_rtld_rlock_acquire(void *lock) 93 { 94 struct pthread *curthread; 95 struct rtld_lock *l; 96 long v; 97 98 curthread = _get_curthread(); 99 l = (struct rtld_lock *)lock; 100 101 THR_CRITICAL_ENTER(curthread); 102 atomic_add_acq_int(&l->lock, RC_INCR); 103 if (!(l->lock & WAFLAG)) 104 return; 105 v = l->rd_cv; 106 atomic_add_int(&l->rd_waiters, 1); 107 while (l->lock & WAFLAG) { 108 _thr_umtx_wait(&l->rd_cv, v, NULL); 109 v = l->rd_cv; 110 } 111 atomic_add_int(&l->rd_waiters, -1); 112 } 113 114 static void 115 _thr_rtld_wlock_acquire(void *lock) 116 { 117 struct pthread *curthread; 118 struct rtld_lock *l; 119 long v; 120 121 curthread = _get_curthread(); 122 l = (struct rtld_lock *)lock; 123 124 _thr_signal_block(curthread); 125 for (;;) { 126 if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) 127 return; 128 v = l->wr_cv; 129 atomic_add_int(&l->wr_waiters, 1); 130 while (l->lock != 0) { 131 _thr_umtx_wait(&l->wr_cv, v, NULL); 132 v = l->wr_cv; 133 } 134 atomic_add_int(&l->wr_waiters, -1); 135 } 136 } 137 138 static void 139 _thr_rtld_lock_release(void *lock) 140 { 141 struct pthread *curthread; 142 struct rtld_lock *l; 143 144 curthread = _get_curthread(); 145 l = (struct rtld_lock *)lock; 146 147 if ((l->lock & WAFLAG) == 0) { 148 atomic_add_rel_int(&l->lock, -RC_INCR); 149 if (l->lock == 0 && l->wr_waiters) { 150 atomic_add_long(&l->wr_cv, 1); 151 _thr_umtx_wake(&l->wr_cv, l->wr_waiters); 152 } 153 THR_CRITICAL_LEAVE(curthread); 154 } else { 155 atomic_add_rel_int(&l->lock, -WAFLAG); 156 if (l->lock == 0 && l->wr_waiters) { 157 atomic_add_long(&l->wr_cv, 1); 158 _thr_umtx_wake(&l->wr_cv, l->wr_waiters); 159 } else if (l->rd_waiters) { 160 atomic_add_long(&l->rd_cv, 1); 161 _thr_umtx_wake(&l->rd_cv, l->rd_waiters); 162 } 163 _thr_signal_unblock(curthread); 164 } 165 } 166 167 static int 168 _thr_rtld_set_flag(int mask __unused) 169 { 170 /* 171 * The caller's code in rtld-elf is broken, it is not signal safe, 172 * just return zero to fool it. 173 */ 174 return (0); 175 } 176 177 static int 178 _thr_rtld_clr_flag(int mask __unused) 179 { 180 return (0); 181 } 182 183 void 184 _thr_rtld_init(void) 185 { 186 struct RtldLockInfo li; 187 struct pthread *curthread; 188 long dummy; 189 190 curthread = _get_curthread(); 191 192 /* force to resolve _umtx_op PLT */ 193 _umtx_op((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0); 194 195 li.lock_create = _thr_rtld_lock_create; 196 li.lock_destroy = _thr_rtld_lock_destroy; 197 li.rlock_acquire = _thr_rtld_rlock_acquire; 198 li.wlock_acquire = _thr_rtld_wlock_acquire; 199 li.lock_release = _thr_rtld_lock_release; 200 li.thread_set_flag = _thr_rtld_set_flag; 201 li.thread_clr_flag = _thr_rtld_clr_flag; 202 li.at_fork = NULL; 203 204 /* mask signals, also force to resolve __sys_sigprocmask PLT */ 205 _thr_signal_block(curthread); 206 _rtld_thread_init(&li); 207 _thr_signal_unblock(curthread); 208 } 209 210 void 211 _thr_rtld_fini(void) 212 { 213 struct pthread *curthread; 214 215 curthread = _get_curthread(); 216 _thr_signal_block(curthread); 217 _rtld_thread_init(NULL); 218 _thr_signal_unblock(curthread); 219 } 220