1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * 4 * This software was developed by Konstantin Belousov 5 * under sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/types.h> 30 #include <sys/mman.h> 31 #include <sys/queue.h> 32 #include <sys/time.h> 33 #include "namespace.h" 34 #include <stdlib.h> 35 #include "un-namespace.h" 36 37 #include "thr_private.h" 38 39 struct psh { 40 LIST_ENTRY(psh) link; 41 void *key; 42 void *val; 43 }; 44 45 LIST_HEAD(pshared_hash_head, psh); 46 #define HASH_SIZE 128 47 static struct pshared_hash_head pshared_hash[HASH_SIZE]; 48 #define PSHARED_KEY_HASH(key) (((unsigned long)(key) >> 8) % HASH_SIZE) 49 /* XXXKIB: lock could be split to per-hash chain, if appears contested */ 50 static struct urwlock pshared_lock = DEFAULT_URWLOCK; 51 static int page_size; 52 53 #define GC_INTERVAL_USEC 25000 54 static struct timeval pshared_gc_time = {0, 0}; 55 bool __thr_pshared_destroy_imm_gc = false; 56 57 void 58 __thr_pshared_init(void) 59 { 60 int i; 61 62 page_size = getpagesize(); 63 THR_ASSERT(page_size >= THR_PAGE_SIZE_MIN, 64 "THR_PAGE_SIZE_MIN is too large"); 65 66 _thr_urwlock_init(&pshared_lock); 67 for (i = 0; i < HASH_SIZE; i++) 68 LIST_INIT(&pshared_hash[i]); 69 } 70 71 static void 72 pshared_rlock(struct pthread *curthread) 73 { 74 75 curthread->locklevel++; 76 _thr_rwl_rdlock(&pshared_lock); 77 } 78 79 static void 80 pshared_wlock(struct pthread *curthread) 81 { 82 83 curthread->locklevel++; 84 _thr_rwl_wrlock(&pshared_lock); 85 } 86 87 static void 88 pshared_unlock(struct pthread *curthread) 89 { 90 91 _thr_rwl_unlock(&pshared_lock); 92 curthread->locklevel--; 93 _thr_ast(curthread); 94 } 95 96 /* 97 * Among all processes sharing a lock only one executes 98 * pthread_lock_destroy(). Other processes still have the hash and 99 * mapped off-page. 100 * 101 * Mitigate the problem by checking the liveness of all hashed keys 102 * periodically. Right now this is executed on pthread_lock_destroy() 103 * if previous gc happen later than 25msecs. 104 */ 105 static void 106 pshared_gc(struct pthread *curthread) 107 { 108 struct pshared_hash_head *hd; 109 struct psh *h, *h1; 110 int error, i; 111 struct timeval now, diff; 112 113 pshared_wlock(curthread); 114 115 if (!__thr_pshared_destroy_imm_gc && gettimeofday(&now, NULL) == 0) { 116 timersub(&now, &pshared_gc_time, &diff); 117 if (diff.tv_sec == 0 && diff.tv_usec < GC_INTERVAL_USEC) { 118 pshared_unlock(curthread); 119 return; 120 } 121 } 122 123 for (i = 0; i < HASH_SIZE; i++) { 124 hd = &pshared_hash[i]; 125 LIST_FOREACH_SAFE(h, hd, link, h1) { 126 error = _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_ALIVE, 127 h->val, NULL); 128 if (error == 0) 129 continue; 130 LIST_REMOVE(h, link); 131 munmap(h->val, page_size); 132 free(h); 133 } 134 } 135 136 if (!__thr_pshared_destroy_imm_gc && gettimeofday(&now, NULL) == 0) { 137 pshared_gc_time = now; 138 } 139 140 pshared_unlock(curthread); 141 } 142 143 static void * 144 pshared_lookup(void *key) 145 { 146 struct pshared_hash_head *hd; 147 struct psh *h; 148 149 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 150 LIST_FOREACH(h, hd, link) { 151 if (h->key == key) 152 return (h->val); 153 } 154 return (NULL); 155 } 156 157 static int 158 pshared_insert(void *key, void **val) 159 { 160 struct pshared_hash_head *hd; 161 struct psh *h; 162 163 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 164 LIST_FOREACH(h, hd, link) { 165 /* 166 * When the key already exists in the hash, we should 167 * return either the new (just mapped) or old (hashed) 168 * val, and the other val should be unmapped to avoid 169 * address space leak. 170 * 171 * If two threads perform lock of the same object 172 * which is not yet stored in the pshared_hash, then 173 * the val already inserted by the first thread should 174 * be returned, and the second val freed (order is by 175 * the pshared_lock()). Otherwise, if we unmap the 176 * value obtained from the hash, the first thread 177 * might operate on an unmapped off-page object. 178 * 179 * There is still an issue: if hashed key was unmapped 180 * and then other page is mapped at the same key 181 * address, the hash would return the old val. I 182 * decided to handle the race of simultaneous hash 183 * insertion, leaving the unlikely remap problem 184 * unaddressed. 185 */ 186 if (h->key == key) { 187 if (h->val != *val) { 188 munmap(*val, page_size); 189 *val = h->val; 190 } 191 return (1); 192 } 193 } 194 195 h = malloc(sizeof(*h)); 196 if (h == NULL) 197 return (0); 198 h->key = key; 199 h->val = *val; 200 LIST_INSERT_HEAD(hd, h, link); 201 return (1); 202 } 203 204 static void * 205 pshared_remove(void *key) 206 { 207 struct pshared_hash_head *hd; 208 struct psh *h; 209 void *val; 210 211 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 212 LIST_FOREACH(h, hd, link) { 213 if (h->key == key) { 214 LIST_REMOVE(h, link); 215 val = h->val; 216 free(h); 217 return (val); 218 } 219 } 220 return (NULL); 221 } 222 223 static void 224 pshared_clean(void *key, void *val) 225 { 226 227 if (val != NULL) 228 munmap(val, page_size); 229 _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_DESTROY, key, NULL); 230 } 231 232 static void 233 pshared_destroy(struct pthread *curthread, void *key) 234 { 235 void *val; 236 237 pshared_wlock(curthread); 238 val = pshared_remove(key); 239 pshared_unlock(curthread); 240 pshared_clean(key, val); 241 } 242 243 void * 244 __thr_pshared_offpage(void *key, int doalloc) 245 { 246 struct pthread *curthread; 247 void *res; 248 int fd, ins_done; 249 250 curthread = _get_curthread(); 251 if (doalloc) { 252 pshared_destroy(curthread, key); 253 res = NULL; 254 } else { 255 pshared_rlock(curthread); 256 res = pshared_lookup(key); 257 pshared_unlock(curthread); 258 if (res != NULL) 259 return (res); 260 } 261 fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT : 262 UMTX_SHM_LOOKUP, key, NULL); 263 if (fd == -1) 264 return (NULL); 265 res = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 266 close(fd); 267 if (res == MAP_FAILED) 268 return (NULL); 269 pshared_wlock(curthread); 270 ins_done = pshared_insert(key, &res); 271 pshared_unlock(curthread); 272 if (!ins_done) { 273 pshared_clean(key, res); 274 res = NULL; 275 } 276 return (res); 277 } 278 279 void 280 __thr_pshared_destroy(void *key) 281 { 282 struct pthread *curthread; 283 284 curthread = _get_curthread(); 285 pshared_destroy(curthread, key); 286 pshared_gc(curthread); 287 } 288 289 void 290 __thr_pshared_atfork_pre(void) 291 { 292 293 _thr_rwl_rdlock(&pshared_lock); 294 } 295 296 void 297 __thr_pshared_atfork_post(void) 298 { 299 300 _thr_rwl_unlock(&pshared_lock); 301 } 302