1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * 4 * This software was developed by Konstantin Belousov 5 * under sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/types.h> 30 #include <sys/mman.h> 31 #include <sys/queue.h> 32 #include "namespace.h" 33 #include <stdlib.h> 34 #include "un-namespace.h" 35 36 #include "thr_private.h" 37 38 struct psh { 39 LIST_ENTRY(psh) link; 40 void *key; 41 void *val; 42 }; 43 44 LIST_HEAD(pshared_hash_head, psh); 45 #define HASH_SIZE 128 46 static struct pshared_hash_head pshared_hash[HASH_SIZE]; 47 #define PSHARED_KEY_HASH(key) (((unsigned long)(key) >> 8) % HASH_SIZE) 48 /* XXXKIB: lock could be split to per-hash chain, if appears contested */ 49 static struct urwlock pshared_lock = DEFAULT_URWLOCK; 50 static int page_size; 51 52 void 53 __thr_pshared_init(void) 54 { 55 int i; 56 57 page_size = getpagesize(); 58 THR_ASSERT(page_size >= THR_PAGE_SIZE_MIN, 59 "THR_PAGE_SIZE_MIN is too large"); 60 61 _thr_urwlock_init(&pshared_lock); 62 for (i = 0; i < HASH_SIZE; i++) 63 LIST_INIT(&pshared_hash[i]); 64 } 65 66 static void 67 pshared_rlock(struct pthread *curthread) 68 { 69 70 curthread->locklevel++; 71 _thr_rwl_rdlock(&pshared_lock); 72 } 73 74 static void 75 pshared_wlock(struct pthread *curthread) 76 { 77 78 curthread->locklevel++; 79 _thr_rwl_wrlock(&pshared_lock); 80 } 81 82 static void 83 pshared_unlock(struct pthread *curthread) 84 { 85 86 _thr_rwl_unlock(&pshared_lock); 87 curthread->locklevel--; 88 _thr_ast(curthread); 89 } 90 91 /* 92 * Among all processes sharing a lock only one executes 93 * pthread_lock_destroy(). Other processes still have the hash and 94 * mapped off-page. 95 * 96 * Mitigate the problem by checking the liveness of all hashed keys 97 * periodically. Right now this is executed on each 98 * pthread_lock_destroy(), but may be done less often if found to be 99 * too time-consuming. 100 */ 101 static void 102 pshared_gc(struct pthread *curthread) 103 { 104 struct pshared_hash_head *hd; 105 struct psh *h, *h1; 106 int error, i; 107 108 pshared_wlock(curthread); 109 for (i = 0; i < HASH_SIZE; i++) { 110 hd = &pshared_hash[i]; 111 LIST_FOREACH_SAFE(h, hd, link, h1) { 112 error = _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_ALIVE, 113 h->val, NULL); 114 if (error == 0) 115 continue; 116 LIST_REMOVE(h, link); 117 munmap(h->val, page_size); 118 free(h); 119 } 120 } 121 pshared_unlock(curthread); 122 } 123 124 static void * 125 pshared_lookup(void *key) 126 { 127 struct pshared_hash_head *hd; 128 struct psh *h; 129 130 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 131 LIST_FOREACH(h, hd, link) { 132 if (h->key == key) 133 return (h->val); 134 } 135 return (NULL); 136 } 137 138 static int 139 pshared_insert(void *key, void **val) 140 { 141 struct pshared_hash_head *hd; 142 struct psh *h; 143 144 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 145 LIST_FOREACH(h, hd, link) { 146 /* 147 * When the key already exists in the hash, we should 148 * return either the new (just mapped) or old (hashed) 149 * val, and the other val should be unmapped to avoid 150 * address space leak. 151 * 152 * If two threads perform lock of the same object 153 * which is not yet stored in the pshared_hash, then 154 * the val already inserted by the first thread should 155 * be returned, and the second val freed (order is by 156 * the pshared_lock()). Otherwise, if we unmap the 157 * value obtained from the hash, the first thread 158 * might operate on an unmapped off-page object. 159 * 160 * There is still an issue: if hashed key was unmapped 161 * and then other page is mapped at the same key 162 * address, the hash would return the old val. I 163 * decided to handle the race of simultaneous hash 164 * insertion, leaving the unlikely remap problem 165 * unaddressed. 166 */ 167 if (h->key == key) { 168 if (h->val != *val) { 169 munmap(*val, page_size); 170 *val = h->val; 171 } 172 return (1); 173 } 174 } 175 176 h = malloc(sizeof(*h)); 177 if (h == NULL) 178 return (0); 179 h->key = key; 180 h->val = *val; 181 LIST_INSERT_HEAD(hd, h, link); 182 return (1); 183 } 184 185 static void * 186 pshared_remove(void *key) 187 { 188 struct pshared_hash_head *hd; 189 struct psh *h; 190 void *val; 191 192 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 193 LIST_FOREACH(h, hd, link) { 194 if (h->key == key) { 195 LIST_REMOVE(h, link); 196 val = h->val; 197 free(h); 198 return (val); 199 } 200 } 201 return (NULL); 202 } 203 204 static void 205 pshared_clean(void *key, void *val) 206 { 207 208 if (val != NULL) 209 munmap(val, page_size); 210 _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_DESTROY, key, NULL); 211 } 212 213 static void 214 pshared_destroy(struct pthread *curthread, void *key) 215 { 216 void *val; 217 218 pshared_wlock(curthread); 219 val = pshared_remove(key); 220 pshared_unlock(curthread); 221 pshared_clean(key, val); 222 } 223 224 void * 225 __thr_pshared_offpage(void *key, int doalloc) 226 { 227 struct pthread *curthread; 228 void *res; 229 int fd, ins_done; 230 231 curthread = _get_curthread(); 232 if (doalloc) { 233 pshared_destroy(curthread, key); 234 res = NULL; 235 } else { 236 pshared_rlock(curthread); 237 res = pshared_lookup(key); 238 pshared_unlock(curthread); 239 if (res != NULL) 240 return (res); 241 } 242 fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT : 243 UMTX_SHM_LOOKUP, key, NULL); 244 if (fd == -1) 245 return (NULL); 246 res = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 247 close(fd); 248 if (res == MAP_FAILED) 249 return (NULL); 250 pshared_wlock(curthread); 251 ins_done = pshared_insert(key, &res); 252 pshared_unlock(curthread); 253 if (!ins_done) { 254 pshared_clean(key, res); 255 res = NULL; 256 } 257 return (res); 258 } 259 260 void 261 __thr_pshared_destroy(void *key) 262 { 263 struct pthread *curthread; 264 265 curthread = _get_curthread(); 266 pshared_destroy(curthread, key); 267 pshared_gc(curthread); 268 } 269 270 void 271 __thr_pshared_atfork_pre(void) 272 { 273 274 _thr_rwl_rdlock(&pshared_lock); 275 } 276 277 void 278 __thr_pshared_atfork_post(void) 279 { 280 281 _thr_rwl_unlock(&pshared_lock); 282 } 283