1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include "lint.h" 30 #include "thr_uberdata.h" 31 #include <stddef.h> 32 33 /* 34 * 128 million keys should be enough for anyone. 35 * This allocates half a gigabyte of memory for the keys themselves and 36 * half a gigabyte of memory for each thread that uses the largest key. 37 */ 38 #define MAX_KEYS 0x08000000U 39 40 #pragma weak thr_keycreate = _thr_keycreate 41 #pragma weak pthread_key_create = _thr_keycreate 42 #pragma weak _pthread_key_create = _thr_keycreate 43 int 44 _thr_keycreate(thread_key_t *pkey, void (*destructor)(void *)) 45 { 46 tsd_metadata_t *tsdm = &curthread->ul_uberdata->tsd_metadata; 47 void (**old_data)(void *) = NULL; 48 void (**new_data)(void *); 49 uint_t old_nkeys; 50 uint_t new_nkeys; 51 52 lmutex_lock(&tsdm->tsdm_lock); 53 54 /* 55 * Unfortunately, pthread_getspecific() specifies that a 56 * pthread_getspecific() on an allocated key upon which the 57 * calling thread has not performed a pthread_setspecifc() 58 * must return NULL. Consider the following sequence: 59 * 60 * pthread_key_create(&key); 61 * pthread_setspecific(key, datum); 62 * pthread_key_delete(&key); 63 * pthread_key_create(&key); 64 * val = pthread_getspecific(key); 65 * 66 * According to POSIX, if the deleted key is reused for the new 67 * key returned by the second pthread_key_create(), then the 68 * pthread_getspecific() in the above example must return NULL 69 * (and not the stale datum). The implementation is thus left 70 * with two alternatives: 71 * 72 * (1) Reuse deleted keys. If this is to be implemented optimally, 73 * it requires that pthread_key_create() somehow associate 74 * the value NULL with the new (reused) key for each thread. 75 * Keeping the hot path fast and lock-free induces substantial 76 * complexity on the implementation. 77 * 78 * (2) Never reuse deleted keys. This allows the pthread_getspecific() 79 * implementation to simply perform a check against the number 80 * of keys set by the calling thread, returning NULL if the 81 * specified key is larger than the highest set key. This has 82 * the disadvantage of wasting memory (a program which simply 83 * loops calling pthread_key_create()/pthread_key_delete() 84 * will ultimately run out of memory), but permits an optimal 85 * pthread_getspecific() while allowing for simple key creation 86 * and deletion. 87 * 88 * All Solaris implementations have opted for (2). Given the 89 * ~10 years that this has been in the field, it is safe to assume 90 * that applications don't loop creating and destroying keys; we 91 * stick with (2). 92 */ 93 if (tsdm->tsdm_nused == (old_nkeys = tsdm->tsdm_nkeys)) { 94 /* 95 * We need to allocate or double the number of keys. 96 * tsdm->tsdm_nused must always be a power of two. 97 */ 98 if ((new_nkeys = (old_nkeys << 1)) == 0) 99 new_nkeys = 8; 100 101 if (new_nkeys > MAX_KEYS) { 102 lmutex_unlock(&tsdm->tsdm_lock); 103 return (EAGAIN); 104 } 105 if ((new_data = lmalloc(new_nkeys * sizeof (void *))) == NULL) { 106 lmutex_unlock(&tsdm->tsdm_lock); 107 return (ENOMEM); 108 } 109 if ((old_data = tsdm->tsdm_destro) == NULL) { 110 /* key == 0 is always invalid */ 111 new_data[0] = TSD_UNALLOCATED; 112 tsdm->tsdm_nused = 1; 113 } else { 114 (void) memcpy(new_data, old_data, 115 old_nkeys * sizeof (void *)); 116 } 117 tsdm->tsdm_destro = new_data; 118 tsdm->tsdm_nkeys = new_nkeys; 119 } 120 121 *pkey = tsdm->tsdm_nused; 122 tsdm->tsdm_destro[tsdm->tsdm_nused++] = destructor; 123 lmutex_unlock(&tsdm->tsdm_lock); 124 125 if (old_data != NULL) 126 lfree(old_data, old_nkeys * sizeof (void *)); 127 128 return (0); 129 } 130 131 /* 132 * Same as _thr_keycreate(), above, except that the key creation 133 * is performed only once. This relies upon the fact that a key 134 * value of THR_ONCE_KEY is invalid, and requires that the key be 135 * allocated with a value of THR_ONCE_KEY before calling here. 136 * THR_ONCE_KEY and PTHREAD_ONCE_KEY_NP, defined in <thread.h> 137 * and <pthread.h> respectively, must have the same value. 138 * Example: 139 * 140 * static pthread_key_t key = PTHREAD_ONCE_KEY_NP; 141 * ... 142 * pthread_key_create_once_np(&key, destructor); 143 */ 144 #pragma weak pthread_key_create_once_np = _thr_keycreate_once 145 #pragma weak _pthread_key_create_once_np = _thr_keycreate_once 146 #pragma weak thr_keycreate_once = _thr_keycreate_once 147 int 148 _thr_keycreate_once(thread_key_t *keyp, void (*destructor)(void *)) 149 { 150 static mutex_t key_lock = DEFAULTMUTEX; 151 thread_key_t key; 152 int error; 153 154 if (*keyp == THR_ONCE_KEY) { 155 lmutex_lock(&key_lock); 156 if (*keyp == THR_ONCE_KEY) { 157 error = _thr_keycreate(&key, destructor); 158 if (error) { 159 lmutex_unlock(&key_lock); 160 return (error); 161 } 162 _membar_producer(); 163 *keyp = key; 164 } 165 lmutex_unlock(&key_lock); 166 } 167 _membar_consumer(); 168 169 return (0); 170 } 171 172 #pragma weak pthread_key_delete = _thr_key_delete 173 #pragma weak _pthread_key_delete = _thr_key_delete 174 int 175 _thr_key_delete(thread_key_t key) 176 { 177 tsd_metadata_t *tsdm = &curthread->ul_uberdata->tsd_metadata; 178 179 lmutex_lock(&tsdm->tsdm_lock); 180 181 if (key >= tsdm->tsdm_nused || 182 tsdm->tsdm_destro[key] == TSD_UNALLOCATED) { 183 lmutex_unlock(&tsdm->tsdm_lock); 184 return (EINVAL); 185 } 186 187 tsdm->tsdm_destro[key] = TSD_UNALLOCATED; 188 lmutex_unlock(&tsdm->tsdm_lock); 189 190 return (0); 191 } 192 193 /* 194 * Blessedly, the pthread_getspecific() interface is much better than the 195 * thr_getspecific() interface in that it cannot return an error status. 196 * Thus, if the key specified is bogus, pthread_getspecific()'s behavior 197 * is undefined. As an added bonus (and as an artificat of not returning 198 * an error code), the requested datum is returned rather than stored 199 * through a parameter -- thereby avoiding the unnecessary store/load pair 200 * incurred by thr_getspecific(). Every once in a while, the Standards 201 * get it right -- but usually by accident. 202 */ 203 #pragma weak pthread_getspecific = _pthread_getspecific 204 void * 205 _pthread_getspecific(pthread_key_t key) 206 { 207 tsd_t *stsd; 208 209 /* 210 * We are cycle-shaving in this function because some 211 * applications make heavy use of it and one machine cycle 212 * can make a measurable difference in performance. This 213 * is why we waste a little memory and allocate a NULL value 214 * for the invalid key == 0 in curthread->ul_ftsd[0] rather 215 * than adjusting the key by subtracting one. 216 */ 217 if (key < TSD_NFAST) 218 return (curthread->ul_ftsd[key]); 219 220 if ((stsd = curthread->ul_stsd) != NULL && key < stsd->tsd_nalloc) 221 return (stsd->tsd_data[key]); 222 223 return (NULL); 224 } 225 226 #pragma weak thr_getspecific = _thr_getspecific 227 int 228 _thr_getspecific(thread_key_t key, void **valuep) 229 { 230 tsd_t *stsd; 231 232 /* 233 * Amazingly, some application code (and worse, some particularly 234 * fugly Solaris library code) _relies_ on the fact that 0 is always 235 * an invalid key. To preserve this semantic, 0 is never returned 236 * as a key from thr_/pthread_key_create(); we explicitly check 237 * for it here and return EINVAL. 238 */ 239 if (key == 0) 240 return (EINVAL); 241 242 if (key < TSD_NFAST) 243 *valuep = curthread->ul_ftsd[key]; 244 else if ((stsd = curthread->ul_stsd) != NULL && key < stsd->tsd_nalloc) 245 *valuep = stsd->tsd_data[key]; 246 else 247 *valuep = NULL; 248 249 return (0); 250 } 251 252 /* 253 * We call _thr_setspecific_slow() when the key specified 254 * is beyond the current thread's currently allocated range. 255 * This case is in a separate function because we want 256 * the compiler to optimize for the common case. 257 */ 258 static int 259 _thr_setspecific_slow(thread_key_t key, void *value) 260 { 261 ulwp_t *self = curthread; 262 tsd_metadata_t *tsdm = &self->ul_uberdata->tsd_metadata; 263 tsd_t *stsd; 264 tsd_t *ntsd; 265 uint_t nkeys; 266 267 /* 268 * It isn't necessary to grab locks in this path; 269 * tsdm->tsdm_nused can only increase. 270 */ 271 if (key >= tsdm->tsdm_nused) 272 return (EINVAL); 273 274 /* 275 * We would like to test (tsdm->tsdm_destro[key] == TSD_UNALLOCATED) 276 * here but that would require acquiring tsdm->tsdm_lock and we 277 * want to avoid locks in this path. 278 * 279 * We have a key which is (or at least _was_) valid. If this key 280 * is later deleted (or indeed, is deleted before we set the value), 281 * we don't care; such a condition would indicate an application 282 * race for which POSIX thankfully leaves the behavior unspecified. 283 * 284 * First, determine our new size. To avoid allocating more than we 285 * have to, continue doubling our size only until the new key fits. 286 * stsd->tsd_nalloc must always be a power of two. 287 */ 288 nkeys = ((stsd = self->ul_stsd) != NULL)? stsd->tsd_nalloc : 8; 289 for (; key >= nkeys; nkeys <<= 1) 290 continue; 291 292 /* 293 * Allocate the new TSD. 294 */ 295 if ((ntsd = lmalloc(nkeys * sizeof (void *))) == NULL) 296 return (ENOMEM); 297 298 if (stsd != NULL) { 299 /* 300 * Copy the old TSD across to the new. 301 */ 302 (void) memcpy(ntsd, stsd, stsd->tsd_nalloc * sizeof (void *)); 303 lfree(stsd, stsd->tsd_nalloc * sizeof (void *)); 304 } 305 306 ntsd->tsd_nalloc = nkeys; 307 ntsd->tsd_data[key] = value; 308 self->ul_stsd = ntsd; 309 310 return (0); 311 } 312 313 #pragma weak thr_setspecific = _thr_setspecific 314 #pragma weak pthread_setspecific = _thr_setspecific 315 #pragma weak _pthread_setspecific = _thr_setspecific 316 int 317 _thr_setspecific(thread_key_t key, void *value) 318 { 319 tsd_t *stsd; 320 int ret; 321 ulwp_t *self = curthread; 322 323 /* 324 * See the comment in _thr_getspecific(), above. 325 */ 326 if (key == 0) 327 return (EINVAL); 328 329 if (key < TSD_NFAST) { 330 curthread->ul_ftsd[key] = value; 331 return (0); 332 } 333 334 if ((stsd = curthread->ul_stsd) != NULL && key < stsd->tsd_nalloc) { 335 stsd->tsd_data[key] = value; 336 return (0); 337 } 338 339 /* 340 * This is a critical region since we are dealing with memory 341 * allocation and free. Similar protection required in tsd_free(). 342 */ 343 enter_critical(self); 344 ret = _thr_setspecific_slow(key, value); 345 exit_critical(self); 346 return (ret); 347 } 348 349 /* 350 * Contract-private interface for java. See PSARC/2003/159 351 * 352 * If the key falls within the TSD_NFAST range, return a non-negative 353 * offset that can be used by the caller to fetch the TSD data value 354 * directly out of the thread structure using %g7 (sparc) or %gs (x86). 355 * With the advent of TLS, %g7 and %gs are part of the ABI, even though 356 * the definition of the thread structure itself (ulwp_t) is private. 357 * 358 * We guarantee that the offset returned on sparc will fit within 359 * a SIMM13 field (that is, it is less than 2048). 360 * 361 * On failure (key is not in the TSD_NFAST range), return -1. 362 */ 363 ptrdiff_t 364 _thr_slot_offset(thread_key_t key) 365 { 366 if (key != 0 && key < TSD_NFAST) 367 return ((ptrdiff_t)offsetof(ulwp_t, ul_ftsd[key])); 368 return (-1); 369 } 370 371 /* 372 * This is called by _thrp_exit() to apply destructors to the thread's tsd. 373 */ 374 void 375 tsd_exit() 376 { 377 ulwp_t *self = curthread; 378 tsd_metadata_t *tsdm = &self->ul_uberdata->tsd_metadata; 379 thread_key_t key; 380 int recheck; 381 void *val; 382 void (*func)(void *); 383 384 lmutex_lock(&tsdm->tsdm_lock); 385 386 do { 387 recheck = 0; 388 389 for (key = 1; key < TSD_NFAST && 390 key < tsdm->tsdm_nused; key++) { 391 if ((func = tsdm->tsdm_destro[key]) != NULL && 392 func != TSD_UNALLOCATED && 393 (val = self->ul_ftsd[key]) != NULL) { 394 self->ul_ftsd[key] = NULL; 395 lmutex_unlock(&tsdm->tsdm_lock); 396 (*func)(val); 397 lmutex_lock(&tsdm->tsdm_lock); 398 recheck = 1; 399 } 400 } 401 402 if (self->ul_stsd == NULL) 403 continue; 404 405 /* 406 * Any of these destructors could cause us to grow the number 407 * TSD keys in the slow TSD; we cannot cache the slow TSD 408 * pointer through this loop. 409 */ 410 for (; key < self->ul_stsd->tsd_nalloc && 411 key < tsdm->tsdm_nused; key++) { 412 if ((func = tsdm->tsdm_destro[key]) != NULL && 413 func != TSD_UNALLOCATED && 414 (val = self->ul_stsd->tsd_data[key]) != NULL) { 415 self->ul_stsd->tsd_data[key] = NULL; 416 lmutex_unlock(&tsdm->tsdm_lock); 417 (*func)(val); 418 lmutex_lock(&tsdm->tsdm_lock); 419 recheck = 1; 420 } 421 } 422 } while (recheck); 423 424 lmutex_unlock(&tsdm->tsdm_lock); 425 426 /* 427 * We're done; if we have slow TSD, we need to free it. 428 */ 429 tsd_free(self); 430 } 431 432 void 433 tsd_free(ulwp_t *ulwp) 434 { 435 tsd_t *stsd; 436 ulwp_t *self = curthread; 437 438 enter_critical(self); 439 if ((stsd = ulwp->ul_stsd) != NULL) 440 lfree(stsd, stsd->tsd_nalloc * sizeof (void *)); 441 ulwp->ul_stsd = NULL; 442 exit_critical(self); 443 } 444