1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include "lint.h" 30 #include "thr_uberdata.h" 31 32 #define MIN_MOD_SLOTS 16 33 34 /* 35 * To inform libc_init that we are on the primary link map. 36 */ 37 int primary_link_map; 38 39 #if defined(_LP64) 40 #define ALIGN 16 41 #else 42 #define ALIGN 8 43 #endif 44 45 /* 46 * Grow the TLS module information array as necessary to include the 47 * specified module-id. tls_modinfo->tls_size must be a power of two. 48 * Return a pointer to the (possibly reallocated) module information array. 49 */ 50 static TLS_modinfo * 51 tls_modinfo_alloc(tls_metadata_t *tlsm, ulong_t moduleid) 52 { 53 tls_t *tls_modinfo = &tlsm->tls_modinfo; 54 TLS_modinfo *modinfo; 55 size_t mod_slots; 56 57 if ((modinfo = tls_modinfo->tls_data) == NULL || 58 tls_modinfo->tls_size <= moduleid) { 59 if ((mod_slots = tls_modinfo->tls_size) == 0) 60 mod_slots = MIN_MOD_SLOTS; 61 while (mod_slots <= moduleid) 62 mod_slots *= 2; 63 modinfo = lmalloc(mod_slots * sizeof (TLS_modinfo)); 64 if (tls_modinfo->tls_data != NULL) { 65 (void) _private_memcpy(modinfo, tls_modinfo->tls_data, 66 tls_modinfo->tls_size * sizeof (TLS_modinfo)); 67 lfree(tls_modinfo->tls_data, 68 tls_modinfo->tls_size * sizeof (TLS_modinfo)); 69 } 70 tls_modinfo->tls_data = modinfo; 71 tls_modinfo->tls_size = mod_slots; 72 } 73 return (modinfo); 74 } 75 76 /* 77 * This is called from the dynamic linker, before libc_init() is called, 78 * to setup all of the TLS blocks that are available at process startup 79 * and hence must be included as part of the static TLS block. 80 * No locks are needed because we are single-threaded at this point. 81 * We must be careful not to call any function that could possibly 82 * invoke the dynamic linker. That is, we must only call functions 83 * that are wholly private to libc. 84 */ 85 void 86 __tls_static_mods(TLS_modinfo **tlslist, unsigned long statictlssize) 87 { 88 ulwp_t *oldself = __curthread(); 89 tls_metadata_t *tlsm; 90 TLS_modinfo **tlspp; 91 TLS_modinfo *tlsp; 92 TLS_modinfo *modinfo; 93 caddr_t data; 94 caddr_t data_end; 95 int max_modid; 96 97 primary_link_map = 1; /* inform libc_init */ 98 if (statictlssize == 0) 99 return; 100 101 /* 102 * Retrieve whatever dynamic TLS metadata was generated by code 103 * running on alternate link maps prior to now (we must be running 104 * on the primary link map now since __tls_static_mods() is only 105 * called on the primary link map). 106 */ 107 tlsm = &__uberdata.tls_metadata; 108 if (oldself != NULL) { 109 (void) _private_memcpy(tlsm, 110 &oldself->ul_uberdata->tls_metadata, sizeof (*tlsm)); 111 ASSERT(tlsm->static_tls.tls_data == NULL); 112 } 113 114 /* 115 * We call lmalloc() to allocate the template even though libc_init() 116 * has not yet been called. lmalloc() must and does deal with this. 117 */ 118 ASSERT((statictlssize & (ALIGN - 1)) == 0); 119 tlsm->static_tls.tls_data = data = lmalloc(statictlssize); 120 data_end = data + statictlssize; 121 tlsm->static_tls.tls_size = statictlssize; 122 /* 123 * Initialize the static TLS template. 124 * We make no assumptions about the order in memory of the TLS 125 * modules we are processing, only that they fit within the 126 * total size we are given and that they are self-consistent. 127 * We do not assume any order for the moduleid's; we only assume 128 * that they are reasonably small integers. 129 */ 130 for (max_modid = 0, tlspp = tlslist; (tlsp = *tlspp) != NULL; tlspp++) { 131 ASSERT(tlsp->tm_flags & TM_FLG_STATICTLS); 132 ASSERT(tlsp->tm_stattlsoffset > 0); 133 ASSERT(tlsp->tm_stattlsoffset <= statictlssize); 134 ASSERT((tlsp->tm_stattlsoffset & (ALIGN - 1)) == 0); 135 ASSERT(tlsp->tm_filesz <= tlsp->tm_memsz); 136 ASSERT(tlsp->tm_memsz <= tlsp->tm_stattlsoffset); 137 if (tlsp->tm_filesz) 138 (void) _private_memcpy(data_end-tlsp->tm_stattlsoffset, 139 tlsp->tm_tlsblock, tlsp->tm_filesz); 140 if (max_modid < tlsp->tm_modid) 141 max_modid = tlsp->tm_modid; 142 } 143 /* 144 * Record the static TLS_modinfo information. 145 */ 146 modinfo = tls_modinfo_alloc(tlsm, max_modid); 147 for (tlspp = tlslist; (tlsp = *tlspp) != NULL; tlspp++) 148 (void) _private_memcpy(&modinfo[tlsp->tm_modid], 149 tlsp, sizeof (*tlsp)); 150 151 /* 152 * Copy the new tls_metadata back to the old, if any, 153 * since it will be copied up again in libc_init(). 154 */ 155 if (oldself != NULL) 156 (void) _private_memcpy(&oldself->ul_uberdata->tls_metadata, 157 tlsm, sizeof (*tlsm)); 158 } 159 160 /* 161 * This is called from the dynamic linker for each module not included 162 * in the static TLS mod list, after the module has been loaded but 163 * before any of the module's init code has been executed. 164 */ 165 void 166 __tls_mod_add(TLS_modinfo *tlsp) 167 { 168 tls_metadata_t *tlsm = &curthread->ul_uberdata->tls_metadata; 169 ulong_t moduleid = tlsp->tm_modid; 170 TLS_modinfo *modinfo; 171 172 lmutex_lock(&tlsm->tls_lock); 173 ASSERT(!(tlsp->tm_flags & TM_FLG_STATICTLS)); 174 ASSERT(tlsp->tm_filesz <= tlsp->tm_memsz); 175 modinfo = tls_modinfo_alloc(tlsm, moduleid); 176 (void) _private_memcpy(&modinfo[moduleid], tlsp, sizeof (*tlsp)); 177 lmutex_unlock(&tlsm->tls_lock); 178 } 179 180 /* 181 * Called for each module as it is unloaded from memory by dlclose(). 182 */ 183 void 184 __tls_mod_remove(TLS_modinfo *tlsp) 185 { 186 tls_metadata_t *tlsm = &curthread->ul_uberdata->tls_metadata; 187 ulong_t moduleid = tlsp->tm_modid; 188 TLS_modinfo *modinfo; 189 190 lmutex_lock(&tlsm->tls_lock); 191 ASSERT(tlsm->tls_modinfo.tls_data != NULL && 192 moduleid < tlsm->tls_modinfo.tls_size); 193 modinfo = tlsm->tls_modinfo.tls_data; 194 (void) _private_memset(&modinfo[moduleid], 0, sizeof (TLS_modinfo)); 195 lmutex_unlock(&tlsm->tls_lock); 196 } 197 198 extern int _preexec_exit_handlers(); 199 extern void libc_init(); 200 201 const Lc_interface tls_rtldinfo[] = { 202 {CI_VERSION, (int(*)())CI_V_CURRENT}, 203 {CI_ATEXIT, (int(*)())_preexec_exit_handlers}, 204 {CI_TLS_MODADD, (int(*)())__tls_mod_add}, 205 {CI_TLS_MODREM, (int(*)())__tls_mod_remove}, 206 {CI_TLS_STATMOD, (int(*)())__tls_static_mods}, 207 {CI_THRINIT, (int(*)())libc_init}, 208 {CI_NULL, (int(*)())NULL} 209 }; 210 211 /* 212 * Return the address of a TLS variable for the current thread. 213 * Run the constructors for newly-allocated dynamic TLS. 214 */ 215 void * 216 slow_tls_get_addr(TLS_index *tls_index) 217 { 218 ulwp_t *self = curthread; 219 tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata; 220 TLS_modinfo *tlsp; 221 ulong_t moduleid; 222 tls_t *tlsent; 223 caddr_t base; 224 void (**initarray)(void); 225 ulong_t arraycnt = 0; 226 227 /* 228 * Defer signals until we have finished calling 229 * all of the constructors. 230 */ 231 sigoff(self); 232 lmutex_lock(&tlsm->tls_lock); 233 if ((moduleid = tls_index->ti_moduleid) < self->ul_ntlsent) 234 tlsent = self->ul_tlsent; 235 else { 236 ASSERT(moduleid < tlsm->tls_modinfo.tls_size); 237 tlsent = lmalloc(tlsm->tls_modinfo.tls_size * sizeof (tls_t)); 238 if (self->ul_tlsent != NULL) { 239 (void) _private_memcpy(tlsent, self->ul_tlsent, 240 self->ul_ntlsent * sizeof (tls_t)); 241 lfree(self->ul_tlsent, 242 self->ul_ntlsent * sizeof (tls_t)); 243 } 244 self->ul_tlsent = tlsent; 245 self->ul_ntlsent = tlsm->tls_modinfo.tls_size; 246 } 247 tlsent += moduleid; 248 if ((base = tlsent->tls_data) == NULL) { 249 tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid; 250 if (tlsp->tm_memsz == 0) { /* dlclose()d module? */ 251 base = NULL; 252 } else if (tlsp->tm_flags & TM_FLG_STATICTLS) { 253 /* static TLS is already allocated/initialized */ 254 base = (caddr_t)self - tlsp->tm_stattlsoffset; 255 tlsent->tls_data = base; 256 tlsent->tls_size = 0; /* don't lfree() this space */ 257 } else { 258 /* allocate/initialize the dynamic TLS */ 259 base = lmalloc(tlsp->tm_memsz); 260 if (tlsp->tm_filesz != 0) 261 (void) _private_memcpy(base, tlsp->tm_tlsblock, 262 tlsp->tm_filesz); 263 tlsent->tls_data = base; 264 tlsent->tls_size = tlsp->tm_memsz; 265 /* remember the constructors */ 266 arraycnt = tlsp->tm_tlsinitarraycnt; 267 initarray = tlsp->tm_tlsinitarray; 268 } 269 } 270 lmutex_unlock(&tlsm->tls_lock); 271 272 /* 273 * Call constructors, if any, in ascending order. 274 * We have to do this after dropping tls_lock because 275 * we have no idea what the constructors will do. 276 * At least we have signals deferred until they are done. 277 */ 278 if (arraycnt) { 279 do { 280 (**initarray++)(); 281 } while (--arraycnt != 0); 282 } 283 284 sigon(self); 285 return (base + tls_index->ti_tlsoffset); 286 } 287 288 #ifdef TLS_GET_ADDR_IS_WRITTEN_IN_ASSEMBLER 289 /* 290 * For speed, we do not make reference to any static data in this function. 291 * If necessary to do so, we do a tail call to slow_tls_get_addr(). 292 */ 293 void * 294 __tls_get_addr(TLS_index *tls_index) 295 { 296 ulwp_t *self = curthread; 297 tls_t *tlsent = self->ul_tlsent; 298 ulong_t moduleid; 299 caddr_t base; 300 301 if ((moduleid = tls_index->ti_moduleid) < self->ul_ntlsent && 302 (base = tlsent[moduleid].tls_data) != NULL) 303 return (base + tls_index->ti_tlsoffset); 304 305 return (slow_tls_get_addr(tls_index)); 306 } 307 #endif /* TLS_GET_ADDR_IS_WRITTEN_IN_ASSEMBLER */ 308 309 /* 310 * This is called by _thr_setup() to initialize the thread's static TLS. 311 * Constructors for initially allocated static TLS are called here. 312 */ 313 void 314 tls_setup() 315 { 316 ulwp_t *self = curthread; 317 tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata; 318 TLS_modinfo *tlsp; 319 long moduleid; 320 ulong_t nmods; 321 322 if (tlsm->static_tls.tls_size == 0) /* no static TLS */ 323 return; 324 325 /* static TLS initialization */ 326 (void) _private_memcpy((caddr_t)self - tlsm->static_tls.tls_size, 327 tlsm->static_tls.tls_data, tlsm->static_tls.tls_size); 328 329 /* call TLS constructors for the static TLS just initialized */ 330 lmutex_lock(&tlsm->tls_lock); 331 nmods = tlsm->tls_modinfo.tls_size; 332 for (moduleid = 0; moduleid < nmods; moduleid++) { 333 /* 334 * Resume where we left off in the module array. 335 * tls_modinfo.tls_data may have changed since we 336 * dropped and reacquired tls_lock, but TLS modules 337 * retain their positions in the new array. 338 */ 339 tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid; 340 /* 341 * Call constructors for this module if there are any 342 * to be called and if it is part of the static TLS. 343 */ 344 if (tlsp->tm_tlsinitarraycnt != 0 && 345 (tlsp->tm_flags & TM_FLG_STATICTLS)) { 346 ulong_t arraycnt = tlsp->tm_tlsinitarraycnt; 347 void (**initarray)(void) = tlsp->tm_tlsinitarray; 348 349 /* 350 * Call the constructors in ascending order. 351 * We must drop tls_lock while doing this because 352 * we have no idea what the constructors will do. 353 */ 354 lmutex_unlock(&tlsm->tls_lock); 355 do { 356 (**initarray++)(); 357 } while (--arraycnt != 0); 358 lmutex_lock(&tlsm->tls_lock); 359 } 360 } 361 lmutex_unlock(&tlsm->tls_lock); 362 } 363 364 /* 365 * This is called by _thrp_exit() to deallocate the thread's TLS. 366 * Destructors for all allocated TLS are called here. 367 */ 368 void 369 tls_exit() 370 { 371 ulwp_t *self = curthread; 372 tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata; 373 tls_t *tlsent; 374 TLS_modinfo *tlsp; 375 long moduleid; 376 ulong_t nmods; 377 378 if (tlsm->static_tls.tls_size == 0 && self->ul_ntlsent == 0) 379 return; /* no TLS */ 380 381 /* 382 * Call TLS destructors for all TLS allocated for this thread. 383 */ 384 lmutex_lock(&tlsm->tls_lock); 385 nmods = tlsm->tls_modinfo.tls_size; 386 for (moduleid = nmods - 1; moduleid >= 0; --moduleid) { 387 /* 388 * Resume where we left off in the module array. 389 * tls_modinfo.tls_data may have changed since we 390 * dropped and reacquired tls_lock, but TLS modules 391 * retain their positions in the new array. 392 */ 393 tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid; 394 /* 395 * Call destructors for this module if there are any 396 * to be called and if it is part of the static TLS or 397 * if the dynamic TLS for the module has been allocated. 398 */ 399 if (tlsp->tm_tlsfiniarraycnt != 0 && 400 ((tlsp->tm_flags & TM_FLG_STATICTLS) || 401 (moduleid < self->ul_ntlsent && 402 (tlsent = self->ul_tlsent) != NULL && 403 tlsent[moduleid].tls_data != NULL))) { 404 ulong_t arraycnt = tlsp->tm_tlsfiniarraycnt; 405 void (**finiarray)(void) = tlsp->tm_tlsfiniarray; 406 407 /* 408 * Call the destructors in descending order. 409 * We must drop tls_lock while doing this because 410 * we have no idea what the destructors will do. 411 */ 412 lmutex_unlock(&tlsm->tls_lock); 413 finiarray += arraycnt; 414 do { 415 (**--finiarray)(); 416 } while (--arraycnt != 0); 417 lmutex_lock(&tlsm->tls_lock); 418 } 419 } 420 lmutex_unlock(&tlsm->tls_lock); 421 422 tls_free(self); 423 } 424 425 /* 426 * We only free the dynamically allocated TLS; the statically 427 * allocated TLS is reused when the ulwp_t is reallocated. 428 */ 429 void 430 tls_free(ulwp_t *ulwp) 431 { 432 ulong_t moduleid; 433 tls_t *tlsent; 434 size_t ntlsent; 435 void *base; 436 size_t size; 437 438 if ((tlsent = ulwp->ul_tlsent) == NULL || 439 (ntlsent = ulwp->ul_ntlsent) == 0) 440 return; 441 442 for (moduleid = 0; moduleid < ntlsent; moduleid++, tlsent++) { 443 if ((base = tlsent->tls_data) != NULL && 444 (size = tlsent->tls_size) != 0) 445 lfree(base, size); 446 tlsent->tls_data = NULL; /* paranoia */ 447 tlsent->tls_size = 0; 448 } 449 lfree(ulwp->ul_tlsent, ntlsent * sizeof (tls_t)); 450 ulwp->ul_tlsent = NULL; 451 ulwp->ul_ntlsent = 0; 452 } 453