1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include "lint.h" 30 #include "thr_uberdata.h" 31 32 #define MIN_MOD_SLOTS 16 33 34 /* 35 * To inform libc_init that we are on the primary link map. 36 */ 37 int primary_link_map; 38 39 #if defined(_LP64) 40 #define ALIGN 16 41 #else 42 #define ALIGN 8 43 #endif 44 45 /* 46 * Grow the TLS module information array as necessary to include the 47 * specified module-id. tls_modinfo->tls_size must be a power of two. 48 * Return a pointer to the (possibly reallocated) module information array. 49 */ 50 static TLS_modinfo * 51 tls_modinfo_alloc(tls_metadata_t *tlsm, ulong_t moduleid) 52 { 53 tls_t *tls_modinfo = &tlsm->tls_modinfo; 54 TLS_modinfo *modinfo; 55 size_t mod_slots; 56 57 if ((modinfo = tls_modinfo->tls_data) == NULL || 58 tls_modinfo->tls_size <= moduleid) { 59 if ((mod_slots = tls_modinfo->tls_size) == 0) 60 mod_slots = MIN_MOD_SLOTS; 61 while (mod_slots <= moduleid) 62 mod_slots *= 2; 63 modinfo = lmalloc(mod_slots * sizeof (TLS_modinfo)); 64 if (tls_modinfo->tls_data != NULL) { 65 (void) _private_memcpy(modinfo, tls_modinfo->tls_data, 66 tls_modinfo->tls_size * sizeof (TLS_modinfo)); 67 lfree(tls_modinfo->tls_data, 68 tls_modinfo->tls_size * sizeof (TLS_modinfo)); 69 } 70 tls_modinfo->tls_data = modinfo; 71 tls_modinfo->tls_size = mod_slots; 72 } 73 return (modinfo); 74 } 75 76 /* 77 * This is called from the dynamic linker, before libc_init() is called, 78 * to setup all of the TLS blocks that are available at process startup 79 * and hence must be included as part of the static TLS block. 80 * No locks are needed because we are single-threaded at this point. 81 * We must be careful not to call any function that could possibly 82 * invoke the dynamic linker. That is, we must only call functions 83 * that are wholly private to libc. 84 */ 85 void 86 __tls_static_mods(TLS_modinfo **tlslist, unsigned long statictlssize) 87 { 88 ulwp_t *oldself = __curthread(); 89 tls_metadata_t *tlsm; 90 TLS_modinfo **tlspp; 91 TLS_modinfo *tlsp; 92 TLS_modinfo *modinfo; 93 caddr_t data; 94 caddr_t data_end; 95 int max_modid; 96 97 primary_link_map = 1; /* inform libc_init */ 98 if (statictlssize == 0) 99 return; 100 101 /* 102 * Retrieve whatever dynamic TLS metadata was generated by code 103 * running on alternate link maps prior to now (we must be running 104 * on the primary link map now since __tls_static_mods() is only 105 * called on the primary link map). 106 */ 107 tlsm = &__uberdata.tls_metadata; 108 if (oldself != NULL) { 109 *tlsm = oldself->ul_uberdata->tls_metadata; 110 ASSERT(tlsm->static_tls.tls_data == NULL); 111 } 112 113 /* 114 * We call lmalloc() to allocate the template even though libc_init() 115 * has not yet been called. lmalloc() must and does deal with this. 116 */ 117 ASSERT((statictlssize & (ALIGN - 1)) == 0); 118 tlsm->static_tls.tls_data = data = lmalloc(statictlssize); 119 data_end = data + statictlssize; 120 tlsm->static_tls.tls_size = statictlssize; 121 /* 122 * Initialize the static TLS template. 123 * We make no assumptions about the order in memory of the TLS 124 * modules we are processing, only that they fit within the 125 * total size we are given and that they are self-consistent. 126 * We do not assume any order for the moduleid's; we only assume 127 * that they are reasonably small integers. 128 */ 129 for (max_modid = 0, tlspp = tlslist; (tlsp = *tlspp) != NULL; tlspp++) { 130 ASSERT(tlsp->tm_flags & TM_FLG_STATICTLS); 131 ASSERT(tlsp->tm_stattlsoffset > 0); 132 ASSERT(tlsp->tm_stattlsoffset <= statictlssize); 133 ASSERT((tlsp->tm_stattlsoffset & (ALIGN - 1)) == 0); 134 ASSERT(tlsp->tm_filesz <= tlsp->tm_memsz); 135 ASSERT(tlsp->tm_memsz <= tlsp->tm_stattlsoffset); 136 if (tlsp->tm_filesz) 137 (void) _private_memcpy(data_end-tlsp->tm_stattlsoffset, 138 tlsp->tm_tlsblock, tlsp->tm_filesz); 139 if (max_modid < tlsp->tm_modid) 140 max_modid = tlsp->tm_modid; 141 } 142 /* 143 * Record the static TLS_modinfo information. 144 */ 145 modinfo = tls_modinfo_alloc(tlsm, max_modid); 146 for (tlspp = tlslist; (tlsp = *tlspp) != NULL; tlspp++) 147 modinfo[tlsp->tm_modid] = *tlsp; 148 149 /* 150 * Copy the new tls_metadata back to the old, if any, 151 * since it will be copied up again in libc_init(). 152 */ 153 if (oldself != NULL) 154 oldself->ul_uberdata->tls_metadata = *tlsm; 155 } 156 157 /* 158 * This is called from the dynamic linker for each module not included 159 * in the static TLS mod list, after the module has been loaded but 160 * before any of the module's init code has been executed. 161 */ 162 void 163 __tls_mod_add(TLS_modinfo *tlsp) 164 { 165 tls_metadata_t *tlsm = &curthread->ul_uberdata->tls_metadata; 166 ulong_t moduleid = tlsp->tm_modid; 167 TLS_modinfo *modinfo; 168 169 lmutex_lock(&tlsm->tls_lock); 170 ASSERT(!(tlsp->tm_flags & TM_FLG_STATICTLS)); 171 ASSERT(tlsp->tm_filesz <= tlsp->tm_memsz); 172 modinfo = tls_modinfo_alloc(tlsm, moduleid); 173 modinfo[moduleid] = *tlsp; 174 lmutex_unlock(&tlsm->tls_lock); 175 } 176 177 /* 178 * Called for each module as it is unloaded from memory by dlclose(). 179 */ 180 void 181 __tls_mod_remove(TLS_modinfo *tlsp) 182 { 183 tls_metadata_t *tlsm = &curthread->ul_uberdata->tls_metadata; 184 ulong_t moduleid = tlsp->tm_modid; 185 TLS_modinfo *modinfo; 186 187 lmutex_lock(&tlsm->tls_lock); 188 ASSERT(tlsm->tls_modinfo.tls_data != NULL && 189 moduleid < tlsm->tls_modinfo.tls_size); 190 modinfo = tlsm->tls_modinfo.tls_data; 191 (void) _private_memset(&modinfo[moduleid], 0, sizeof (TLS_modinfo)); 192 lmutex_unlock(&tlsm->tls_lock); 193 } 194 195 extern int _preexec_exit_handlers(); 196 extern void libc_init(); 197 198 const Lc_interface tls_rtldinfo[] = { 199 {CI_VERSION, (int(*)())CI_V_CURRENT}, 200 {CI_ATEXIT, (int(*)())_preexec_exit_handlers}, 201 {CI_TLS_MODADD, (int(*)())__tls_mod_add}, 202 {CI_TLS_MODREM, (int(*)())__tls_mod_remove}, 203 {CI_TLS_STATMOD, (int(*)())__tls_static_mods}, 204 {CI_THRINIT, (int(*)())libc_init}, 205 {CI_NULL, (int(*)())NULL} 206 }; 207 208 /* 209 * Return the address of a TLS variable for the current thread. 210 * Run the constructors for newly-allocated dynamic TLS. 211 */ 212 void * 213 slow_tls_get_addr(TLS_index *tls_index) 214 { 215 ulwp_t *self = curthread; 216 tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata; 217 TLS_modinfo *tlsp; 218 ulong_t moduleid; 219 tls_t *tlsent; 220 caddr_t base; 221 void (**initarray)(void); 222 ulong_t arraycnt = 0; 223 224 /* 225 * Defer signals until we have finished calling 226 * all of the constructors. 227 */ 228 sigoff(self); 229 lmutex_lock(&tlsm->tls_lock); 230 if ((moduleid = tls_index->ti_moduleid) < self->ul_ntlsent) 231 tlsent = self->ul_tlsent; 232 else { 233 ASSERT(moduleid < tlsm->tls_modinfo.tls_size); 234 tlsent = lmalloc(tlsm->tls_modinfo.tls_size * sizeof (tls_t)); 235 if (self->ul_tlsent != NULL) { 236 (void) _private_memcpy(tlsent, self->ul_tlsent, 237 self->ul_ntlsent * sizeof (tls_t)); 238 lfree(self->ul_tlsent, 239 self->ul_ntlsent * sizeof (tls_t)); 240 } 241 self->ul_tlsent = tlsent; 242 self->ul_ntlsent = tlsm->tls_modinfo.tls_size; 243 } 244 tlsent += moduleid; 245 if ((base = tlsent->tls_data) == NULL) { 246 tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid; 247 if (tlsp->tm_memsz == 0) { /* dlclose()d module? */ 248 base = NULL; 249 } else if (tlsp->tm_flags & TM_FLG_STATICTLS) { 250 /* static TLS is already allocated/initialized */ 251 base = (caddr_t)self - tlsp->tm_stattlsoffset; 252 tlsent->tls_data = base; 253 tlsent->tls_size = 0; /* don't lfree() this space */ 254 } else { 255 /* allocate/initialize the dynamic TLS */ 256 base = lmalloc(tlsp->tm_memsz); 257 if (tlsp->tm_filesz != 0) 258 (void) _private_memcpy(base, tlsp->tm_tlsblock, 259 tlsp->tm_filesz); 260 tlsent->tls_data = base; 261 tlsent->tls_size = tlsp->tm_memsz; 262 /* remember the constructors */ 263 arraycnt = tlsp->tm_tlsinitarraycnt; 264 initarray = tlsp->tm_tlsinitarray; 265 } 266 } 267 lmutex_unlock(&tlsm->tls_lock); 268 269 /* 270 * Call constructors, if any, in ascending order. 271 * We have to do this after dropping tls_lock because 272 * we have no idea what the constructors will do. 273 * At least we have signals deferred until they are done. 274 */ 275 if (arraycnt) { 276 do { 277 (**initarray++)(); 278 } while (--arraycnt != 0); 279 } 280 281 sigon(self); 282 return (base + tls_index->ti_tlsoffset); 283 } 284 285 #ifdef TLS_GET_ADDR_IS_WRITTEN_IN_ASSEMBLER 286 /* 287 * For speed, we do not make reference to any static data in this function. 288 * If necessary to do so, we do a tail call to slow_tls_get_addr(). 289 */ 290 void * 291 __tls_get_addr(TLS_index *tls_index) 292 { 293 ulwp_t *self = curthread; 294 tls_t *tlsent = self->ul_tlsent; 295 ulong_t moduleid; 296 caddr_t base; 297 298 if ((moduleid = tls_index->ti_moduleid) < self->ul_ntlsent && 299 (base = tlsent[moduleid].tls_data) != NULL) 300 return (base + tls_index->ti_tlsoffset); 301 302 return (slow_tls_get_addr(tls_index)); 303 } 304 #endif /* TLS_GET_ADDR_IS_WRITTEN_IN_ASSEMBLER */ 305 306 /* 307 * This is called by _thr_setup() to initialize the thread's static TLS. 308 * Constructors for initially allocated static TLS are called here. 309 */ 310 void 311 tls_setup() 312 { 313 ulwp_t *self = curthread; 314 tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata; 315 TLS_modinfo *tlsp; 316 long moduleid; 317 ulong_t nmods; 318 319 if (tlsm->static_tls.tls_size == 0) /* no static TLS */ 320 return; 321 322 /* static TLS initialization */ 323 (void) _private_memcpy((caddr_t)self - tlsm->static_tls.tls_size, 324 tlsm->static_tls.tls_data, tlsm->static_tls.tls_size); 325 326 /* call TLS constructors for the static TLS just initialized */ 327 lmutex_lock(&tlsm->tls_lock); 328 nmods = tlsm->tls_modinfo.tls_size; 329 for (moduleid = 0; moduleid < nmods; moduleid++) { 330 /* 331 * Resume where we left off in the module array. 332 * tls_modinfo.tls_data may have changed since we 333 * dropped and reacquired tls_lock, but TLS modules 334 * retain their positions in the new array. 335 */ 336 tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid; 337 /* 338 * Call constructors for this module if there are any 339 * to be called and if it is part of the static TLS. 340 */ 341 if (tlsp->tm_tlsinitarraycnt != 0 && 342 (tlsp->tm_flags & TM_FLG_STATICTLS)) { 343 ulong_t arraycnt = tlsp->tm_tlsinitarraycnt; 344 void (**initarray)(void) = tlsp->tm_tlsinitarray; 345 346 /* 347 * Call the constructors in ascending order. 348 * We must drop tls_lock while doing this because 349 * we have no idea what the constructors will do. 350 */ 351 lmutex_unlock(&tlsm->tls_lock); 352 do { 353 (**initarray++)(); 354 } while (--arraycnt != 0); 355 lmutex_lock(&tlsm->tls_lock); 356 } 357 } 358 lmutex_unlock(&tlsm->tls_lock); 359 } 360 361 /* 362 * This is called by _thrp_exit() to deallocate the thread's TLS. 363 * Destructors for all allocated TLS are called here. 364 */ 365 void 366 tls_exit() 367 { 368 ulwp_t *self = curthread; 369 tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata; 370 tls_t *tlsent; 371 TLS_modinfo *tlsp; 372 long moduleid; 373 ulong_t nmods; 374 375 if (tlsm->static_tls.tls_size == 0 && self->ul_ntlsent == 0) 376 return; /* no TLS */ 377 378 /* 379 * Call TLS destructors for all TLS allocated for this thread. 380 */ 381 lmutex_lock(&tlsm->tls_lock); 382 nmods = tlsm->tls_modinfo.tls_size; 383 for (moduleid = nmods - 1; moduleid >= 0; --moduleid) { 384 /* 385 * Resume where we left off in the module array. 386 * tls_modinfo.tls_data may have changed since we 387 * dropped and reacquired tls_lock, but TLS modules 388 * retain their positions in the new array. 389 */ 390 tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid; 391 /* 392 * Call destructors for this module if there are any 393 * to be called and if it is part of the static TLS or 394 * if the dynamic TLS for the module has been allocated. 395 */ 396 if (tlsp->tm_tlsfiniarraycnt != 0 && 397 ((tlsp->tm_flags & TM_FLG_STATICTLS) || 398 (moduleid < self->ul_ntlsent && 399 (tlsent = self->ul_tlsent) != NULL && 400 tlsent[moduleid].tls_data != NULL))) { 401 ulong_t arraycnt = tlsp->tm_tlsfiniarraycnt; 402 void (**finiarray)(void) = tlsp->tm_tlsfiniarray; 403 404 /* 405 * Call the destructors in descending order. 406 * We must drop tls_lock while doing this because 407 * we have no idea what the destructors will do. 408 */ 409 lmutex_unlock(&tlsm->tls_lock); 410 finiarray += arraycnt; 411 do { 412 (**--finiarray)(); 413 } while (--arraycnt != 0); 414 lmutex_lock(&tlsm->tls_lock); 415 } 416 } 417 lmutex_unlock(&tlsm->tls_lock); 418 419 tls_free(self); 420 } 421 422 /* 423 * We only free the dynamically allocated TLS; the statically 424 * allocated TLS is reused when the ulwp_t is reallocated. 425 */ 426 void 427 tls_free(ulwp_t *ulwp) 428 { 429 ulong_t moduleid; 430 tls_t *tlsent; 431 size_t ntlsent; 432 void *base; 433 size_t size; 434 435 if ((tlsent = ulwp->ul_tlsent) == NULL || 436 (ntlsent = ulwp->ul_ntlsent) == 0) 437 return; 438 439 for (moduleid = 0; moduleid < ntlsent; moduleid++, tlsent++) { 440 if ((base = tlsent->tls_data) != NULL && 441 (size = tlsent->tls_size) != 0) 442 lfree(base, size); 443 tlsent->tls_data = NULL; /* paranoia */ 444 tlsent->tls_size = 0; 445 } 446 lfree(ulwp->ul_tlsent, ntlsent * sizeof (tls_t)); 447 ulwp->ul_tlsent = NULL; 448 ulwp->ul_ntlsent = 0; 449 } 450