1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2014 Garrett D'Amore <garrett@damore.org>
14 * Copyright 2025 Oxide Computer Company
15 * Copyright 2026 Bill Sommerfeld <sommerfeld@hamachi.org>
16 */
17
18 /*
19 * This file implements the 2008 newlocale and friends handling.
20 */
21
22 #ifndef _LCONV_C99
23 #define _LCONV_C99
24 #endif
25
26 #include "lint.h"
27 #include <atomic.h>
28 #include <locale.h>
29 #include <sys/types.h>
30 #include <sys/mman.h>
31 #include <errno.h>
32 #include <string.h>
33 #include "libc.h"
34 #include "mtlib.h"
35 #include "tsd.h"
36 #include "localeimpl.h"
37 #include "lctype.h"
38
39 /*
40 * Big Theory of Locales:
41 *
42 * (It is recommended that readers familiarize themselves with the POSIX
43 * 2008 (XPG Issue 7) specifications for locales, first.)
44 *
45 * Historically, we had a bunch of global variables that stored locale
46 * data. While this worked well, it limited applications to a single locale
47 * at a time. This doesn't work well in certain server applications.
48 *
49 * Issue 7, X/Open introduced the concept of a locale_t object, along with
50 * versions of functions that can take this object as a parameter, along
51 * with functions to clone and manipulate these locale objects. The new
52 * functions are named with a _l() suffix.
53 *
54 * Additionally uselocale() is introduced which can change the locale of
55 * of a single thread. However, setlocale() can still be used to change
56 * the global locale.
57 *
58 * In our implementation, we use libc's TSD to store the locale data that
59 * was previously global. We still have global data because some applications
60 * have had those global objects compiled into them. (Such applications will
61 * be unable to benefit from uselocale(), btw.) The legacy routines are
62 * reimplemented as wrappers that retrieve the appropriate locale object by
63 * calling __curlocale(). Note that once the TSD data is set, the only way
64 * to revert to the global locale is to pass the global locale LC_GLOBAL_LOCALE
65 * to uselocale().
66 *
67 * We are careful to minimize performance impact of multiple calls to
68 * uselocale() or setlocale() by using a cache of locale data whenever possible.
69 * As a consequence of this, applications that iterate over all possible
70 * locales will burn through a lot of virtual memory, but we find such
71 * applications rare. (locale -a might be an exception, but it is short lived.)
72 *
73 * Category data is never released (although enclosing locale objects might be),
74 * in order to guarantee thread-safety. Calling freelocale() on an object
75 * while it is in use by another thread is a programmer error (use-after-free)
76 * and we don't bother to note it further.
77 *
78 * Locale objects (global locales) established by setlocale() are also
79 * never freed (for MT safety), but we will save previous locale objects
80 * and reuse them when we can.
81 *
82 * We once used the tsdalloc() destructor callback to free
83 * thread-specific locales on thread exit, but that is at odds with
84 * both the POSIX spec and the behavior of other implementations.
85 */
86
87 typedef struct locdata *(*loadfn_t)(const char *);
88
89 static const loadfn_t loaders[LC_ALL] = {
90 __lc_ctype_load,
91 __lc_numeric_load,
92 __lc_time_load,
93 __lc_collate_load,
94 __lc_monetary_load,
95 __lc_messages_load,
96 };
97
98 extern struct lc_monetary lc_monetary_posix;
99 extern struct lc_numeric lc_numeric_posix;
100 extern struct lc_messages lc_messages_posix;
101 extern struct lc_time lc_time_posix;
102 extern struct lc_ctype lc_ctype_posix;
103 extern struct lc_collate lc_collate_posix;
104 extern struct _RuneLocale _DefaultRuneLocale;
105
106 static struct _locale posix_locale = {
107 /* locdata */
108 .locdata = {
109 &__posix_ctype_locdata,
110 &__posix_numeric_locdata,
111 &__posix_time_locdata,
112 &__posix_collate_locdata,
113 &__posix_monetary_locdata,
114 &__posix_messages_locdata,
115 },
116 .locname = "C",
117 .ctype = &lc_ctype_posix,
118 .numeric = &lc_numeric_posix,
119 .collate = &lc_collate_posix,
120 .monetary = &lc_monetary_posix,
121 .messages = &lc_messages_posix,
122 .time = &lc_time_posix,
123 .runelocale = &_DefaultRuneLocale,
124 };
125
126 locale_t ___global_locale = &posix_locale;
127
128 locale_t
__global_locale(void)129 __global_locale(void)
130 {
131 return (___global_locale);
132 }
133
134 /*
135 * Locale data for hybrid C.UTF-8 locale having all the characteristics of
136 * default C/POSIX locale, except for LC_CTYPE data which is retrieved from
137 * cache/file as for other UTF-8 locales.
138 */
139 static struct locdata cutf_locdata[LC_ALL] = {
140 { "C.UTF-8", NULL }, /* unused */
141 { "C.UTF-8", &lc_numeric_posix },
142 { "C.UTF-8", &lc_time_posix },
143 { "C.UTF-8", &lc_collate_posix },
144 { "C.UTF-8", &lc_monetary_posix },
145 { "C.UTF-8", &lc_messages_posix },
146 };
147
148 /*
149 * Category names for getenv() Note that this was modified
150 * for Solaris. See <iso/locale_iso.h>.
151 */
152 #define NUM_CATS 7
153 static char *categories[7] = {
154 "LC_CTYPE",
155 "LC_NUMERIC",
156 "LC_TIME",
157 "LC_COLLATE",
158 "LC_MONETARY",
159 "LC_MESSAGES",
160 "LC_ALL",
161 };
162
163 /*
164 * Prototypes.
165 */
166 static const char *get_locale_env(int);
167 static struct locdata *locdata_get(int, const char *);
168 static struct locdata *locdata_get_cache(int, const char *);
169 static locale_t mklocname(locale_t);
170
171 /*
172 * Some utility routines.
173 */
174
175 struct locdata *
__locdata_alloc(const char * name,size_t memsz)176 __locdata_alloc(const char *name, size_t memsz)
177 {
178 struct locdata *ldata;
179
180 if ((ldata = lmalloc(sizeof (*ldata))) == NULL) {
181 return (NULL);
182 }
183 if ((ldata->l_data[0] = libc_malloc(memsz)) == NULL) {
184 lfree(ldata, sizeof (*ldata));
185 errno = ENOMEM;
186 return (NULL);
187 }
188 (void) strlcpy(ldata->l_lname, name, sizeof (ldata->l_lname));
189
190 return (ldata);
191 }
192
193 /*
194 * Normally we never free locale data truly, but if we failed to load it
195 * for some reason, this routine is used to cleanup the partial mess.
196 */
197 void
__locdata_free(struct locdata * ldata)198 __locdata_free(struct locdata *ldata)
199 {
200 for (int i = 0; i < NLOCDATA; i++)
201 libc_free(ldata->l_data[i]);
202 if (ldata->l_map != NULL && ldata->l_map_len)
203 (void) munmap(ldata->l_map, ldata->l_map_len);
204 lfree(ldata, sizeof (*ldata));
205 }
206
207 /*
208 * It turns out that for performance reasons we would really like to
209 * cache the most recently referenced locale data to avoid wasteful
210 * loading from files.
211 */
212
213 static struct locdata *cache_data[LC_ALL];
214 static struct locdata *cat_data[LC_ALL];
215 static mutex_t cache_lock = DEFAULTMUTEX;
216
217 /*
218 * Returns the cached data if the locale name is the same. If not,
219 * returns NULL (cache miss). The locdata is returned with a hold on
220 * it, taken on behalf of the caller. The caller should drop the hold
221 * when it is finished.
222 */
223 static struct locdata *
locdata_get_cache(int category,const char * locname)224 locdata_get_cache(int category, const char *locname)
225 {
226 struct locdata *loc;
227
228 if (category < 0 || category >= LC_ALL)
229 return (NULL);
230
231 /* Try cache first. */
232 lmutex_lock(&cache_lock);
233 loc = cache_data[category];
234
235 if ((loc != NULL) && (strcmp(loc->l_lname, locname) == 0)) {
236 lmutex_unlock(&cache_lock);
237 return (loc);
238 }
239
240 /*
241 * Failing that try previously loaded locales (linear search) --
242 * this could be optimized to a hash, but its unlikely that a single
243 * application will ever need to work with more than a few locales.
244 */
245 for (loc = cat_data[category]; loc != NULL; loc = loc->l_next) {
246 if (strcmp(locname, loc->l_lname) == 0) {
247 break;
248 }
249 }
250
251 /*
252 * Finally, if we still don't have one, try loading the locale
253 * data from the actual on-disk data.
254 *
255 * We drop the lock (libc wants to ensure no internal locks
256 * are held when we call other routines required to read from
257 * files, allocate memory, etc.) There is a small race here,
258 * but the consequences of the race are benign -- if multiple
259 * threads hit this at precisely the same point, we could
260 * wind up with duplicates of the locale data in the cache.
261 *
262 * This wastes the memory for an extra copy of the locale
263 * data, but there is no further harm beyond that. Its not
264 * worth the effort to recode this to something "safe"
265 * (which would require rescanning the list, etc.), given
266 * that this race will probably never actually occur.
267 */
268 if (loc == NULL) {
269 lmutex_unlock(&cache_lock);
270 loc = (*loaders[category])(locname);
271 lmutex_lock(&cache_lock);
272 if (loc != NULL)
273 (void) strlcpy(loc->l_lname, locname,
274 sizeof (loc->l_lname));
275 }
276
277 /*
278 * Assuming we got one, update the cache, and stick us on the list
279 * of loaded locale data. We insert into the head (more recent
280 * use is likely to win.)
281 */
282 if (loc != NULL) {
283 cache_data[category] = loc;
284 if (!loc->l_cached) {
285 loc->l_cached = 1;
286 loc->l_next = cat_data[category];
287 cat_data[category] = loc;
288 }
289 }
290
291 lmutex_unlock(&cache_lock);
292 return (loc);
293 }
294
295 /* Charmap aliases, mostly found in Linux */
296 static const struct {
297 const char *alias;
298 const char *name;
299 } cmalias[] = {
300 { "utf8", "UTF-8" },
301 { "iso88591", "ISO8859-1" },
302 { "iso885915", "ISO8859-15" },
303 { "gb18030", "GB18030" },
304 { "koi8r", "KOI8-R" },
305 { NULL, NULL }
306 };
307
308 /*
309 * Routine to get the locdata for a given category and locale.
310 * This includes retrieving it from cache, retrieving it from
311 * a file, etc.
312 */
313 static struct locdata *
locdata_get(int category,const char * locname)314 locdata_get(int category, const char *locname)
315 {
316 char scratch[ENCODING_LEN + 1];
317 char scratch2[ENCODING_LEN + 1];
318 const char *sep, *cm;
319 int cnt;
320 int len;
321 int i;
322
323 if (locname == NULL || *locname == 0) {
324 locname = get_locale_env(category);
325 }
326
327 /*
328 * Extract the locale name for the category if it is a composite
329 * locale.
330 */
331 if ((sep = strchr(locname, '/')) != NULL) {
332 for (cnt = category; cnt && sep != NULL; cnt--) {
333 locname = sep + 1;
334 sep = strchr(locname, '/');
335 }
336 if (sep) {
337 len = sep - locname + 1;
338 if (len >= sizeof (scratch)) {
339 len = sizeof (scratch);
340 }
341 } else {
342 len = sizeof (scratch);
343 }
344 (void) strlcpy(scratch, locname, len);
345 locname = scratch;
346 } else if ((sep = strchr(locname, ';')) != NULL) {
347 /*
348 * Accept glibc-style composite locale as libstdc++ expects.
349 * glibc names composite locales using a semicolon-
350 * separated list of <category>=<value> assignments.
351 *
352 * Segment the string at semicolons, checking if each segment
353 * starts with the name of the category we're looking for,
354 * followed by an equals sign. Ignore everything else.
355 */
356 const char *catname = categories[category];
357 size_t catlen = strlen(catname);
358 const char *locnameend = locname + strlen(locname);
359 const char *locp = locname; /* start of this entry */
360 const char *endp = sep; /* end of this entry */
361 const char *value = NULL; /* start of match */
362 const char *endvalue = NULL; /* end of match */
363 size_t copylen;
364
365 for (;;) {
366 if (((endp - locp) > catlen) &&
367 (memcmp(locp, catname, catlen) == 0) &&
368 (locp[catlen] == '=')) {
369 value = &locp[catlen + 1];
370 endvalue = endp;
371 }
372 if (endp >= locnameend)
373 break;
374 locp = endp + 1;
375 endp = strchr(locp, ';');
376 if (endp == NULL)
377 endp = locnameend;
378 }
379
380 /* match glibc errno */
381 if (value == NULL) {
382 errno = EINVAL;
383 return (NULL);
384 }
385 copylen = endvalue - value;
386 /*
387 * As real locale names are all shorter
388 * than this, anything longer is
389 * unparsable garbage. Return failure.
390 */
391 if (copylen > ENCODING_LEN) {
392 errno = ENOENT;
393 return (NULL);
394 }
395 if (copylen == 0) {
396 /*
397 * glibc falls through to the
398 * environment, so we must too..
399 */
400 locname = get_locale_env(category);
401 } else {
402 memcpy(scratch, value, copylen);
403 scratch[copylen] = 0;
404 locname = scratch;
405 }
406 }
407
408 if ((strcmp(locname, "C") == 0) || (strcmp(locname, "POSIX") == 0))
409 return (posix_locale.locdata[category]);
410
411 /* Handle charmap aliases */
412 for (i = 0; cmalias[i].alias != NULL; i++) {
413 if ((cm = strstr(locname, cmalias[i].alias)) != NULL &&
414 strlen(cm) == strlen(cmalias[i].alias)) {
415 len = cm - locname + 1;
416 if (len + strlen(cmalias[i].name) >= sizeof (scratch2))
417 break;
418 (void) strlcpy(scratch2, locname, len);
419 (void) strlcat(scratch2, cmalias[i].name,
420 sizeof (scratch2));
421 locname = scratch2;
422 break;
423 }
424 }
425
426 if ((strcmp(locname, "C.UTF-8") == 0) && (category != LC_CTYPE))
427 return (&cutf_locdata[category]);
428
429 return (locdata_get_cache(category, locname));
430 }
431
432 static const char *
get_locale_env(int category)433 get_locale_env(int category)
434 {
435 const char *env;
436
437 /* 1. check LC_ALL. */
438 env = getenv(categories[LC_ALL]);
439
440 /* 2. check LC_* */
441 if (env == NULL || *env == '\0')
442 env = getenv(categories[category]);
443
444 /* 3. check LANG */
445 if (env == NULL || *env == '\0')
446 env = getenv("LANG");
447
448 /* 4. if none is set, fall to "C" */
449 if (env == NULL || *env == '\0')
450 env = "C";
451
452 return (env);
453 }
454
455
456 /*
457 * This routine is exposed via the MB_CUR_MAX macro. Note that legacy
458 * code will continue to use _ctype[520], but we prefer this function as
459 * it is the only way to get thread-specific information.
460 */
461 unsigned char
__mb_cur_max_l(locale_t loc)462 __mb_cur_max_l(locale_t loc)
463 {
464 return (loc->ctype->lc_max_mblen);
465 }
466
467 unsigned char
__mb_cur_max(void)468 __mb_cur_max(void)
469 {
470 return (__mb_cur_max_l(__curlocale()));
471 }
472
473 /*
474 * Do everything but set locname; used by duplocale and
475 * newlocale.
476 */
477 static locale_t
duplocale_noname(locale_t src)478 duplocale_noname(locale_t src)
479 {
480 locale_t loc;
481 int i;
482
483 loc = lmalloc(sizeof (*loc));
484 if (loc == NULL) {
485 return (NULL);
486 }
487 if (src == NULL) {
488 /* illumos extension: POSIX says LC_GLOBAL_LOCALE here */
489 src = ___global_locale;
490 }
491 for (i = 0; i < LC_ALL; i++) {
492 loc->locdata[i] = src->locdata[i];
493 loc->loaded[i] = 0;
494 }
495 loc->collate = loc->locdata[LC_COLLATE]->l_data[0];
496 loc->ctype = loc->locdata[LC_CTYPE]->l_data[0];
497 loc->runelocale = loc->locdata[LC_CTYPE]->l_data[1];
498 loc->messages = loc->locdata[LC_MESSAGES]->l_data[0];
499 loc->monetary = loc->locdata[LC_MONETARY]->l_data[0];
500 loc->numeric = loc->locdata[LC_NUMERIC]->l_data[0];
501 loc->time = loc->locdata[LC_TIME]->l_data[0];
502 return (loc);
503 }
504
505 /*
506 * Public interfaces.
507 */
508 locale_t
duplocale(locale_t src)509 duplocale(locale_t src)
510 {
511 locale_t loc = duplocale_noname(src);
512 if (loc == NULL)
513 return (NULL);
514 return (mklocname(loc));
515 }
516
517 void
freelocale(locale_t loc)518 freelocale(locale_t loc)
519 {
520 /*
521 * We take extra care never to free a saved locale created by
522 * setlocale(). This shouldn't be strictly necessary, but a little
523 * extra safety doesn't hurt here.
524 */
525 if ((loc != NULL) && (loc != &posix_locale) && (!loc->on_list))
526 lfree(loc, sizeof (*loc));
527 }
528
529 locale_t
newlocale(int catmask,const char * locname,locale_t base)530 newlocale(int catmask, const char *locname, locale_t base)
531 {
532 locale_t loc;
533 int i, e;
534
535 if (catmask & ~(LC_ALL_MASK)) {
536 errno = EINVAL;
537 return (NULL);
538 }
539
540 /*
541 * Technically passing LC_GLOBAL_LOCALE here is illegal,
542 * but we allow it.
543 */
544 if (base == NULL || base == ___global_locale) {
545 loc = duplocale_noname(___global_locale);
546 } else {
547 loc = duplocale_noname(base);
548 }
549 if (loc == NULL) {
550 return (NULL);
551 }
552
553 for (i = 0; i < LC_ALL; i++) {
554 struct locdata *ldata;
555 loc->loaded[i] = 0;
556 if (((1 << i) & catmask) == 0) {
557 /* Default to base locale if not overriding */
558 continue;
559 }
560 ldata = locdata_get(i, locname);
561 if (ldata == NULL) {
562 e = errno;
563 freelocale(loc);
564 errno = e;
565 return (NULL);
566 }
567 loc->locdata[i] = ldata;
568 }
569 loc->collate = loc->locdata[LC_COLLATE]->l_data[0];
570 loc->ctype = loc->locdata[LC_CTYPE]->l_data[0];
571 loc->runelocale = loc->locdata[LC_CTYPE]->l_data[1];
572 loc->messages = loc->locdata[LC_MESSAGES]->l_data[0];
573 loc->monetary = loc->locdata[LC_MONETARY]->l_data[0];
574 loc->numeric = loc->locdata[LC_NUMERIC]->l_data[0];
575 loc->time = loc->locdata[LC_TIME]->l_data[0];
576 freelocale(base);
577
578 return (mklocname(loc));
579 }
580
581 /*
582 * Transitional implementation of __curlocale() based on code from uselocale().
583 * This is a placeholder to introduce this interface into libc.
584 *
585 * A future change will move the locale pointer into the ulwp_t structure and
586 * make __curlocale() an inline function.
587 */
588 locale_t
__curlocale(void)589 __curlocale(void)
590 {
591 locale_t *locptr;
592
593 locptr = tsdalloc(_T_SETLOCALE, sizeof (locale_t), NULL);
594 /* Should never occur */
595 if (locptr == NULL) {
596 errno = EINVAL;
597 return (NULL);
598 }
599
600 if (*locptr != NULL)
601 return (*locptr);
602
603 return (___global_locale);
604 }
605
606 locale_t
uselocale(locale_t loc)607 uselocale(locale_t loc)
608 {
609 locale_t lastloc = ___global_locale;
610 locale_t *locptr;
611
612 locptr = tsdalloc(_T_SETLOCALE, sizeof (locale_t), NULL);
613 /* Should never occur */
614 if (locptr == NULL) {
615 errno = EINVAL;
616 return (NULL);
617 }
618
619 if (*locptr != NULL)
620 lastloc = *locptr;
621
622 /* Argument loc is NULL if we are just querying. */
623 if (loc != NULL) {
624 /*
625 * Set it to LC_GLOBAL_LOCAL to return to using
626 * the global locale (setlocale).
627 */
628 if (loc == ___global_locale) {
629 *locptr = NULL;
630 } else {
631 /* No validation of the provided locale at present */
632 *locptr = loc;
633 }
634 }
635
636 /*
637 * The caller is responsible for freeing, of course it would be
638 * gross error to call freelocale() on a locale object that is still
639 * in use.
640 */
641 return (lastloc);
642 }
643
644 static locale_t
mklocname(locale_t loc)645 mklocname(locale_t loc)
646 {
647 int composite = 0;
648
649 /* Look to see if any category is different */
650 for (int i = 1; i < LC_ALL; ++i) {
651 if (strcmp(loc->locdata[0]->l_lname,
652 loc->locdata[i]->l_lname) != 0) {
653 composite = 1;
654 break;
655 }
656 }
657
658 if (composite) {
659 /*
660 * Note ordering of these follows the numeric order,
661 * if the order is changed, then setlocale() will need
662 * to be changed as well.
663 */
664 (void) snprintf(loc->locname, sizeof (loc->locname),
665 "%s/%s/%s/%s/%s/%s",
666 loc->locdata[LC_CTYPE]->l_lname,
667 loc->locdata[LC_NUMERIC]->l_lname,
668 loc->locdata[LC_TIME]->l_lname,
669 loc->locdata[LC_COLLATE]->l_lname,
670 loc->locdata[LC_MONETARY]->l_lname,
671 loc->locdata[LC_MESSAGES]->l_lname);
672 } else {
673 (void) strlcpy(loc->locname, loc->locdata[LC_CTYPE]->l_lname,
674 sizeof (loc->locname));
675 }
676 return (loc);
677 }
678
679 /*
680 * POSIX has several lifetime requirements that vary on the type of locale.
681 *
682 * If the locale is LC_GLOBAL_LOCALE, the returned string is required to live
683 * beyond the locale's use as the global locale. The specification suggests that
684 * this use a thread-local buffer and cautions that it may disappear when the
685 * thread terminates or another LC_GLOBAL_LOCALE call is made. In our case,
686 * because we will never free a locale that is set with setlocale() (see
687 * port/locale/setlocale.c), we can simply return the name of the locale
688 * directly.
689 *
690 * If the locale is any other locale, it is allowed to be invalidated by a call
691 * to uselocale() or newlocale().
692 *
693 * In both of these cases this means that we can simply return the string from
694 * the current object. POSIX importantly states that the application is not
695 * allowed to assume the name will stay the same across invocations and
696 * therefore it cannot be relied upon for serialization. However, it will work
697 * with setlocale() again.
698 */
699 const char *
getlocalename_l(int category,locale_t loc)700 getlocalename_l(int category, locale_t loc)
701 {
702 if (loc == NULL) {
703 return (NULL);
704 }
705
706 switch (category) {
707 case LC_CTYPE:
708 case LC_NUMERIC:
709 case LC_TIME:
710 case LC_COLLATE:
711 case LC_MONETARY:
712 case LC_MESSAGES:
713 return (loc->locdata[category]->l_lname);
714 case LC_ALL:
715 return (loc->locname);
716 default:
717 /*
718 * POSIX does not define any errors here so we can't indicate
719 * anything via errno or similar.
720 */
721 return (NULL);
722 }
723 }
724