1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 2014 Garrett D'Amore <garrett@damore.org> 5 * Copyright 2010 Nexenta Systems, Inc. All rights reserved. 6 * Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua> 7 * at Electronni Visti IA, Kiev, Ukraine. 8 * All rights reserved. 9 * 10 * Copyright (c) 2011 The FreeBSD Foundation 11 * All rights reserved. 12 * Portions of this software were developed by David Chisnall 13 * under sponsorship from the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * Adapted to xlocale by John Marino <draco@marino.st> 37 */ 38 39 #include <sys/cdefs.h> 40 __FBSDID("$FreeBSD$"); 41 42 #include "namespace.h" 43 44 #include <sys/types.h> 45 #include <sys/stat.h> 46 #include <sys/mman.h> 47 48 #include <assert.h> 49 #include <stdio.h> 50 #include <stdlib.h> 51 #include <string.h> 52 #include <wchar.h> 53 #include <errno.h> 54 #include <unistd.h> 55 #include <fcntl.h> 56 #include "un-namespace.h" 57 58 #include "collate.h" 59 #include "setlocale.h" 60 #include "ldpart.h" 61 #include "libc_private.h" 62 63 struct xlocale_collate __xlocale_global_collate = { 64 {{0}, "C"}, 1, 0, 0, 0 65 }; 66 67 struct xlocale_collate __xlocale_C_collate = { 68 {{0}, "C"}, 1, 0, 0, 0 69 }; 70 71 static int 72 __collate_load_tables_l(const char *encoding, struct xlocale_collate *table); 73 74 static void 75 destruct_collate(void *t) 76 { 77 struct xlocale_collate *table = t; 78 if (table->map && (table->maplen > 0)) { 79 (void) munmap(table->map, table->maplen); 80 } 81 free(t); 82 } 83 84 void * 85 __collate_load(const char *encoding, __unused locale_t unused) 86 { 87 if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0 || 88 strncmp(encoding, "C.", 2) == 0) { 89 return &__xlocale_C_collate; 90 } 91 struct xlocale_collate *table = calloc(sizeof(struct xlocale_collate), 1); 92 table->header.header.destructor = destruct_collate; 93 // FIXME: Make sure that _LDP_CACHE is never returned. We should be doing 94 // the caching outside of this section 95 if (__collate_load_tables_l(encoding, table) != _LDP_LOADED) { 96 xlocale_release(table); 97 return NULL; 98 } 99 return table; 100 } 101 102 /** 103 * Load the collation tables for the specified encoding into the global table. 104 */ 105 int 106 __collate_load_tables(const char *encoding) 107 { 108 109 return (__collate_load_tables_l(encoding, &__xlocale_global_collate)); 110 } 111 112 static int 113 __collate_load_tables_l(const char *encoding, struct xlocale_collate *table) 114 { 115 int i, chains, z; 116 char *buf; 117 char *TMP; 118 char *map; 119 collate_info_t *info; 120 struct stat sbuf; 121 int fd; 122 123 table->__collate_load_error = 1; 124 125 /* 'encoding' must be already checked. */ 126 if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0 || 127 strncmp(encoding, "C.", 2) == 0) { 128 return (_LDP_CACHE); 129 } 130 131 if (asprintf(&buf, "%s/%s/LC_COLLATE", _PathLocale, encoding) == -1) 132 return (_LDP_ERROR); 133 134 if ((fd = _open(buf, O_RDONLY)) < 0) { 135 free(buf); 136 return (_LDP_ERROR); 137 } 138 free(buf); 139 if (_fstat(fd, &sbuf) < 0) { 140 (void) _close(fd); 141 return (_LDP_ERROR); 142 } 143 if (sbuf.st_size < (COLLATE_FMT_VERSION_LEN + 144 XLOCALE_DEF_VERSION_LEN + 145 sizeof (info))) { 146 (void) _close(fd); 147 errno = EINVAL; 148 return (_LDP_ERROR); 149 } 150 map = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 151 (void) _close(fd); 152 if ((TMP = map) == MAP_FAILED) { 153 return (_LDP_ERROR); 154 } 155 156 if (strncmp(TMP, COLLATE_FMT_VERSION, COLLATE_FMT_VERSION_LEN) != 0) { 157 (void) munmap(map, sbuf.st_size); 158 errno = EINVAL; 159 return (_LDP_ERROR); 160 } 161 TMP += COLLATE_FMT_VERSION_LEN; 162 strlcat(table->header.version, TMP, sizeof (table->header.version)); 163 TMP += XLOCALE_DEF_VERSION_LEN; 164 165 info = (void *)TMP; 166 TMP += sizeof (*info); 167 168 if ((info->directive_count < 1) || 169 (info->directive_count >= COLL_WEIGHTS_MAX) || 170 ((chains = info->chain_count) < 0)) { 171 (void) munmap(map, sbuf.st_size); 172 errno = EINVAL; 173 return (_LDP_ERROR); 174 } 175 176 i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) + 177 (sizeof (collate_chain_t) * chains) + 178 (sizeof (collate_large_t) * info->large_count); 179 for (z = 0; z < info->directive_count; z++) { 180 i += sizeof (collate_subst_t) * info->subst_count[z]; 181 } 182 if (i != (sbuf.st_size - (TMP - map))) { 183 (void) munmap(map, sbuf.st_size); 184 errno = EINVAL; 185 return (_LDP_ERROR); 186 } 187 188 if (table->map && (table->maplen > 0)) { 189 (void) munmap(table->map, table->maplen); 190 } 191 table->map = map; 192 table->maplen = sbuf.st_size; 193 table->info = info; 194 table->char_pri_table = (void *)TMP; 195 TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1); 196 197 for (z = 0; z < info->directive_count; z++) { 198 if (info->subst_count[z] > 0) { 199 table->subst_table[z] = (void *)TMP; 200 TMP += info->subst_count[z] * sizeof (collate_subst_t); 201 } else { 202 table->subst_table[z] = NULL; 203 } 204 } 205 206 if (chains > 0) { 207 table->chain_pri_table = (void *)TMP; 208 TMP += chains * sizeof (collate_chain_t); 209 } else 210 table->chain_pri_table = NULL; 211 if (info->large_count > 0) 212 table->large_pri_table = (void *)TMP; 213 else 214 table->large_pri_table = NULL; 215 216 table->__collate_load_error = 0; 217 return (_LDP_LOADED); 218 } 219 220 static const int32_t * 221 substsearch(struct xlocale_collate *table, const wchar_t key, int pass) 222 { 223 const collate_subst_t *p; 224 int n = table->info->subst_count[pass]; 225 226 if (n == 0) 227 return (NULL); 228 229 if (pass >= table->info->directive_count) 230 return (NULL); 231 232 if (!(key & COLLATE_SUBST_PRIORITY)) 233 return (NULL); 234 235 p = table->subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY); 236 assert(p->key == key); 237 238 return (p->pri); 239 } 240 241 static collate_chain_t * 242 chainsearch(struct xlocale_collate *table, const wchar_t *key, int *len) 243 { 244 int low = 0; 245 int high = table->info->chain_count - 1; 246 int next, compar, l; 247 collate_chain_t *p; 248 collate_chain_t *tab = table->chain_pri_table; 249 250 if (high < 0) 251 return (NULL); 252 253 while (low <= high) { 254 next = (low + high) / 2; 255 p = tab + next; 256 compar = *key - *p->str; 257 if (compar == 0) { 258 l = wcsnlen(p->str, COLLATE_STR_LEN); 259 compar = wcsncmp(key, p->str, l); 260 if (compar == 0) { 261 *len = l; 262 return (p); 263 } 264 } 265 if (compar > 0) 266 low = next + 1; 267 else 268 high = next - 1; 269 } 270 return (NULL); 271 } 272 273 static collate_large_t * 274 largesearch(struct xlocale_collate *table, const wchar_t key) 275 { 276 int low = 0; 277 int high = table->info->large_count - 1; 278 int next, compar; 279 collate_large_t *p; 280 collate_large_t *tab = table->large_pri_table; 281 282 if (high < 0) 283 return (NULL); 284 285 while (low <= high) { 286 next = (low + high) / 2; 287 p = tab + next; 288 compar = key - p->val; 289 if (compar == 0) 290 return (p); 291 if (compar > 0) 292 low = next + 1; 293 else 294 high = next - 1; 295 } 296 return (NULL); 297 } 298 299 void 300 _collate_lookup(struct xlocale_collate *table, const wchar_t *t, int *len, 301 int *pri, int which, const int **state) 302 { 303 collate_chain_t *p2; 304 collate_large_t *match; 305 int p, l; 306 const int *sptr; 307 308 /* 309 * If this is the "last" pass for the UNDEFINED, then 310 * we just return the priority itself. 311 */ 312 if (which >= table->info->directive_count) { 313 *pri = *t; 314 *len = 1; 315 *state = NULL; 316 return; 317 } 318 319 /* 320 * If we have remaining substitution data from a previous 321 * call, consume it first. 322 */ 323 if ((sptr = *state) != NULL) { 324 *pri = *sptr; 325 sptr++; 326 if ((sptr == *state) || (sptr == NULL)) 327 *state = NULL; 328 else 329 *state = sptr; 330 *len = 0; 331 return; 332 } 333 334 /* No active substitutions */ 335 *len = 1; 336 337 /* 338 * Check for composites such as diphthongs that collate as a 339 * single element (aka chains or collating-elements). 340 */ 341 if (((p2 = chainsearch(table, t, &l)) != NULL) && 342 ((p = p2->pri[which]) >= 0)) { 343 344 *len = l; 345 *pri = p; 346 347 } else if (*t <= UCHAR_MAX) { 348 349 /* 350 * Character is a small (8-bit) character. 351 * We just look these up directly for speed. 352 */ 353 *pri = table->char_pri_table[*t].pri[which]; 354 355 } else if ((table->info->large_count > 0) && 356 ((match = largesearch(table, *t)) != NULL)) { 357 358 /* 359 * Character was found in the extended table. 360 */ 361 *pri = match->pri.pri[which]; 362 363 } else { 364 /* 365 * Character lacks a specific definition. 366 */ 367 if (table->info->directive[which] & DIRECTIVE_UNDEFINED) { 368 /* Mask off sign bit to prevent ordering confusion. */ 369 *pri = (*t & COLLATE_MAX_PRIORITY); 370 } else { 371 *pri = table->info->undef_pri[which]; 372 } 373 /* No substitutions for undefined characters! */ 374 return; 375 } 376 377 /* 378 * Try substituting (expanding) the character. We are 379 * currently doing this *after* the chain compression. I 380 * think it should not matter, but this way might be slightly 381 * faster. 382 * 383 * We do this after the priority search, as this will help us 384 * to identify a single key value. In order for this to work, 385 * its important that the priority assigned to a given element 386 * to be substituted be unique for that level. The localedef 387 * code ensures this for us. 388 */ 389 if ((sptr = substsearch(table, *pri, which)) != NULL) { 390 if ((*pri = *sptr) > 0) { 391 sptr++; 392 *state = *sptr ? sptr : NULL; 393 } 394 } 395 396 } 397 398 /* 399 * This is the meaty part of wcsxfrm & strxfrm. Note that it does 400 * NOT NULL terminate. That is left to the caller. 401 */ 402 size_t 403 _collate_wxfrm(struct xlocale_collate *table, const wchar_t *src, wchar_t *xf, 404 size_t room) 405 { 406 int pri; 407 int len; 408 const wchar_t *t; 409 wchar_t *tr = NULL; 410 int direc; 411 int pass; 412 const int32_t *state; 413 size_t want = 0; 414 size_t need = 0; 415 int ndir = table->info->directive_count; 416 417 assert(src); 418 419 for (pass = 0; pass <= ndir; pass++) { 420 421 state = NULL; 422 423 if (pass != 0) { 424 /* insert level separator from the previous pass */ 425 if (room) { 426 *xf++ = 1; 427 room--; 428 } 429 want++; 430 } 431 432 /* special pass for undefined */ 433 if (pass == ndir) { 434 direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED; 435 } else { 436 direc = table->info->directive[pass]; 437 } 438 439 t = src; 440 441 if (direc & DIRECTIVE_BACKWARD) { 442 wchar_t *bp, *fp, c; 443 free(tr); 444 if ((tr = wcsdup(t)) == NULL) { 445 errno = ENOMEM; 446 goto fail; 447 } 448 bp = tr; 449 fp = tr + wcslen(tr) - 1; 450 while (bp < fp) { 451 c = *bp; 452 *bp++ = *fp; 453 *fp-- = c; 454 } 455 t = (const wchar_t *)tr; 456 } 457 458 if (direc & DIRECTIVE_POSITION) { 459 while (*t || state) { 460 _collate_lookup(table, t, &len, &pri, pass, &state); 461 t += len; 462 if (pri <= 0) { 463 if (pri < 0) { 464 errno = EINVAL; 465 goto fail; 466 } 467 state = NULL; 468 pri = COLLATE_MAX_PRIORITY; 469 } 470 if (room) { 471 *xf++ = pri; 472 room--; 473 } 474 want++; 475 need = want; 476 } 477 } else { 478 while (*t || state) { 479 _collate_lookup(table, t, &len, &pri, pass, &state); 480 t += len; 481 if (pri <= 0) { 482 if (pri < 0) { 483 errno = EINVAL; 484 goto fail; 485 } 486 state = NULL; 487 continue; 488 } 489 if (room) { 490 *xf++ = pri; 491 room--; 492 } 493 want++; 494 need = want; 495 } 496 } 497 } 498 free(tr); 499 return (need); 500 501 fail: 502 free(tr); 503 return ((size_t)(-1)); 504 } 505 506 /* 507 * In the non-POSIX case, we transform each character into a string of 508 * characters representing the character's priority. Since char is usually 509 * signed, we are limited by 7 bits per byte. To avoid zero, we need to add 510 * XFRM_OFFSET, so we can't use a full 7 bits. For simplicity, we choose 6 511 * bits per byte. 512 * 513 * It turns out that we sometimes have real priorities that are 514 * 31-bits wide. (But: be careful using priorities where the high 515 * order bit is set -- i.e. the priority is negative. The sort order 516 * may be surprising!) 517 * 518 * TODO: This would be a good area to optimize somewhat. It turns out 519 * that real prioririties *except for the last UNDEFINED pass* are generally 520 * very small. We need the localedef code to precalculate the max 521 * priority for us, and ideally also give us a mask, and then we could 522 * severely limit what we expand to. 523 */ 524 #define XFRM_BYTES 6 525 #define XFRM_OFFSET ('0') /* make all printable characters */ 526 #define XFRM_SHIFT 6 527 #define XFRM_MASK ((1 << XFRM_SHIFT) - 1) 528 #define XFRM_SEP ('.') /* chosen to be less than XFRM_OFFSET */ 529 530 static int 531 xfrm(struct xlocale_collate *table, unsigned char *p, int pri, int pass) 532 { 533 /* we use unsigned to ensure zero fill on right shift */ 534 uint32_t val = (uint32_t)table->info->pri_count[pass]; 535 int nc = 0; 536 537 while (val) { 538 *p = (pri & XFRM_MASK) + XFRM_OFFSET; 539 pri >>= XFRM_SHIFT; 540 val >>= XFRM_SHIFT; 541 p++; 542 nc++; 543 } 544 return (nc); 545 } 546 547 size_t 548 _collate_sxfrm(struct xlocale_collate *table, const wchar_t *src, char *xf, 549 size_t room) 550 { 551 int pri; 552 int len; 553 const wchar_t *t; 554 wchar_t *tr = NULL; 555 int direc; 556 int pass; 557 const int32_t *state; 558 size_t want = 0; 559 size_t need = 0; 560 int b; 561 uint8_t buf[XFRM_BYTES]; 562 int ndir = table->info->directive_count; 563 564 assert(src); 565 566 for (pass = 0; pass <= ndir; pass++) { 567 568 state = NULL; 569 570 if (pass != 0) { 571 /* insert level separator from the previous pass */ 572 if (room) { 573 *xf++ = XFRM_SEP; 574 room--; 575 } 576 want++; 577 } 578 579 /* special pass for undefined */ 580 if (pass == ndir) { 581 direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED; 582 } else { 583 direc = table->info->directive[pass]; 584 } 585 586 t = src; 587 588 if (direc & DIRECTIVE_BACKWARD) { 589 wchar_t *bp, *fp, c; 590 free(tr); 591 if ((tr = wcsdup(t)) == NULL) { 592 errno = ENOMEM; 593 goto fail; 594 } 595 bp = tr; 596 fp = tr + wcslen(tr) - 1; 597 while (bp < fp) { 598 c = *bp; 599 *bp++ = *fp; 600 *fp-- = c; 601 } 602 t = (const wchar_t *)tr; 603 } 604 605 if (direc & DIRECTIVE_POSITION) { 606 while (*t || state) { 607 608 _collate_lookup(table, t, &len, &pri, pass, &state); 609 t += len; 610 if (pri <= 0) { 611 if (pri < 0) { 612 errno = EINVAL; 613 goto fail; 614 } 615 state = NULL; 616 pri = COLLATE_MAX_PRIORITY; 617 } 618 619 b = xfrm(table, buf, pri, pass); 620 want += b; 621 if (room) { 622 while (b) { 623 b--; 624 if (room) { 625 *xf++ = buf[b]; 626 room--; 627 } 628 } 629 } 630 need = want; 631 } 632 } else { 633 while (*t || state) { 634 _collate_lookup(table, t, &len, &pri, pass, &state); 635 t += len; 636 if (pri <= 0) { 637 if (pri < 0) { 638 errno = EINVAL; 639 goto fail; 640 } 641 state = NULL; 642 continue; 643 } 644 645 b = xfrm(table, buf, pri, pass); 646 want += b; 647 if (room) { 648 649 while (b) { 650 b--; 651 if (room) { 652 *xf++ = buf[b]; 653 room--; 654 } 655 } 656 } 657 need = want; 658 } 659 } 660 } 661 free(tr); 662 return (need); 663 664 fail: 665 free(tr); 666 return ((size_t)(-1)); 667 } 668 669 /* 670 * __collate_equiv_value returns the primary collation value for the given 671 * collating symbol specified by str and len. Zero or negative is returned 672 * if the collating symbol was not found. This function is used by bracket 673 * code in the TRE regex library. 674 */ 675 int 676 __collate_equiv_value(locale_t locale, const wchar_t *str, size_t len) 677 { 678 int32_t e; 679 680 if (len < 1 || len >= COLLATE_STR_LEN) 681 return (-1); 682 683 FIX_LOCALE(locale); 684 struct xlocale_collate *table = 685 (struct xlocale_collate*)locale->components[XLC_COLLATE]; 686 687 if (table->__collate_load_error) 688 return ((len == 1 && *str <= UCHAR_MAX) ? *str : -1); 689 690 if (len == 1) { 691 e = -1; 692 if (*str <= UCHAR_MAX) 693 e = table->char_pri_table[*str].pri[0]; 694 else if (table->info->large_count > 0) { 695 collate_large_t *match_large; 696 match_large = largesearch(table, *str); 697 if (match_large) 698 e = match_large->pri.pri[0]; 699 } 700 if (e == 0) 701 return (1); 702 return (e > 0 ? e : 0); 703 } 704 if (table->info->chain_count > 0) { 705 wchar_t name[COLLATE_STR_LEN]; 706 collate_chain_t *match_chain; 707 int clen; 708 709 wcsncpy (name, str, len); 710 name[len] = 0; 711 match_chain = chainsearch(table, name, &clen); 712 if (match_chain) { 713 e = match_chain->pri[0]; 714 if (e == 0) 715 return (1); 716 return (e < 0 ? -e : e); 717 } 718 } 719 return (0); 720 } 721