1 /* 2 * Copyright (c) Meta Platforms, Inc. and affiliates. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 /*-**************************************** 12 * Dependencies 13 ******************************************/ 14 #include "util.h" /* note : ensure that platform.h is included first ! */ 15 #include <stdlib.h> /* malloc, realloc, free */ 16 #include <stdio.h> /* fprintf */ 17 #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ 18 #include <errno.h> 19 #include <assert.h> 20 21 #if defined(__FreeBSD__) 22 #include <sys/param.h> /* __FreeBSD_version */ 23 #endif /* #ifdef __FreeBSD__ */ 24 25 #if defined(_WIN32) 26 # include <sys/utime.h> /* utime */ 27 # include <io.h> /* _chmod */ 28 # define ZSTD_USE_UTIMENSAT 0 29 #else 30 # include <unistd.h> /* chown, stat */ 31 # include <sys/stat.h> /* utimensat, st_mtime */ 32 # if (PLATFORM_POSIX_VERSION >= 200809L && defined(st_mtime)) \ 33 || (defined(__FreeBSD__) && __FreeBSD_version >= 1100056) 34 # define ZSTD_USE_UTIMENSAT 1 35 # else 36 # define ZSTD_USE_UTIMENSAT 0 37 # endif 38 # if ZSTD_USE_UTIMENSAT 39 # include <fcntl.h> /* AT_FDCWD */ 40 # else 41 # include <utime.h> /* utime */ 42 # endif 43 #endif 44 45 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) 46 #include <direct.h> /* needed for _mkdir in windows */ 47 #endif 48 49 #if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ 50 # include <dirent.h> /* opendir, readdir */ 51 # include <string.h> /* strerror, memcpy */ 52 #endif /* #ifdef _WIN32 */ 53 54 /*-**************************************** 55 * Internal Macros 56 ******************************************/ 57 58 /* CONTROL is almost like an assert(), but is never disabled. 59 * It's designed for failures that may happen rarely, 60 * but we don't want to maintain a specific error code path for them, 61 * such as a malloc() returning NULL for example. 62 * Since it's always active, this macro can trigger side effects. 63 */ 64 #define CONTROL(c) { \ 65 if (!(c)) { \ 66 UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \ 67 __FILE__, __LINE__, #c); \ 68 exit(1); \ 69 } } 70 71 /* console log */ 72 #define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) 73 #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } 74 75 static int g_traceDepth = 0; 76 int g_traceFileStat = 0; 77 78 #define UTIL_TRACE_CALL(...) \ 79 { \ 80 if (g_traceFileStat) { \ 81 UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \ 82 UTIL_DISPLAY(__VA_ARGS__); \ 83 UTIL_DISPLAY("\n"); \ 84 ++g_traceDepth; \ 85 } \ 86 } 87 88 #define UTIL_TRACE_RET(ret) \ 89 { \ 90 if (g_traceFileStat) { \ 91 --g_traceDepth; \ 92 UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \ 93 } \ 94 } 95 96 /* A modified version of realloc(). 97 * If UTIL_realloc() fails the original block is freed. 98 */ 99 UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) 100 { 101 void *newptr = realloc(ptr, size); 102 if (newptr) return newptr; 103 free(ptr); 104 return NULL; 105 } 106 107 #if defined(_MSC_VER) 108 #define chmod _chmod 109 #endif 110 111 #ifndef ZSTD_HAVE_FCHMOD 112 #if PLATFORM_POSIX_VERSION >= 199309L 113 #define ZSTD_HAVE_FCHMOD 114 #endif 115 #endif 116 117 #ifndef ZSTD_HAVE_FCHOWN 118 #if PLATFORM_POSIX_VERSION >= 200809L 119 #define ZSTD_HAVE_FCHOWN 120 #endif 121 #endif 122 123 /*-**************************************** 124 * Console log 125 ******************************************/ 126 int g_utilDisplayLevel; 127 128 int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, 129 const char* acceptableLetters, int hasStdinInput) { 130 int ch, result; 131 132 if (hasStdinInput) { 133 UTIL_DISPLAY("stdin is an input - not proceeding.\n"); 134 return 1; 135 } 136 137 UTIL_DISPLAY("%s", prompt); 138 ch = getchar(); 139 result = 0; 140 if (strchr(acceptableLetters, ch) == NULL) { 141 UTIL_DISPLAY("%s \n", abortMsg); 142 result = 1; 143 } 144 /* flush the rest */ 145 while ((ch!=EOF) && (ch!='\n')) 146 ch = getchar(); 147 return result; 148 } 149 150 151 /*-************************************* 152 * Constants 153 ***************************************/ 154 #define LIST_SIZE_INCREASE (8*1024) 155 #define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50 156 157 158 /*-************************************* 159 * Functions 160 ***************************************/ 161 162 void UTIL_traceFileStat(void) 163 { 164 g_traceFileStat = 1; 165 } 166 167 int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf) 168 { 169 int ret; 170 UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename); 171 #if defined(_MSC_VER) 172 if (fd >= 0) { 173 ret = !_fstat64(fd, statbuf); 174 } else { 175 ret = !_stat64(filename, statbuf); 176 } 177 #elif defined(__MINGW32__) && defined (__MSVCRT__) 178 if (fd >= 0) { 179 ret = !_fstati64(fd, statbuf); 180 } else { 181 ret = !_stati64(filename, statbuf); 182 } 183 #else 184 if (fd >= 0) { 185 ret = !fstat(fd, statbuf); 186 } else { 187 ret = !stat(filename, statbuf); 188 } 189 #endif 190 UTIL_TRACE_RET(ret); 191 return ret; 192 } 193 194 int UTIL_stat(const char* filename, stat_t* statbuf) 195 { 196 return UTIL_fstat(-1, filename, statbuf); 197 } 198 199 int UTIL_isRegularFile(const char* infilename) 200 { 201 stat_t statbuf; 202 int ret; 203 UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename); 204 ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf); 205 UTIL_TRACE_RET(ret); 206 return ret; 207 } 208 209 int UTIL_isRegularFileStat(const stat_t* statbuf) 210 { 211 #if defined(_MSC_VER) 212 return (statbuf->st_mode & S_IFREG) != 0; 213 #else 214 return S_ISREG(statbuf->st_mode) != 0; 215 #endif 216 } 217 218 /* like chmod, but avoid changing permission of /dev/null */ 219 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) 220 { 221 return UTIL_fchmod(-1, filename, statbuf, permissions); 222 } 223 224 int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions) 225 { 226 stat_t localStatBuf; 227 UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions); 228 if (statbuf == NULL) { 229 if (!UTIL_fstat(fd, filename, &localStatBuf)) { 230 UTIL_TRACE_RET(0); 231 return 0; 232 } 233 statbuf = &localStatBuf; 234 } 235 if (!UTIL_isRegularFileStat(statbuf)) { 236 UTIL_TRACE_RET(0); 237 return 0; /* pretend success, but don't change anything */ 238 } 239 #ifdef ZSTD_HAVE_FCHMOD 240 if (fd >= 0) { 241 int ret; 242 UTIL_TRACE_CALL("fchmod"); 243 ret = fchmod(fd, permissions); 244 UTIL_TRACE_RET(ret); 245 UTIL_TRACE_RET(ret); 246 return ret; 247 } else 248 #endif 249 { 250 int ret; 251 UTIL_TRACE_CALL("chmod"); 252 ret = chmod(filename, permissions); 253 UTIL_TRACE_RET(ret); 254 UTIL_TRACE_RET(ret); 255 return ret; 256 } 257 } 258 259 /* set access and modification times */ 260 int UTIL_utime(const char* filename, const stat_t *statbuf) 261 { 262 int ret; 263 UTIL_TRACE_CALL("UTIL_utime(%s)", filename); 264 /* We check that st_mtime is a macro here in order to give us confidence 265 * that struct stat has a struct timespec st_mtim member. We need this 266 * check because there are some platforms that claim to be POSIX 2008 267 * compliant but which do not have st_mtim... */ 268 /* FreeBSD has implemented POSIX 2008 for a long time but still only 269 * advertises support for POSIX 2001. They have a version macro that 270 * lets us safely gate them in. 271 * See https://docs.freebsd.org/en/books/porters-handbook/versions/. 272 */ 273 #if ZSTD_USE_UTIMENSAT 274 { 275 /* (atime, mtime) */ 276 struct timespec timebuf[2] = { {0, UTIME_NOW} }; 277 timebuf[1] = statbuf->st_mtim; 278 ret = utimensat(AT_FDCWD, filename, timebuf, 0); 279 } 280 #else 281 { 282 struct utimbuf timebuf; 283 timebuf.actime = time(NULL); 284 timebuf.modtime = statbuf->st_mtime; 285 ret = utime(filename, &timebuf); 286 } 287 #endif 288 errno = 0; 289 UTIL_TRACE_RET(ret); 290 return ret; 291 } 292 293 int UTIL_setFileStat(const char *filename, const stat_t *statbuf) 294 { 295 return UTIL_setFDStat(-1, filename, statbuf); 296 } 297 298 int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf) 299 { 300 int res = 0; 301 stat_t curStatBuf; 302 UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename); 303 304 if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) { 305 UTIL_TRACE_RET(-1); 306 return -1; 307 } 308 309 /* Mimic gzip's behavior: 310 * 311 * "Change the group first, then the permissions, then the owner. 312 * That way, the permissions will be correct on systems that allow 313 * users to give away files, without introducing a security hole. 314 * Security depends on permissions not containing the setuid or 315 * setgid bits." */ 316 317 #if !defined(_WIN32) 318 #ifdef ZSTD_HAVE_FCHOWN 319 if (fd >= 0) { 320 res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */ 321 } else 322 #endif 323 { 324 res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */ 325 } 326 #endif 327 328 res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */ 329 330 #if !defined(_WIN32) 331 #ifdef ZSTD_HAVE_FCHOWN 332 if (fd >= 0) { 333 res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */ 334 } else 335 #endif 336 { 337 res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */ 338 } 339 #endif 340 341 errno = 0; 342 UTIL_TRACE_RET(-res); 343 return -res; /* number of errors is returned */ 344 } 345 346 int UTIL_isDirectory(const char* infilename) 347 { 348 stat_t statbuf; 349 int ret; 350 UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename); 351 ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf); 352 UTIL_TRACE_RET(ret); 353 return ret; 354 } 355 356 int UTIL_isDirectoryStat(const stat_t* statbuf) 357 { 358 int ret; 359 UTIL_TRACE_CALL("UTIL_isDirectoryStat()"); 360 #if defined(_MSC_VER) 361 ret = (statbuf->st_mode & _S_IFDIR) != 0; 362 #else 363 ret = S_ISDIR(statbuf->st_mode) != 0; 364 #endif 365 UTIL_TRACE_RET(ret); 366 return ret; 367 } 368 369 int UTIL_compareStr(const void *p1, const void *p2) { 370 return strcmp(* (char * const *) p1, * (char * const *) p2); 371 } 372 373 int UTIL_isSameFile(const char* fName1, const char* fName2) 374 { 375 int ret; 376 assert(fName1 != NULL); assert(fName2 != NULL); 377 UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2); 378 #if defined(_MSC_VER) || defined(_WIN32) 379 /* note : Visual does not support file identification by inode. 380 * inode does not work on Windows, even with a posix layer, like msys2. 381 * The following work-around is limited to detecting exact name repetition only, 382 * aka `filename` is considered different from `subdir/../filename` */ 383 ret = !strcmp(fName1, fName2); 384 #else 385 { stat_t file1Stat; 386 stat_t file2Stat; 387 ret = UTIL_stat(fName1, &file1Stat) 388 && UTIL_stat(fName2, &file2Stat) 389 && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat); 390 } 391 #endif 392 UTIL_TRACE_RET(ret); 393 return ret; 394 } 395 396 int UTIL_isSameFileStat( 397 const char* fName1, const char* fName2, 398 const stat_t* file1Stat, const stat_t* file2Stat) 399 { 400 int ret; 401 assert(fName1 != NULL); assert(fName2 != NULL); 402 UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2); 403 #if defined(_MSC_VER) || defined(_WIN32) 404 /* note : Visual does not support file identification by inode. 405 * inode does not work on Windows, even with a posix layer, like msys2. 406 * The following work-around is limited to detecting exact name repetition only, 407 * aka `filename` is considered different from `subdir/../filename` */ 408 (void)file1Stat; 409 (void)file2Stat; 410 ret = !strcmp(fName1, fName2); 411 #else 412 { 413 ret = (file1Stat->st_dev == file2Stat->st_dev) 414 && (file1Stat->st_ino == file2Stat->st_ino); 415 } 416 #endif 417 UTIL_TRACE_RET(ret); 418 return ret; 419 } 420 421 /* UTIL_isFIFO : distinguish named pipes */ 422 int UTIL_isFIFO(const char* infilename) 423 { 424 UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename); 425 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 426 #if PLATFORM_POSIX_VERSION >= 200112L 427 { 428 stat_t statbuf; 429 if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) { 430 UTIL_TRACE_RET(1); 431 return 1; 432 } 433 } 434 #endif 435 (void)infilename; 436 UTIL_TRACE_RET(0); 437 return 0; 438 } 439 440 /* UTIL_isFIFO : distinguish named pipes */ 441 int UTIL_isFIFOStat(const stat_t* statbuf) 442 { 443 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 444 #if PLATFORM_POSIX_VERSION >= 200112L 445 if (S_ISFIFO(statbuf->st_mode)) return 1; 446 #endif 447 (void)statbuf; 448 return 0; 449 } 450 451 /* UTIL_isBlockDevStat : distinguish named pipes */ 452 int UTIL_isBlockDevStat(const stat_t* statbuf) 453 { 454 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 455 #if PLATFORM_POSIX_VERSION >= 200112L 456 if (S_ISBLK(statbuf->st_mode)) return 1; 457 #endif 458 (void)statbuf; 459 return 0; 460 } 461 462 int UTIL_isLink(const char* infilename) 463 { 464 UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename); 465 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 466 #if PLATFORM_POSIX_VERSION >= 200112L 467 { 468 stat_t statbuf; 469 int const r = lstat(infilename, &statbuf); 470 if (!r && S_ISLNK(statbuf.st_mode)) { 471 UTIL_TRACE_RET(1); 472 return 1; 473 } 474 } 475 #endif 476 (void)infilename; 477 UTIL_TRACE_RET(0); 478 return 0; 479 } 480 481 static int g_fakeStdinIsConsole = 0; 482 static int g_fakeStderrIsConsole = 0; 483 static int g_fakeStdoutIsConsole = 0; 484 485 int UTIL_isConsole(FILE* file) 486 { 487 int ret; 488 UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file)); 489 if (file == stdin && g_fakeStdinIsConsole) 490 ret = 1; 491 else if (file == stderr && g_fakeStderrIsConsole) 492 ret = 1; 493 else if (file == stdout && g_fakeStdoutIsConsole) 494 ret = 1; 495 else 496 ret = IS_CONSOLE(file); 497 UTIL_TRACE_RET(ret); 498 return ret; 499 } 500 501 void UTIL_fakeStdinIsConsole(void) 502 { 503 g_fakeStdinIsConsole = 1; 504 } 505 void UTIL_fakeStdoutIsConsole(void) 506 { 507 g_fakeStdoutIsConsole = 1; 508 } 509 void UTIL_fakeStderrIsConsole(void) 510 { 511 g_fakeStderrIsConsole = 1; 512 } 513 514 U64 UTIL_getFileSize(const char* infilename) 515 { 516 stat_t statbuf; 517 UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename); 518 if (!UTIL_stat(infilename, &statbuf)) { 519 UTIL_TRACE_RET(-1); 520 return UTIL_FILESIZE_UNKNOWN; 521 } 522 { 523 U64 const size = UTIL_getFileSizeStat(&statbuf); 524 UTIL_TRACE_RET((int)size); 525 return size; 526 } 527 } 528 529 U64 UTIL_getFileSizeStat(const stat_t* statbuf) 530 { 531 if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN; 532 #if defined(_MSC_VER) 533 if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; 534 #elif defined(__MINGW32__) && defined (__MSVCRT__) 535 if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; 536 #else 537 if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN; 538 #endif 539 return (U64)statbuf->st_size; 540 } 541 542 UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) 543 { 544 UTIL_HumanReadableSize_t hrs; 545 546 if (g_utilDisplayLevel > 3) { 547 /* In verbose mode, do not scale sizes down, except in the case of 548 * values that exceed the integral precision of a double. */ 549 if (size >= (1ull << 53)) { 550 hrs.value = (double)size / (1ull << 20); 551 hrs.suffix = " MiB"; 552 /* At worst, a double representation of a maximal size will be 553 * accurate to better than tens of kilobytes. */ 554 hrs.precision = 2; 555 } else { 556 hrs.value = (double)size; 557 hrs.suffix = " B"; 558 hrs.precision = 0; 559 } 560 } else { 561 /* In regular mode, scale sizes down and use suffixes. */ 562 if (size >= (1ull << 60)) { 563 hrs.value = (double)size / (1ull << 60); 564 hrs.suffix = " EiB"; 565 } else if (size >= (1ull << 50)) { 566 hrs.value = (double)size / (1ull << 50); 567 hrs.suffix = " PiB"; 568 } else if (size >= (1ull << 40)) { 569 hrs.value = (double)size / (1ull << 40); 570 hrs.suffix = " TiB"; 571 } else if (size >= (1ull << 30)) { 572 hrs.value = (double)size / (1ull << 30); 573 hrs.suffix = " GiB"; 574 } else if (size >= (1ull << 20)) { 575 hrs.value = (double)size / (1ull << 20); 576 hrs.suffix = " MiB"; 577 } else if (size >= (1ull << 10)) { 578 hrs.value = (double)size / (1ull << 10); 579 hrs.suffix = " KiB"; 580 } else { 581 hrs.value = (double)size; 582 hrs.suffix = " B"; 583 } 584 585 if (hrs.value >= 100 || (U64)hrs.value == size) { 586 hrs.precision = 0; 587 } else if (hrs.value >= 10) { 588 hrs.precision = 1; 589 } else if (hrs.value > 1) { 590 hrs.precision = 2; 591 } else { 592 hrs.precision = 3; 593 } 594 } 595 596 return hrs; 597 } 598 599 U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) 600 { 601 U64 total = 0; 602 unsigned n; 603 UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles); 604 for (n=0; n<nbFiles; n++) { 605 U64 const size = UTIL_getFileSize(fileNamesTable[n]); 606 if (size == UTIL_FILESIZE_UNKNOWN) { 607 UTIL_TRACE_RET(-1); 608 return UTIL_FILESIZE_UNKNOWN; 609 } 610 total += size; 611 } 612 UTIL_TRACE_RET((int)total); 613 return total; 614 } 615 616 617 /* condition : @file must be valid, and not have reached its end. 618 * @return : length of line written into @buf, ended with `\0` instead of '\n', 619 * or 0, if there is no new line */ 620 static size_t readLineFromFile(char* buf, size_t len, FILE* file) 621 { 622 assert(!feof(file)); 623 if ( fgets(buf, (int) len, file) == NULL ) return 0; 624 { size_t linelen = strlen(buf); 625 if (strlen(buf)==0) return 0; 626 if (buf[linelen-1] == '\n') linelen--; 627 buf[linelen] = '\0'; 628 return linelen+1; 629 } 630 } 631 632 /* Conditions : 633 * size of @inputFileName file must be < @dstCapacity 634 * @dst must be initialized 635 * @return : nb of lines 636 * or -1 if there's an error 637 */ 638 static int 639 readLinesFromFile(void* dst, size_t dstCapacity, 640 const char* inputFileName) 641 { 642 int nbFiles = 0; 643 size_t pos = 0; 644 char* const buf = (char*)dst; 645 FILE* const inputFile = fopen(inputFileName, "r"); 646 647 assert(dst != NULL); 648 649 if(!inputFile) { 650 if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile"); 651 return -1; 652 } 653 654 while ( !feof(inputFile) ) { 655 size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile); 656 if (lineLength == 0) break; 657 assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */ 658 pos += lineLength; 659 ++nbFiles; 660 } 661 662 CONTROL( fclose(inputFile) == 0 ); 663 664 return nbFiles; 665 } 666 667 /*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/ 668 FileNamesTable* 669 UTIL_createFileNamesTable_fromFileName(const char* inputFileName) 670 { 671 size_t nbFiles = 0; 672 char* buf; 673 size_t bufSize; 674 stat_t statbuf; 675 676 if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf)) 677 return NULL; 678 679 { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf); 680 if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE) 681 return NULL; 682 bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */ 683 } 684 685 buf = (char*) malloc(bufSize); 686 CONTROL( buf != NULL ); 687 688 { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName); 689 690 if (ret_nbFiles <= 0) { 691 free(buf); 692 return NULL; 693 } 694 nbFiles = (size_t)ret_nbFiles; 695 } 696 697 { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable)); 698 CONTROL(filenamesTable != NULL); 699 700 { size_t fnb, pos = 0; 701 for (fnb = 0; fnb < nbFiles; fnb++) { 702 filenamesTable[fnb] = buf+pos; 703 pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */ 704 } 705 assert(pos <= bufSize); 706 } 707 708 return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf); 709 } 710 } 711 712 static FileNamesTable* 713 UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf) 714 { 715 FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table)); 716 CONTROL(table != NULL); 717 table->fileNames = filenames; 718 table->buf = buf; 719 table->tableSize = tableSize; 720 table->tableCapacity = tableCapacity; 721 return table; 722 } 723 724 FileNamesTable* 725 UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf) 726 { 727 return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf); 728 } 729 730 void UTIL_freeFileNamesTable(FileNamesTable* table) 731 { 732 if (table==NULL) return; 733 free((void*)table->fileNames); 734 free(table->buf); 735 free(table); 736 } 737 738 FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize) 739 { 740 const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable)); 741 FileNamesTable* fnt; 742 if (fnTable==NULL) return NULL; 743 fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL); 744 fnt->tableSize = 0; /* the table is empty */ 745 return fnt; 746 } 747 748 int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) { 749 size_t i; 750 for(i=0 ;i < table->tableSize; i++) { 751 if(!strcmp(table->fileNames[i], name)) { 752 return (int)i; 753 } 754 } 755 return -1; 756 } 757 758 void UTIL_refFilename(FileNamesTable* fnt, const char* filename) 759 { 760 assert(fnt->tableSize < fnt->tableCapacity); 761 fnt->fileNames[fnt->tableSize] = filename; 762 fnt->tableSize++; 763 } 764 765 static size_t getTotalTableSize(FileNamesTable* table) 766 { 767 size_t fnb, totalSize = 0; 768 for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) { 769 totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */ 770 } 771 return totalSize; 772 } 773 774 FileNamesTable* 775 UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2) 776 { 777 unsigned newTableIdx = 0; 778 size_t pos = 0; 779 size_t newTotalTableSize; 780 char* buf; 781 782 FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL); 783 CONTROL( newTable != NULL ); 784 785 newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2); 786 787 buf = (char*) calloc(newTotalTableSize, sizeof(*buf)); 788 CONTROL ( buf != NULL ); 789 790 newTable->buf = buf; 791 newTable->tableSize = table1->tableSize + table2->tableSize; 792 newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames))); 793 CONTROL ( newTable->fileNames != NULL ); 794 795 { unsigned idx1; 796 for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) { 797 size_t const curLen = strlen(table1->fileNames[idx1]); 798 memcpy(buf+pos, table1->fileNames[idx1], curLen); 799 assert(newTableIdx <= newTable->tableSize); 800 newTable->fileNames[newTableIdx] = buf+pos; 801 pos += curLen+1; 802 } } 803 804 { unsigned idx2; 805 for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { 806 size_t const curLen = strlen(table2->fileNames[idx2]); 807 memcpy(buf+pos, table2->fileNames[idx2], curLen); 808 assert(newTableIdx < newTable->tableSize); 809 newTable->fileNames[newTableIdx] = buf+pos; 810 pos += curLen+1; 811 } } 812 assert(pos <= newTotalTableSize); 813 newTable->tableSize = newTableIdx; 814 815 UTIL_freeFileNamesTable(table1); 816 UTIL_freeFileNamesTable(table2); 817 818 return newTable; 819 } 820 821 #ifdef _WIN32 822 static int UTIL_prepareFileList(const char* dirName, 823 char** bufStart, size_t* pos, 824 char** bufEnd, int followLinks) 825 { 826 char* path; 827 size_t dirLength, pathLength; 828 int nbFiles = 0; 829 WIN32_FIND_DATAA cFile; 830 HANDLE hFile; 831 832 dirLength = strlen(dirName); 833 path = (char*) malloc(dirLength + 3); 834 if (!path) return 0; 835 836 memcpy(path, dirName, dirLength); 837 path[dirLength] = '\\'; 838 path[dirLength+1] = '*'; 839 path[dirLength+2] = 0; 840 841 hFile=FindFirstFileA(path, &cFile); 842 if (hFile == INVALID_HANDLE_VALUE) { 843 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName); 844 return 0; 845 } 846 free(path); 847 848 do { 849 size_t const fnameLength = strlen(cFile.cFileName); 850 path = (char*) malloc(dirLength + fnameLength + 2); 851 if (!path) { FindClose(hFile); return 0; } 852 memcpy(path, dirName, dirLength); 853 path[dirLength] = '\\'; 854 memcpy(path+dirLength+1, cFile.cFileName, fnameLength); 855 pathLength = dirLength+1+fnameLength; 856 path[pathLength] = 0; 857 if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { 858 if ( strcmp (cFile.cFileName, "..") == 0 859 || strcmp (cFile.cFileName, ".") == 0 ) 860 continue; 861 /* Recursively call "UTIL_prepareFileList" with the new path. */ 862 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); 863 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } 864 } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) 865 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) 866 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) { 867 if (*bufStart + *pos + pathLength >= *bufEnd) { 868 ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; 869 *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); 870 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } 871 *bufEnd = *bufStart + newListSize; 872 } 873 if (*bufStart + *pos + pathLength < *bufEnd) { 874 memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */); 875 *pos += pathLength + 1; 876 nbFiles++; 877 } } 878 free(path); 879 } while (FindNextFileA(hFile, &cFile)); 880 881 FindClose(hFile); 882 return nbFiles; 883 } 884 885 #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ 886 887 static int UTIL_prepareFileList(const char *dirName, 888 char** bufStart, size_t* pos, 889 char** bufEnd, int followLinks) 890 { 891 DIR* dir; 892 struct dirent * entry; 893 size_t dirLength; 894 int nbFiles = 0; 895 896 if (!(dir = opendir(dirName))) { 897 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); 898 return 0; 899 } 900 901 dirLength = strlen(dirName); 902 errno = 0; 903 while ((entry = readdir(dir)) != NULL) { 904 char* path; 905 size_t fnameLength, pathLength; 906 if (strcmp (entry->d_name, "..") == 0 || 907 strcmp (entry->d_name, ".") == 0) continue; 908 fnameLength = strlen(entry->d_name); 909 path = (char*) malloc(dirLength + fnameLength + 2); 910 if (!path) { closedir(dir); return 0; } 911 memcpy(path, dirName, dirLength); 912 913 path[dirLength] = '/'; 914 memcpy(path+dirLength+1, entry->d_name, fnameLength); 915 pathLength = dirLength+1+fnameLength; 916 path[pathLength] = 0; 917 918 if (!followLinks && UTIL_isLink(path)) { 919 UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); 920 free(path); 921 continue; 922 } 923 924 if (UTIL_isDirectory(path)) { 925 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ 926 if (*bufStart == NULL) { free(path); closedir(dir); return 0; } 927 } else { 928 if (*bufStart + *pos + pathLength >= *bufEnd) { 929 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; 930 assert(newListSize >= 0); 931 *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); 932 if (*bufStart != NULL) { 933 *bufEnd = *bufStart + newListSize; 934 } else { 935 free(path); closedir(dir); return 0; 936 } 937 } 938 if (*bufStart + *pos + pathLength < *bufEnd) { 939 memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ 940 *pos += pathLength + 1; 941 nbFiles++; 942 } } 943 free(path); 944 errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ 945 } 946 947 if (errno != 0) { 948 UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno)); 949 free(*bufStart); 950 *bufStart = NULL; 951 } 952 closedir(dir); 953 return nbFiles; 954 } 955 956 #else 957 958 static int UTIL_prepareFileList(const char *dirName, 959 char** bufStart, size_t* pos, 960 char** bufEnd, int followLinks) 961 { 962 (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks; 963 UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName); 964 return 0; 965 } 966 967 #endif /* #ifdef _WIN32 */ 968 969 int UTIL_isCompressedFile(const char *inputName, const char *extensionList[]) 970 { 971 const char* ext = UTIL_getFileExtension(inputName); 972 while(*extensionList!=NULL) 973 { 974 const int isCompressedExtension = strcmp(ext,*extensionList); 975 if(isCompressedExtension==0) 976 return 1; 977 ++extensionList; 978 } 979 return 0; 980 } 981 982 /*Utility function to get file extension from file */ 983 const char* UTIL_getFileExtension(const char* infilename) 984 { 985 const char* extension = strrchr(infilename, '.'); 986 if(!extension || extension==infilename) return ""; 987 return extension; 988 } 989 990 static int pathnameHas2Dots(const char *pathname) 991 { 992 /* We need to figure out whether any ".." present in the path is a whole 993 * path token, which is the case if it is bordered on both sides by either 994 * the beginning/end of the path or by a directory separator. 995 */ 996 const char *needle = pathname; 997 while (1) { 998 needle = strstr(needle, ".."); 999 1000 if (needle == NULL) { 1001 return 0; 1002 } 1003 1004 if ((needle == pathname || needle[-1] == PATH_SEP) 1005 && (needle[2] == '\0' || needle[2] == PATH_SEP)) { 1006 return 1; 1007 } 1008 1009 /* increment so we search for the next match */ 1010 needle++; 1011 }; 1012 return 0; 1013 } 1014 1015 static int isFileNameValidForMirroredOutput(const char *filename) 1016 { 1017 return !pathnameHas2Dots(filename); 1018 } 1019 1020 1021 #define DIR_DEFAULT_MODE 0755 1022 static mode_t getDirMode(const char *dirName) 1023 { 1024 stat_t st; 1025 if (!UTIL_stat(dirName, &st)) { 1026 UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno)); 1027 return DIR_DEFAULT_MODE; 1028 } 1029 if (!UTIL_isDirectoryStat(&st)) { 1030 UTIL_DISPLAY("zstd: expected directory: %s\n", dirName); 1031 return DIR_DEFAULT_MODE; 1032 } 1033 return st.st_mode; 1034 } 1035 1036 static int makeDir(const char *dir, mode_t mode) 1037 { 1038 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) 1039 int ret = _mkdir(dir); 1040 (void) mode; 1041 #else 1042 int ret = mkdir(dir, mode); 1043 #endif 1044 if (ret != 0) { 1045 if (errno == EEXIST) 1046 return 0; 1047 UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno)); 1048 } 1049 return ret; 1050 } 1051 1052 /* this function requires a mutable input string */ 1053 static void convertPathnameToDirName(char *pathname) 1054 { 1055 size_t len = 0; 1056 char* pos = NULL; 1057 /* get dir name from pathname similar to 'dirname()' */ 1058 assert(pathname != NULL); 1059 1060 /* remove trailing '/' chars */ 1061 len = strlen(pathname); 1062 assert(len > 0); 1063 while (pathname[len] == PATH_SEP) { 1064 pathname[len] = '\0'; 1065 len--; 1066 } 1067 if (len == 0) return; 1068 1069 /* if input is a single file, return '.' instead. i.e. 1070 * "xyz/abc/file.txt" => "xyz/abc" 1071 "./file.txt" => "." 1072 "file.txt" => "." 1073 */ 1074 pos = strrchr(pathname, PATH_SEP); 1075 if (pos == NULL) { 1076 pathname[0] = '.'; 1077 pathname[1] = '\0'; 1078 } else { 1079 *pos = '\0'; 1080 } 1081 } 1082 1083 /* pathname must be valid */ 1084 static const char* trimLeadingRootChar(const char *pathname) 1085 { 1086 assert(pathname != NULL); 1087 if (pathname[0] == PATH_SEP) 1088 return pathname + 1; 1089 return pathname; 1090 } 1091 1092 /* pathname must be valid */ 1093 static const char* trimLeadingCurrentDirConst(const char *pathname) 1094 { 1095 assert(pathname != NULL); 1096 if ((pathname[0] == '.') && (pathname[1] == PATH_SEP)) 1097 return pathname + 2; 1098 return pathname; 1099 } 1100 1101 static char* 1102 trimLeadingCurrentDir(char *pathname) 1103 { 1104 /* 'union charunion' can do const-cast without compiler warning */ 1105 union charunion { 1106 char *chr; 1107 const char* cchr; 1108 } ptr; 1109 ptr.cchr = trimLeadingCurrentDirConst(pathname); 1110 return ptr.chr; 1111 } 1112 1113 /* remove leading './' or '/' chars here */ 1114 static const char * trimPath(const char *pathname) 1115 { 1116 return trimLeadingRootChar( 1117 trimLeadingCurrentDirConst(pathname)); 1118 } 1119 1120 static char* mallocAndJoin2Dir(const char *dir1, const char *dir2) 1121 { 1122 assert(dir1 != NULL && dir2 != NULL); 1123 { const size_t dir1Size = strlen(dir1); 1124 const size_t dir2Size = strlen(dir2); 1125 char *outDirBuffer, *buffer; 1126 1127 outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2); 1128 CONTROL(outDirBuffer != NULL); 1129 1130 memcpy(outDirBuffer, dir1, dir1Size); 1131 outDirBuffer[dir1Size] = '\0'; 1132 1133 buffer = outDirBuffer + dir1Size; 1134 if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) { 1135 *buffer = PATH_SEP; 1136 buffer++; 1137 } 1138 memcpy(buffer, dir2, dir2Size); 1139 buffer[dir2Size] = '\0'; 1140 1141 return outDirBuffer; 1142 } 1143 } 1144 1145 /* this function will return NULL if input srcFileName is not valid name for mirrored output path */ 1146 char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName) 1147 { 1148 char* pathname = NULL; 1149 if (!isFileNameValidForMirroredOutput(srcFileName)) 1150 return NULL; 1151 1152 pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName)); 1153 1154 convertPathnameToDirName(pathname); 1155 return pathname; 1156 } 1157 1158 static int 1159 mirrorSrcDir(char* srcDirName, const char* outDirName) 1160 { 1161 mode_t srcMode; 1162 int status = 0; 1163 char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName)); 1164 if (!newDir) 1165 return -ENOMEM; 1166 1167 srcMode = getDirMode(srcDirName); 1168 status = makeDir(newDir, srcMode); 1169 free(newDir); 1170 return status; 1171 } 1172 1173 static int 1174 mirrorSrcDirRecursive(char* srcDirName, const char* outDirName) 1175 { 1176 int status = 0; 1177 char* pp = trimLeadingCurrentDir(srcDirName); 1178 char* sp = NULL; 1179 1180 while ((sp = strchr(pp, PATH_SEP)) != NULL) { 1181 if (sp != pp) { 1182 *sp = '\0'; 1183 status = mirrorSrcDir(srcDirName, outDirName); 1184 if (status != 0) 1185 return status; 1186 *sp = PATH_SEP; 1187 } 1188 pp = sp + 1; 1189 } 1190 status = mirrorSrcDir(srcDirName, outDirName); 1191 return status; 1192 } 1193 1194 static void 1195 makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName) 1196 { 1197 unsigned int i = 0; 1198 for (i = 0; i < nbFile; i++) 1199 mirrorSrcDirRecursive(srcDirNames[i], outDirName); 1200 } 1201 1202 static int 1203 firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir) 1204 { 1205 size_t firstDirLen = strlen(firstDir), 1206 secondDirLen = strlen(secondDir); 1207 return firstDirLen <= secondDirLen && 1208 (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') && 1209 0 == strncmp(firstDir, secondDir, firstDirLen); 1210 } 1211 1212 static int compareDir(const void* pathname1, const void* pathname2) { 1213 /* sort it after remove the leading '/' or './'*/ 1214 const char* s1 = trimPath(*(char * const *) pathname1); 1215 const char* s2 = trimPath(*(char * const *) pathname2); 1216 return strcmp(s1, s2); 1217 } 1218 1219 static void 1220 makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName) 1221 { 1222 unsigned int i = 0, uniqueDirNr = 0; 1223 char** uniqueDirNames = NULL; 1224 1225 if (nbFile == 0) 1226 return; 1227 1228 uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *)); 1229 CONTROL(uniqueDirNames != NULL); 1230 1231 /* if dirs is "a/b/c" and "a/b/c/d", we only need call: 1232 * we just need "a/b/c/d" */ 1233 qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir); 1234 1235 uniqueDirNr = 1; 1236 uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0]; 1237 for (i = 1; i < nbFile; i++) { 1238 char* prevDirName = srcDirNames[i - 1]; 1239 char* currDirName = srcDirNames[i]; 1240 1241 /* note: we always compare trimmed path, i.e.: 1242 * src dir of "./foo" and "/foo" will be both saved into: 1243 * "outDirName/foo/" */ 1244 if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName), 1245 trimPath(currDirName))) 1246 uniqueDirNr++; 1247 1248 /* we need to maintain original src dir name instead of trimmed 1249 * dir, so we can retrieve the original src dir's mode_t */ 1250 uniqueDirNames[uniqueDirNr - 1] = currDirName; 1251 } 1252 1253 makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName); 1254 1255 free(uniqueDirNames); 1256 } 1257 1258 static void 1259 makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName) 1260 { 1261 unsigned int i = 0; 1262 for (i = 0; i < nbFile; ++i) 1263 convertPathnameToDirName(srcFileNames[i]); 1264 makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName); 1265 } 1266 1267 void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName) 1268 { 1269 unsigned int i = 0, validFilenamesNr = 0; 1270 char** srcFileNames = (char **) malloc(nbFile * sizeof (char *)); 1271 CONTROL(srcFileNames != NULL); 1272 1273 /* check input filenames is valid */ 1274 for (i = 0; i < nbFile; ++i) { 1275 if (isFileNameValidForMirroredOutput(inFileNames[i])) { 1276 char* fname = STRDUP(inFileNames[i]); 1277 CONTROL(fname != NULL); 1278 srcFileNames[validFilenamesNr++] = fname; 1279 } 1280 } 1281 1282 if (validFilenamesNr > 0) { 1283 makeDir(outDirName, DIR_DEFAULT_MODE); 1284 makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName); 1285 } 1286 1287 for (i = 0; i < validFilenamesNr; i++) 1288 free(srcFileNames[i]); 1289 free(srcFileNames); 1290 } 1291 1292 FileNamesTable* 1293 UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks) 1294 { 1295 unsigned nbFiles; 1296 char* buf = (char*)malloc(LIST_SIZE_INCREASE); 1297 char* bufend = buf + LIST_SIZE_INCREASE; 1298 1299 if (!buf) return NULL; 1300 1301 { size_t ifnNb, pos; 1302 for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) { 1303 if (!UTIL_isDirectory(inputNames[ifnNb])) { 1304 size_t const len = strlen(inputNames[ifnNb]); 1305 if (buf + pos + len >= bufend) { 1306 ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; 1307 assert(newListSize >= 0); 1308 buf = (char*)UTIL_realloc(buf, (size_t)newListSize); 1309 if (!buf) return NULL; 1310 bufend = buf + newListSize; 1311 } 1312 if (buf + pos + len < bufend) { 1313 memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */ 1314 pos += len + 1; 1315 nbFiles++; 1316 } 1317 } else { 1318 nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks); 1319 if (buf == NULL) return NULL; 1320 } } } 1321 1322 /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */ 1323 1324 { size_t ifnNb, pos; 1325 size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */ 1326 const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable)); 1327 if (!fileNamesTable) { free(buf); return NULL; } 1328 1329 for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) { 1330 fileNamesTable[ifnNb] = buf + pos; 1331 if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; } 1332 pos += strlen(fileNamesTable[ifnNb]) + 1; 1333 } 1334 return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf); 1335 } 1336 } 1337 1338 1339 void UTIL_expandFNT(FileNamesTable** fnt, int followLinks) 1340 { 1341 FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks); 1342 CONTROL(newFNT != NULL); 1343 UTIL_freeFileNamesTable(*fnt); 1344 *fnt = newFNT; 1345 } 1346 1347 FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames) 1348 { 1349 size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames); 1350 const char** const newFNTable = (const char**)malloc(sizeof_FNTable); 1351 if (newFNTable==NULL) return NULL; 1352 memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */ 1353 return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL); 1354 } 1355 1356 1357 /*-**************************************** 1358 * count the number of cores 1359 ******************************************/ 1360 1361 #if defined(_WIN32) || defined(WIN32) 1362 1363 #include <windows.h> 1364 1365 typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); 1366 1367 DWORD CountSetBits(ULONG_PTR bitMask) 1368 { 1369 DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; 1370 DWORD bitSetCount = 0; 1371 ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; 1372 DWORD i; 1373 1374 for (i = 0; i <= LSHIFT; ++i) 1375 { 1376 bitSetCount += ((bitMask & bitTest)?1:0); 1377 bitTest/=2; 1378 } 1379 1380 return bitSetCount; 1381 } 1382 1383 int UTIL_countCores(int logical) 1384 { 1385 static int numCores = 0; 1386 if (numCores != 0) return numCores; 1387 1388 { LPFN_GLPI glpi; 1389 BOOL done = FALSE; 1390 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; 1391 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; 1392 DWORD returnLength = 0; 1393 size_t byteOffset = 0; 1394 1395 #if defined(_MSC_VER) 1396 /* Visual Studio does not like the following cast */ 1397 # pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */ 1398 # pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */ 1399 #endif 1400 glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")), 1401 "GetLogicalProcessorInformation"); 1402 1403 if (glpi == NULL) { 1404 goto failed; 1405 } 1406 1407 while(!done) { 1408 DWORD rc = glpi(buffer, &returnLength); 1409 if (FALSE == rc) { 1410 if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 1411 if (buffer) 1412 free(buffer); 1413 buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); 1414 1415 if (buffer == NULL) { 1416 perror("zstd"); 1417 exit(1); 1418 } 1419 } else { 1420 /* some other error */ 1421 goto failed; 1422 } 1423 } else { 1424 done = TRUE; 1425 } } 1426 1427 ptr = buffer; 1428 1429 while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { 1430 1431 if (ptr->Relationship == RelationProcessorCore) { 1432 if (logical) 1433 numCores += CountSetBits(ptr->ProcessorMask); 1434 else 1435 numCores++; 1436 } 1437 1438 ptr++; 1439 byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); 1440 } 1441 1442 free(buffer); 1443 1444 return numCores; 1445 } 1446 1447 failed: 1448 /* try to fall back on GetSystemInfo */ 1449 { SYSTEM_INFO sysinfo; 1450 GetSystemInfo(&sysinfo); 1451 numCores = sysinfo.dwNumberOfProcessors; 1452 if (numCores == 0) numCores = 1; /* just in case */ 1453 } 1454 return numCores; 1455 } 1456 1457 #elif defined(__APPLE__) 1458 1459 #include <sys/sysctl.h> 1460 1461 /* Use apple-provided syscall 1462 * see: man 3 sysctl */ 1463 int UTIL_countCores(int logical) 1464 { 1465 static S32 numCores = 0; /* apple specifies int32_t */ 1466 if (numCores != 0) return numCores; 1467 1468 { size_t size = sizeof(S32); 1469 int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0); 1470 if (ret != 0) { 1471 if (errno == ENOENT) { 1472 /* entry not present, fall back on 1 */ 1473 numCores = 1; 1474 } else { 1475 perror("zstd: can't get number of cpus"); 1476 exit(1); 1477 } 1478 } 1479 1480 return numCores; 1481 } 1482 } 1483 1484 #elif defined(__linux__) 1485 1486 /* parse /proc/cpuinfo 1487 * siblings / cpu cores should give hyperthreading ratio 1488 * otherwise fall back on sysconf */ 1489 int UTIL_countCores(int logical) 1490 { 1491 static int numCores = 0; 1492 1493 if (numCores != 0) return numCores; 1494 1495 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 1496 if (numCores == -1) { 1497 /* value not queryable, fall back on 1 */ 1498 return numCores = 1; 1499 } 1500 1501 /* try to determine if there's hyperthreading */ 1502 { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); 1503 #define BUF_SIZE 80 1504 char buff[BUF_SIZE]; 1505 1506 int siblings = 0; 1507 int cpu_cores = 0; 1508 int ratio = 1; 1509 1510 if (cpuinfo == NULL) { 1511 /* fall back on the sysconf value */ 1512 return numCores; 1513 } 1514 1515 /* assume the cpu cores/siblings values will be constant across all 1516 * present processors */ 1517 while (!feof(cpuinfo)) { 1518 if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { 1519 if (strncmp(buff, "siblings", 8) == 0) { 1520 const char* const sep = strchr(buff, ':'); 1521 if (sep == NULL || *sep == '\0') { 1522 /* formatting was broken? */ 1523 goto failed; 1524 } 1525 1526 siblings = atoi(sep + 1); 1527 } 1528 if (strncmp(buff, "cpu cores", 9) == 0) { 1529 const char* const sep = strchr(buff, ':'); 1530 if (sep == NULL || *sep == '\0') { 1531 /* formatting was broken? */ 1532 goto failed; 1533 } 1534 1535 cpu_cores = atoi(sep + 1); 1536 } 1537 } else if (ferror(cpuinfo)) { 1538 /* fall back on the sysconf value */ 1539 goto failed; 1540 } } 1541 if (siblings && cpu_cores && siblings > cpu_cores) { 1542 ratio = siblings / cpu_cores; 1543 } 1544 1545 if (ratio && numCores > ratio && !logical) { 1546 numCores = numCores / ratio; 1547 } 1548 1549 failed: 1550 fclose(cpuinfo); 1551 return numCores; 1552 } 1553 } 1554 1555 #elif defined(__FreeBSD__) 1556 1557 #include <sys/sysctl.h> 1558 1559 /* Use physical core sysctl when available 1560 * see: man 4 smp, man 3 sysctl */ 1561 int UTIL_countCores(int logical) 1562 { 1563 static int numCores = 0; /* freebsd sysctl is native int sized */ 1564 #if __FreeBSD_version >= 1300008 1565 static int perCore = 1; 1566 #endif 1567 if (numCores != 0) return numCores; 1568 1569 #if __FreeBSD_version >= 1300008 1570 { size_t size = sizeof(numCores); 1571 int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0); 1572 if (ret == 0) { 1573 if (logical) { 1574 ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0); 1575 /* default to physical cores if logical cannot be read */ 1576 if (ret == 0) 1577 numCores *= perCore; 1578 } 1579 1580 return numCores; 1581 } 1582 if (errno != ENOENT) { 1583 perror("zstd: can't get number of cpus"); 1584 exit(1); 1585 } 1586 /* sysctl not present, fall through to older sysconf method */ 1587 } 1588 #else 1589 /* suppress unused parameter warning */ 1590 (void) logical; 1591 #endif 1592 1593 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 1594 if (numCores == -1) { 1595 /* value not queryable, fall back on 1 */ 1596 numCores = 1; 1597 } 1598 return numCores; 1599 } 1600 1601 #elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) 1602 1603 /* Use POSIX sysconf 1604 * see: man 3 sysconf */ 1605 int UTIL_countCores(int logical) 1606 { 1607 static int numCores = 0; 1608 1609 /* suppress unused parameter warning */ 1610 (void)logical; 1611 1612 if (numCores != 0) return numCores; 1613 1614 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 1615 if (numCores == -1) { 1616 /* value not queryable, fall back on 1 */ 1617 return numCores = 1; 1618 } 1619 return numCores; 1620 } 1621 1622 #else 1623 1624 int UTIL_countCores(int logical) 1625 { 1626 /* suppress unused parameter warning */ 1627 (void)logical; 1628 1629 /* assume 1 */ 1630 return 1; 1631 } 1632 1633 #endif 1634 1635 int UTIL_countPhysicalCores(void) 1636 { 1637 return UTIL_countCores(0); 1638 } 1639 1640 int UTIL_countLogicalCores(void) 1641 { 1642 return UTIL_countCores(1); 1643 } 1644