1 /* 2 * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 #if defined (__cplusplus) 12 extern "C" { 13 #endif 14 15 16 /*-**************************************** 17 * Dependencies 18 ******************************************/ 19 #include "util.h" /* note : ensure that platform.h is included first ! */ 20 #include <errno.h> 21 #include <assert.h> 22 23 24 int UTIL_fileExist(const char* filename) 25 { 26 stat_t statbuf; 27 #if defined(_MSC_VER) 28 int const stat_error = _stat64(filename, &statbuf); 29 #else 30 int const stat_error = stat(filename, &statbuf); 31 #endif 32 return !stat_error; 33 } 34 35 int UTIL_isRegularFile(const char* infilename) 36 { 37 stat_t statbuf; 38 return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ 39 } 40 41 int UTIL_getFileStat(const char* infilename, stat_t *statbuf) 42 { 43 int r; 44 #if defined(_MSC_VER) 45 r = _stat64(infilename, statbuf); 46 if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ 47 #else 48 r = stat(infilename, statbuf); 49 if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ 50 #endif 51 return 1; 52 } 53 54 int UTIL_setFileStat(const char *filename, stat_t *statbuf) 55 { 56 int res = 0; 57 struct utimbuf timebuf; 58 59 if (!UTIL_isRegularFile(filename)) 60 return -1; 61 62 timebuf.actime = time(NULL); 63 timebuf.modtime = statbuf->st_mtime; 64 res += utime(filename, &timebuf); /* set access and modification times */ 65 66 #if !defined(_WIN32) 67 res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ 68 #endif 69 70 res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ 71 72 errno = 0; 73 return -res; /* number of errors is returned */ 74 } 75 76 U32 UTIL_isDirectory(const char* infilename) 77 { 78 int r; 79 stat_t statbuf; 80 #if defined(_MSC_VER) 81 r = _stat64(infilename, &statbuf); 82 if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; 83 #else 84 r = stat(infilename, &statbuf); 85 if (!r && S_ISDIR(statbuf.st_mode)) return 1; 86 #endif 87 return 0; 88 } 89 90 int UTIL_isSameFile(const char* file1, const char* file2) 91 { 92 #if defined(_MSC_VER) 93 /* note : Visual does not support file identification by inode. 94 * The following work-around is limited to detecting exact name repetition only, 95 * aka `filename` is considered different from `subdir/../filename` */ 96 return !strcmp(file1, file2); 97 #else 98 stat_t file1Stat; 99 stat_t file2Stat; 100 return UTIL_getFileStat(file1, &file1Stat) 101 && UTIL_getFileStat(file2, &file2Stat) 102 && (file1Stat.st_dev == file2Stat.st_dev) 103 && (file1Stat.st_ino == file2Stat.st_ino); 104 #endif 105 } 106 107 U32 UTIL_isLink(const char* infilename) 108 { 109 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 110 #ifndef __STRICT_ANSI__ 111 #if defined(_BSD_SOURCE) \ 112 || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \ 113 || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \ 114 || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \ 115 || (defined(__APPLE__) && defined(__MACH__)) \ 116 || defined(__OpenBSD__) \ 117 || defined(__FreeBSD__) 118 int r; 119 stat_t statbuf; 120 r = lstat(infilename, &statbuf); 121 if (!r && S_ISLNK(statbuf.st_mode)) return 1; 122 #endif 123 #endif 124 (void)infilename; 125 return 0; 126 } 127 128 U64 UTIL_getFileSize(const char* infilename) 129 { 130 if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN; 131 { int r; 132 #if defined(_MSC_VER) 133 struct __stat64 statbuf; 134 r = _stat64(infilename, &statbuf); 135 if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; 136 #elif defined(__MINGW32__) && defined (__MSVCRT__) 137 struct _stati64 statbuf; 138 r = _stati64(infilename, &statbuf); 139 if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; 140 #else 141 struct stat statbuf; 142 r = stat(infilename, &statbuf); 143 if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN; 144 #endif 145 return (U64)statbuf.st_size; 146 } 147 } 148 149 150 U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles) 151 { 152 U64 total = 0; 153 int error = 0; 154 unsigned n; 155 for (n=0; n<nbFiles; n++) { 156 U64 const size = UTIL_getFileSize(fileNamesTable[n]); 157 error |= (size == UTIL_FILESIZE_UNKNOWN); 158 total += size; 159 } 160 return error ? UTIL_FILESIZE_UNKNOWN : total; 161 } 162 163 #ifdef _WIN32 164 int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) 165 { 166 char* path; 167 int dirLength, fnameLength, pathLength, nbFiles = 0; 168 WIN32_FIND_DATAA cFile; 169 HANDLE hFile; 170 171 dirLength = (int)strlen(dirName); 172 path = (char*) malloc(dirLength + 3); 173 if (!path) return 0; 174 175 memcpy(path, dirName, dirLength); 176 path[dirLength] = '\\'; 177 path[dirLength+1] = '*'; 178 path[dirLength+2] = 0; 179 180 hFile=FindFirstFileA(path, &cFile); 181 if (hFile == INVALID_HANDLE_VALUE) { 182 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName); 183 return 0; 184 } 185 free(path); 186 187 do { 188 fnameLength = (int)strlen(cFile.cFileName); 189 path = (char*) malloc(dirLength + fnameLength + 2); 190 if (!path) { FindClose(hFile); return 0; } 191 memcpy(path, dirName, dirLength); 192 path[dirLength] = '\\'; 193 memcpy(path+dirLength+1, cFile.cFileName, fnameLength); 194 pathLength = dirLength+1+fnameLength; 195 path[pathLength] = 0; 196 if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { 197 if ( strcmp (cFile.cFileName, "..") == 0 198 || strcmp (cFile.cFileName, ".") == 0 ) 199 continue; 200 /* Recursively call "UTIL_prepareFileList" with the new path. */ 201 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); 202 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } 203 } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) 204 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) 205 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) { 206 if (*bufStart + *pos + pathLength >= *bufEnd) { 207 ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; 208 *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); 209 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } 210 *bufEnd = *bufStart + newListSize; 211 } 212 if (*bufStart + *pos + pathLength < *bufEnd) { 213 memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */); 214 *pos += pathLength + 1; 215 nbFiles++; 216 } 217 } 218 free(path); 219 } while (FindNextFileA(hFile, &cFile)); 220 221 FindClose(hFile); 222 return nbFiles; 223 } 224 225 #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ 226 227 int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) 228 { 229 DIR *dir; 230 struct dirent *entry; 231 char* path; 232 int dirLength, fnameLength, pathLength, nbFiles = 0; 233 234 if (!(dir = opendir(dirName))) { 235 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); 236 return 0; 237 } 238 239 dirLength = (int)strlen(dirName); 240 errno = 0; 241 while ((entry = readdir(dir)) != NULL) { 242 if (strcmp (entry->d_name, "..") == 0 || 243 strcmp (entry->d_name, ".") == 0) continue; 244 fnameLength = (int)strlen(entry->d_name); 245 path = (char*) malloc(dirLength + fnameLength + 2); 246 if (!path) { closedir(dir); return 0; } 247 memcpy(path, dirName, dirLength); 248 249 path[dirLength] = '/'; 250 memcpy(path+dirLength+1, entry->d_name, fnameLength); 251 pathLength = dirLength+1+fnameLength; 252 path[pathLength] = 0; 253 254 if (!followLinks && UTIL_isLink(path)) { 255 UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); 256 continue; 257 } 258 259 if (UTIL_isDirectory(path)) { 260 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ 261 if (*bufStart == NULL) { free(path); closedir(dir); return 0; } 262 } else { 263 if (*bufStart + *pos + pathLength >= *bufEnd) { 264 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; 265 *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); 266 *bufEnd = *bufStart + newListSize; 267 if (*bufStart == NULL) { free(path); closedir(dir); return 0; } 268 } 269 if (*bufStart + *pos + pathLength < *bufEnd) { 270 memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ 271 *pos += pathLength + 1; 272 nbFiles++; 273 } 274 } 275 free(path); 276 errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ 277 } 278 279 if (errno != 0) { 280 UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno)); 281 free(*bufStart); 282 *bufStart = NULL; 283 } 284 closedir(dir); 285 return nbFiles; 286 } 287 288 #else 289 290 int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) 291 { 292 (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks; 293 UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); 294 return 0; 295 } 296 297 #endif /* #ifdef _WIN32 */ 298 299 /* 300 * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, 301 * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). 302 * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) 303 * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. 304 */ 305 const char** 306 UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, 307 char** allocatedBuffer, unsigned* allocatedNamesNb, 308 int followLinks) 309 { 310 size_t pos; 311 unsigned i, nbFiles; 312 char* buf = (char*)malloc(LIST_SIZE_INCREASE); 313 char* bufend = buf + LIST_SIZE_INCREASE; 314 const char** fileTable; 315 316 if (!buf) return NULL; 317 318 for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) { 319 if (!UTIL_isDirectory(inputNames[i])) { 320 size_t const len = strlen(inputNames[i]); 321 if (buf + pos + len >= bufend) { 322 ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; 323 buf = (char*)UTIL_realloc(buf, newListSize); 324 bufend = buf + newListSize; 325 if (!buf) return NULL; 326 } 327 if (buf + pos + len < bufend) { 328 memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */ 329 pos += len + 1; 330 nbFiles++; 331 } 332 } else { 333 nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks); 334 if (buf == NULL) return NULL; 335 } } 336 337 if (nbFiles == 0) { free(buf); return NULL; } 338 339 fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); 340 if (!fileTable) { free(buf); return NULL; } 341 342 for (i=0, pos=0; i<nbFiles; i++) { 343 fileTable[i] = buf + pos; 344 pos += strlen(fileTable[i]) + 1; 345 } 346 347 if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; } 348 349 *allocatedBuffer = buf; 350 *allocatedNamesNb = nbFiles; 351 352 return fileTable; 353 } 354 355 356 /*-**************************************** 357 * Console log 358 ******************************************/ 359 int g_utilDisplayLevel; 360 361 362 363 /*-**************************************** 364 * count the number of physical cores 365 ******************************************/ 366 367 #if defined(_WIN32) || defined(WIN32) 368 369 #include <windows.h> 370 371 typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); 372 373 int UTIL_countPhysicalCores(void) 374 { 375 static int numPhysicalCores = 0; 376 if (numPhysicalCores != 0) return numPhysicalCores; 377 378 { LPFN_GLPI glpi; 379 BOOL done = FALSE; 380 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; 381 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; 382 DWORD returnLength = 0; 383 size_t byteOffset = 0; 384 385 glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), 386 "GetLogicalProcessorInformation"); 387 388 if (glpi == NULL) { 389 goto failed; 390 } 391 392 while(!done) { 393 DWORD rc = glpi(buffer, &returnLength); 394 if (FALSE == rc) { 395 if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 396 if (buffer) 397 free(buffer); 398 buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); 399 400 if (buffer == NULL) { 401 perror("zstd"); 402 exit(1); 403 } 404 } else { 405 /* some other error */ 406 goto failed; 407 } 408 } else { 409 done = TRUE; 410 } 411 } 412 413 ptr = buffer; 414 415 while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { 416 417 if (ptr->Relationship == RelationProcessorCore) { 418 numPhysicalCores++; 419 } 420 421 ptr++; 422 byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); 423 } 424 425 free(buffer); 426 427 return numPhysicalCores; 428 } 429 430 failed: 431 /* try to fall back on GetSystemInfo */ 432 { SYSTEM_INFO sysinfo; 433 GetSystemInfo(&sysinfo); 434 numPhysicalCores = sysinfo.dwNumberOfProcessors; 435 if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */ 436 } 437 return numPhysicalCores; 438 } 439 440 #elif defined(__APPLE__) 441 442 #include <sys/sysctl.h> 443 444 /* Use apple-provided syscall 445 * see: man 3 sysctl */ 446 int UTIL_countPhysicalCores(void) 447 { 448 static S32 numPhysicalCores = 0; /* apple specifies int32_t */ 449 if (numPhysicalCores != 0) return numPhysicalCores; 450 451 { size_t size = sizeof(S32); 452 int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0); 453 if (ret != 0) { 454 if (errno == ENOENT) { 455 /* entry not present, fall back on 1 */ 456 numPhysicalCores = 1; 457 } else { 458 perror("zstd: can't get number of physical cpus"); 459 exit(1); 460 } 461 } 462 463 return numPhysicalCores; 464 } 465 } 466 467 #elif defined(__linux__) 468 469 /* parse /proc/cpuinfo 470 * siblings / cpu cores should give hyperthreading ratio 471 * otherwise fall back on sysconf */ 472 int UTIL_countPhysicalCores(void) 473 { 474 static int numPhysicalCores = 0; 475 476 if (numPhysicalCores != 0) return numPhysicalCores; 477 478 numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 479 if (numPhysicalCores == -1) { 480 /* value not queryable, fall back on 1 */ 481 return numPhysicalCores = 1; 482 } 483 484 /* try to determine if there's hyperthreading */ 485 { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); 486 #define BUF_SIZE 80 487 char buff[BUF_SIZE]; 488 489 int siblings = 0; 490 int cpu_cores = 0; 491 int ratio = 1; 492 493 if (cpuinfo == NULL) { 494 /* fall back on the sysconf value */ 495 return numPhysicalCores; 496 } 497 498 /* assume the cpu cores/siblings values will be constant across all 499 * present processors */ 500 while (!feof(cpuinfo)) { 501 if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { 502 if (strncmp(buff, "siblings", 8) == 0) { 503 const char* const sep = strchr(buff, ':'); 504 if (*sep == '\0') { 505 /* formatting was broken? */ 506 goto failed; 507 } 508 509 siblings = atoi(sep + 1); 510 } 511 if (strncmp(buff, "cpu cores", 9) == 0) { 512 const char* const sep = strchr(buff, ':'); 513 if (*sep == '\0') { 514 /* formatting was broken? */ 515 goto failed; 516 } 517 518 cpu_cores = atoi(sep + 1); 519 } 520 } else if (ferror(cpuinfo)) { 521 /* fall back on the sysconf value */ 522 goto failed; 523 } 524 } 525 if (siblings && cpu_cores) { 526 ratio = siblings / cpu_cores; 527 } 528 failed: 529 fclose(cpuinfo); 530 return numPhysicalCores = numPhysicalCores / ratio; 531 } 532 } 533 534 #elif defined(__FreeBSD__) 535 536 #include <sys/param.h> 537 #include <sys/sysctl.h> 538 539 /* Use physical core sysctl when available 540 * see: man 4 smp, man 3 sysctl */ 541 int UTIL_countPhysicalCores(void) 542 { 543 static int numPhysicalCores = 0; /* freebsd sysctl is native int sized */ 544 if (numPhysicalCores != 0) return numPhysicalCores; 545 546 #if __FreeBSD_version >= 1300008 547 { size_t size = sizeof(numPhysicalCores); 548 int ret = sysctlbyname("kern.smp.cores", &numPhysicalCores, &size, NULL, 0); 549 if (ret == 0) return numPhysicalCores; 550 if (errno != ENOENT) { 551 perror("zstd: can't get number of physical cpus"); 552 exit(1); 553 } 554 /* sysctl not present, fall through to older sysconf method */ 555 } 556 #endif 557 558 numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 559 if (numPhysicalCores == -1) { 560 /* value not queryable, fall back on 1 */ 561 numPhysicalCores = 1; 562 } 563 return numPhysicalCores; 564 } 565 566 #elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) 567 568 /* Use POSIX sysconf 569 * see: man 3 sysconf */ 570 int UTIL_countPhysicalCores(void) 571 { 572 static int numPhysicalCores = 0; 573 574 if (numPhysicalCores != 0) return numPhysicalCores; 575 576 numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 577 if (numPhysicalCores == -1) { 578 /* value not queryable, fall back on 1 */ 579 return numPhysicalCores = 1; 580 } 581 return numPhysicalCores; 582 } 583 584 #else 585 586 int UTIL_countPhysicalCores(void) 587 { 588 /* assume 1 */ 589 return 1; 590 } 591 592 #endif 593 594 #if defined (__cplusplus) 595 } 596 #endif 597