1 //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" 10 #include "clang/Lex/DependencyDirectivesSourceMinimizer.h" 11 #include "llvm/Support/MemoryBuffer.h" 12 #include "llvm/Support/Threading.h" 13 14 using namespace clang; 15 using namespace tooling; 16 using namespace dependencies; 17 18 CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( 19 StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { 20 // Load the file and its content from the file system. 21 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile = 22 FS.openFileForRead(Filename); 23 if (!MaybeFile) 24 return MaybeFile.getError(); 25 llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status(); 26 if (!Stat) 27 return Stat.getError(); 28 29 llvm::vfs::File &F = **MaybeFile; 30 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer = 31 F.getBuffer(Stat->getName()); 32 if (!MaybeBuffer) 33 return MaybeBuffer.getError(); 34 35 llvm::SmallString<1024> MinimizedFileContents; 36 // Minimize the file down to directives that might affect the dependencies. 37 const auto &Buffer = *MaybeBuffer; 38 SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens; 39 if (!Minimize || minimizeSourceToDependencyDirectives( 40 Buffer->getBuffer(), MinimizedFileContents, Tokens)) { 41 // Use the original file unless requested otherwise, or 42 // if the minimization failed. 43 // FIXME: Propage the diagnostic if desired by the client. 44 CachedFileSystemEntry Result; 45 Result.MaybeStat = std::move(*Stat); 46 Result.Contents.reserve(Buffer->getBufferSize() + 1); 47 Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); 48 // Implicitly null terminate the contents for Clang's lexer. 49 Result.Contents.push_back('\0'); 50 Result.Contents.pop_back(); 51 return Result; 52 } 53 54 CachedFileSystemEntry Result; 55 size_t Size = MinimizedFileContents.size(); 56 Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), 57 Stat->getLastModificationTime(), 58 Stat->getUser(), Stat->getGroup(), Size, 59 Stat->getType(), Stat->getPermissions()); 60 // The contents produced by the minimizer must be null terminated. 61 assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && 62 "not null terminated contents"); 63 // Even though there's an implicit null terminator in the minimized contents, 64 // we want to temporarily make it explicit. This will ensure that the 65 // std::move will preserve it even if it needs to do a copy if the 66 // SmallString still has the small capacity. 67 MinimizedFileContents.push_back('\0'); 68 Result.Contents = std::move(MinimizedFileContents); 69 // Now make the null terminator implicit again, so that Clang's lexer can find 70 // it right where the buffer ends. 71 Result.Contents.pop_back(); 72 73 // Compute the skipped PP ranges that speedup skipping over inactive 74 // preprocessor blocks. 75 llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32> 76 SkippedRanges; 77 minimize_source_to_dependency_directives::computeSkippedRanges(Tokens, 78 SkippedRanges); 79 PreprocessorSkippedRangeMapping Mapping; 80 for (const auto &Range : SkippedRanges) { 81 if (Range.Length < 16) { 82 // Ignore small ranges as non-profitable. 83 // FIXME: This is a heuristic, its worth investigating the tradeoffs 84 // when it should be applied. 85 continue; 86 } 87 Mapping[Range.Offset] = Range.Length; 88 } 89 Result.PPSkippedRangeMapping = std::move(Mapping); 90 91 return Result; 92 } 93 94 CachedFileSystemEntry 95 CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { 96 assert(Stat.isDirectory() && "not a directory!"); 97 auto Result = CachedFileSystemEntry(); 98 Result.MaybeStat = std::move(Stat); 99 return Result; 100 } 101 102 DependencyScanningFilesystemSharedCache:: 103 DependencyScanningFilesystemSharedCache() { 104 // This heuristic was chosen using a empirical testing on a 105 // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache 106 // sharding gives a performance edge by reducing the lock contention. 107 // FIXME: A better heuristic might also consider the OS to account for 108 // the different cost of lock contention on different OSes. 109 NumShards = 110 std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4); 111 CacheShards = std::make_unique<CacheShard[]>(NumShards); 112 } 113 114 /// Returns a cache entry for the corresponding key. 115 /// 116 /// A new cache entry is created if the key is not in the cache. This is a 117 /// thread safe call. 118 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & 119 DependencyScanningFilesystemSharedCache::get(StringRef Key) { 120 CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; 121 std::unique_lock<std::mutex> LockGuard(Shard.CacheLock); 122 auto It = Shard.Cache.try_emplace(Key); 123 return It.first->getValue(); 124 } 125 126 /// Whitelist file extensions that should be minimized, treating no extension as 127 /// a source file that should be minimized. 128 /// 129 /// This is kinda hacky, it would be better if we knew what kind of file Clang 130 /// was expecting instead. 131 static bool shouldMinimize(StringRef Filename) { 132 StringRef Ext = llvm::sys::path::extension(Filename); 133 if (Ext.empty()) 134 return true; // C++ standard library 135 return llvm::StringSwitch<bool>(Ext) 136 .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) 137 .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) 138 .CasesLower(".m", ".mm", true) 139 .CasesLower(".i", ".ii", ".mi", ".mmi", true) 140 .CasesLower(".def", ".inc", true) 141 .Default(false); 142 } 143 144 145 static bool shouldCacheStatFailures(StringRef Filename) { 146 StringRef Ext = llvm::sys::path::extension(Filename); 147 if (Ext.empty()) 148 return false; // This may be the module cache directory. 149 return shouldMinimize(Filename); // Only cache stat failures on source files. 150 } 151 152 llvm::ErrorOr<const CachedFileSystemEntry *> 153 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( 154 const StringRef Filename) { 155 if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) { 156 return Entry; 157 } 158 159 // FIXME: Handle PCM/PCH files. 160 // FIXME: Handle module map files. 161 162 bool KeepOriginalSource = IgnoredFiles.count(Filename) || 163 !shouldMinimize(Filename); 164 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry 165 &SharedCacheEntry = SharedCache.get(Filename); 166 const CachedFileSystemEntry *Result; 167 { 168 std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock); 169 CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; 170 171 if (!CacheEntry.isValid()) { 172 llvm::vfs::FileSystem &FS = getUnderlyingFS(); 173 auto MaybeStatus = FS.status(Filename); 174 if (!MaybeStatus) { 175 if (!shouldCacheStatFailures(Filename)) 176 // HACK: We need to always restat non source files if the stat fails. 177 // This is because Clang first looks up the module cache and module 178 // files before building them, and then looks for them again. If we 179 // cache the stat failure, it won't see them the second time. 180 return MaybeStatus.getError(); 181 else 182 CacheEntry = CachedFileSystemEntry(MaybeStatus.getError()); 183 } else if (MaybeStatus->isDirectory()) 184 CacheEntry = CachedFileSystemEntry::createDirectoryEntry( 185 std::move(*MaybeStatus)); 186 else 187 CacheEntry = CachedFileSystemEntry::createFileEntry( 188 Filename, FS, !KeepOriginalSource); 189 } 190 191 Result = &CacheEntry; 192 } 193 194 // Store the result in the local cache. 195 setCachedEntry(Filename, Result); 196 return Result; 197 } 198 199 llvm::ErrorOr<llvm::vfs::Status> 200 DependencyScanningWorkerFilesystem::status(const Twine &Path) { 201 SmallString<256> OwnedFilename; 202 StringRef Filename = Path.toStringRef(OwnedFilename); 203 const llvm::ErrorOr<const CachedFileSystemEntry *> Result = 204 getOrCreateFileSystemEntry(Filename); 205 if (!Result) 206 return Result.getError(); 207 return (*Result)->getStatus(); 208 } 209 210 namespace { 211 212 /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using 213 /// this subclass. 214 class MinimizedVFSFile final : public llvm::vfs::File { 215 public: 216 MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer, 217 llvm::vfs::Status Stat) 218 : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} 219 220 static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> 221 create(const CachedFileSystemEntry *Entry, 222 ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings); 223 224 llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; } 225 226 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> 227 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, 228 bool IsVolatile) override { 229 return std::move(Buffer); 230 } 231 232 std::error_code close() override { return {}; } 233 234 private: 235 std::unique_ptr<llvm::MemoryBuffer> Buffer; 236 llvm::vfs::Status Stat; 237 }; 238 239 } // end anonymous namespace 240 241 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create( 242 const CachedFileSystemEntry *Entry, 243 ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { 244 if (Entry->isDirectory()) 245 return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( 246 std::make_error_code(std::errc::is_a_directory)); 247 llvm::ErrorOr<StringRef> Contents = Entry->getContents(); 248 if (!Contents) 249 return Contents.getError(); 250 auto Result = std::make_unique<MinimizedVFSFile>( 251 llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), 252 /*RequiresNullTerminator=*/false), 253 *Entry->getStatus()); 254 if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings) 255 (*PPSkipMappings)[Result->Buffer->getBufferStart()] = 256 &Entry->getPPSkippedRangeMapping(); 257 return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( 258 std::unique_ptr<llvm::vfs::File>(std::move(Result))); 259 } 260 261 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> 262 DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { 263 SmallString<256> OwnedFilename; 264 StringRef Filename = Path.toStringRef(OwnedFilename); 265 266 const llvm::ErrorOr<const CachedFileSystemEntry *> Result = 267 getOrCreateFileSystemEntry(Filename); 268 if (!Result) 269 return Result.getError(); 270 return MinimizedVFSFile::create(Result.get(), PPSkipMappings); 271 } 272