1 //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" 10 #include "clang/Lex/DependencyDirectivesSourceMinimizer.h" 11 #include "llvm/Support/MemoryBuffer.h" 12 #include "llvm/Support/Threading.h" 13 14 using namespace clang; 15 using namespace tooling; 16 using namespace dependencies; 17 18 CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( 19 StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { 20 // Load the file and its content from the file system. 21 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile = 22 FS.openFileForRead(Filename); 23 if (!MaybeFile) 24 return MaybeFile.getError(); 25 llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status(); 26 if (!Stat) 27 return Stat.getError(); 28 29 llvm::vfs::File &F = **MaybeFile; 30 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer = 31 F.getBuffer(Stat->getName()); 32 if (!MaybeBuffer) 33 return MaybeBuffer.getError(); 34 35 llvm::SmallString<1024> MinimizedFileContents; 36 // Minimize the file down to directives that might affect the dependencies. 37 const auto &Buffer = *MaybeBuffer; 38 SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens; 39 if (!Minimize || minimizeSourceToDependencyDirectives( 40 Buffer->getBuffer(), MinimizedFileContents, Tokens)) { 41 // Use the original file unless requested otherwise, or 42 // if the minimization failed. 43 // FIXME: Propage the diagnostic if desired by the client. 44 CachedFileSystemEntry Result; 45 Result.MaybeStat = std::move(*Stat); 46 Result.Contents.reserve(Buffer->getBufferSize() + 1); 47 Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); 48 // Implicitly null terminate the contents for Clang's lexer. 49 Result.Contents.push_back('\0'); 50 Result.Contents.pop_back(); 51 return Result; 52 } 53 54 CachedFileSystemEntry Result; 55 size_t Size = MinimizedFileContents.size(); 56 Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), 57 Stat->getLastModificationTime(), 58 Stat->getUser(), Stat->getGroup(), Size, 59 Stat->getType(), Stat->getPermissions()); 60 // The contents produced by the minimizer must be null terminated. 61 assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && 62 "not null terminated contents"); 63 // Even though there's an implicit null terminator in the minimized contents, 64 // we want to temporarily make it explicit. This will ensure that the 65 // std::move will preserve it even if it needs to do a copy if the 66 // SmallString still has the small capacity. 67 MinimizedFileContents.push_back('\0'); 68 Result.Contents = std::move(MinimizedFileContents); 69 // Now make the null terminator implicit again, so that Clang's lexer can find 70 // it right where the buffer ends. 71 Result.Contents.pop_back(); 72 73 // Compute the skipped PP ranges that speedup skipping over inactive 74 // preprocessor blocks. 75 llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32> 76 SkippedRanges; 77 minimize_source_to_dependency_directives::computeSkippedRanges(Tokens, 78 SkippedRanges); 79 PreprocessorSkippedRangeMapping Mapping; 80 for (const auto &Range : SkippedRanges) { 81 if (Range.Length < 16) { 82 // Ignore small ranges as non-profitable. 83 // FIXME: This is a heuristic, its worth investigating the tradeoffs 84 // when it should be applied. 85 continue; 86 } 87 Mapping[Range.Offset] = Range.Length; 88 } 89 Result.PPSkippedRangeMapping = std::move(Mapping); 90 91 return Result; 92 } 93 94 CachedFileSystemEntry 95 CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { 96 assert(Stat.isDirectory() && "not a directory!"); 97 auto Result = CachedFileSystemEntry(); 98 Result.MaybeStat = std::move(Stat); 99 return Result; 100 } 101 102 DependencyScanningFilesystemSharedCache:: 103 DependencyScanningFilesystemSharedCache() { 104 // This heuristic was chosen using a empirical testing on a 105 // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache 106 // sharding gives a performance edge by reducing the lock contention. 107 // FIXME: A better heuristic might also consider the OS to account for 108 // the different cost of lock contention on different OSes. 109 NumShards = std::max(2u, llvm::hardware_concurrency() / 4); 110 CacheShards = std::make_unique<CacheShard[]>(NumShards); 111 } 112 113 /// Returns a cache entry for the corresponding key. 114 /// 115 /// A new cache entry is created if the key is not in the cache. This is a 116 /// thread safe call. 117 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & 118 DependencyScanningFilesystemSharedCache::get(StringRef Key) { 119 CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; 120 std::unique_lock<std::mutex> LockGuard(Shard.CacheLock); 121 auto It = Shard.Cache.try_emplace(Key); 122 return It.first->getValue(); 123 } 124 125 llvm::ErrorOr<const CachedFileSystemEntry *> 126 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( 127 const StringRef Filename) { 128 if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) { 129 return Entry; 130 } 131 132 // FIXME: Handle PCM/PCH files. 133 // FIXME: Handle module map files. 134 135 bool KeepOriginalSource = IgnoredFiles.count(Filename); 136 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry 137 &SharedCacheEntry = SharedCache.get(Filename); 138 const CachedFileSystemEntry *Result; 139 { 140 std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock); 141 CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; 142 143 if (!CacheEntry.isValid()) { 144 llvm::vfs::FileSystem &FS = getUnderlyingFS(); 145 auto MaybeStatus = FS.status(Filename); 146 if (!MaybeStatus) 147 CacheEntry = CachedFileSystemEntry(MaybeStatus.getError()); 148 else if (MaybeStatus->isDirectory()) 149 CacheEntry = CachedFileSystemEntry::createDirectoryEntry( 150 std::move(*MaybeStatus)); 151 else 152 CacheEntry = CachedFileSystemEntry::createFileEntry( 153 Filename, FS, !KeepOriginalSource); 154 } 155 156 Result = &CacheEntry; 157 } 158 159 // Store the result in the local cache. 160 setCachedEntry(Filename, Result); 161 return Result; 162 } 163 164 llvm::ErrorOr<llvm::vfs::Status> 165 DependencyScanningWorkerFilesystem::status(const Twine &Path) { 166 SmallString<256> OwnedFilename; 167 StringRef Filename = Path.toStringRef(OwnedFilename); 168 const llvm::ErrorOr<const CachedFileSystemEntry *> Result = 169 getOrCreateFileSystemEntry(Filename); 170 if (!Result) 171 return Result.getError(); 172 return (*Result)->getStatus(); 173 } 174 175 namespace { 176 177 /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using 178 /// this subclass. 179 class MinimizedVFSFile final : public llvm::vfs::File { 180 public: 181 MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer, 182 llvm::vfs::Status Stat) 183 : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} 184 185 llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; } 186 187 const llvm::MemoryBuffer *getBufferPtr() const { return Buffer.get(); } 188 189 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> 190 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, 191 bool IsVolatile) override { 192 return std::move(Buffer); 193 } 194 195 std::error_code close() override { return {}; } 196 197 private: 198 std::unique_ptr<llvm::MemoryBuffer> Buffer; 199 llvm::vfs::Status Stat; 200 }; 201 202 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> 203 createFile(const CachedFileSystemEntry *Entry, 204 ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { 205 if (Entry->isDirectory()) 206 return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( 207 std::make_error_code(std::errc::is_a_directory)); 208 llvm::ErrorOr<StringRef> Contents = Entry->getContents(); 209 if (!Contents) 210 return Contents.getError(); 211 auto Result = std::make_unique<MinimizedVFSFile>( 212 llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), 213 /*RequiresNullTerminator=*/false), 214 *Entry->getStatus()); 215 if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings) 216 (*PPSkipMappings)[Result->getBufferPtr()] = 217 &Entry->getPPSkippedRangeMapping(); 218 return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( 219 std::unique_ptr<llvm::vfs::File>(std::move(Result))); 220 } 221 222 } // end anonymous namespace 223 224 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> 225 DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { 226 SmallString<256> OwnedFilename; 227 StringRef Filename = Path.toStringRef(OwnedFilename); 228 229 const llvm::ErrorOr<const CachedFileSystemEntry *> Result = 230 getOrCreateFileSystemEntry(Filename); 231 if (!Result) 232 return Result.getError(); 233 return createFile(Result.get(), PPSkipMappings); 234 } 235