1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the pruning of a directory based on least recently used. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Support/CachePruning.h" 14 #include "llvm/ADT/StringRef.h" 15 #include "llvm/Support/Debug.h" 16 #include "llvm/Support/Errc.h" 17 #include "llvm/Support/Error.h" 18 #include "llvm/Support/FileSystem.h" 19 #include "llvm/Support/Path.h" 20 #include "llvm/Support/raw_ostream.h" 21 22 #define DEBUG_TYPE "cache-pruning" 23 24 #include <set> 25 #include <system_error> 26 27 using namespace llvm; 28 29 namespace { 30 struct FileInfo { 31 sys::TimePoint<> Time; 32 uint64_t Size; 33 std::string Path; 34 35 /// Used to determine which files to prune first. Also used to determine 36 /// set membership, so must take into account all fields. 37 bool operator<(const FileInfo &Other) const { 38 return std::tie(Time, Other.Size, Path) < 39 std::tie(Other.Time, Size, Other.Path); 40 } 41 }; 42 } // anonymous namespace 43 44 /// Write a new timestamp file with the given path. This is used for the pruning 45 /// interval option. 46 static void writeTimestampFile(StringRef TimestampFile) { 47 std::error_code EC; 48 raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::OF_None); 49 } 50 51 static Expected<std::chrono::seconds> parseDuration(StringRef Duration) { 52 if (Duration.empty()) 53 return make_error<StringError>("Duration must not be empty", 54 inconvertibleErrorCode()); 55 56 StringRef NumStr = Duration.slice(0, Duration.size()-1); 57 uint64_t Num; 58 if (NumStr.getAsInteger(0, Num)) 59 return make_error<StringError>("'" + NumStr + "' not an integer", 60 inconvertibleErrorCode()); 61 62 switch (Duration.back()) { 63 case 's': 64 return std::chrono::seconds(Num); 65 case 'm': 66 return std::chrono::minutes(Num); 67 case 'h': 68 return std::chrono::hours(Num); 69 default: 70 return make_error<StringError>("'" + Duration + 71 "' must end with one of 's', 'm' or 'h'", 72 inconvertibleErrorCode()); 73 } 74 } 75 76 Expected<CachePruningPolicy> 77 llvm::parseCachePruningPolicy(StringRef PolicyStr) { 78 CachePruningPolicy Policy; 79 std::pair<StringRef, StringRef> P = {"", PolicyStr}; 80 while (!P.second.empty()) { 81 P = P.second.split(':'); 82 83 StringRef Key, Value; 84 std::tie(Key, Value) = P.first.split('='); 85 if (Key == "prune_interval") { 86 auto DurationOrErr = parseDuration(Value); 87 if (!DurationOrErr) 88 return DurationOrErr.takeError(); 89 Policy.Interval = *DurationOrErr; 90 } else if (Key == "prune_after") { 91 auto DurationOrErr = parseDuration(Value); 92 if (!DurationOrErr) 93 return DurationOrErr.takeError(); 94 Policy.Expiration = *DurationOrErr; 95 } else if (Key == "cache_size") { 96 if (Value.back() != '%') 97 return make_error<StringError>("'" + Value + "' must be a percentage", 98 inconvertibleErrorCode()); 99 StringRef SizeStr = Value.drop_back(); 100 uint64_t Size; 101 if (SizeStr.getAsInteger(0, Size)) 102 return make_error<StringError>("'" + SizeStr + "' not an integer", 103 inconvertibleErrorCode()); 104 if (Size > 100) 105 return make_error<StringError>("'" + SizeStr + 106 "' must be between 0 and 100", 107 inconvertibleErrorCode()); 108 Policy.MaxSizePercentageOfAvailableSpace = Size; 109 } else if (Key == "cache_size_bytes") { 110 uint64_t Mult = 1; 111 switch (tolower(Value.back())) { 112 case 'k': 113 Mult = 1024; 114 Value = Value.drop_back(); 115 break; 116 case 'm': 117 Mult = 1024 * 1024; 118 Value = Value.drop_back(); 119 break; 120 case 'g': 121 Mult = 1024 * 1024 * 1024; 122 Value = Value.drop_back(); 123 break; 124 } 125 uint64_t Size; 126 if (Value.getAsInteger(0, Size)) 127 return make_error<StringError>("'" + Value + "' not an integer", 128 inconvertibleErrorCode()); 129 Policy.MaxSizeBytes = Size * Mult; 130 } else if (Key == "cache_size_files") { 131 if (Value.getAsInteger(0, Policy.MaxSizeFiles)) 132 return make_error<StringError>("'" + Value + "' not an integer", 133 inconvertibleErrorCode()); 134 } else { 135 return make_error<StringError>("Unknown key: '" + Key + "'", 136 inconvertibleErrorCode()); 137 } 138 } 139 140 return Policy; 141 } 142 143 /// Prune the cache of files that haven't been accessed in a long time. 144 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) { 145 using namespace std::chrono; 146 147 if (Path.empty()) 148 return false; 149 150 bool isPathDir; 151 if (sys::fs::is_directory(Path, isPathDir)) 152 return false; 153 154 if (!isPathDir) 155 return false; 156 157 Policy.MaxSizePercentageOfAvailableSpace = 158 std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u); 159 160 if (Policy.Expiration == seconds(0) && 161 Policy.MaxSizePercentageOfAvailableSpace == 0 && 162 Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) { 163 LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n"); 164 // Nothing will be pruned, early exit 165 return false; 166 } 167 168 // Try to stat() the timestamp file. 169 SmallString<128> TimestampFile(Path); 170 sys::path::append(TimestampFile, "llvmcache.timestamp"); 171 sys::fs::file_status FileStatus; 172 const auto CurrentTime = system_clock::now(); 173 if (auto EC = sys::fs::status(TimestampFile, FileStatus)) { 174 if (EC == errc::no_such_file_or_directory) { 175 // If the timestamp file wasn't there, create one now. 176 writeTimestampFile(TimestampFile); 177 } else { 178 // Unknown error? 179 return false; 180 } 181 } else { 182 if (!Policy.Interval) 183 return false; 184 if (Policy.Interval != seconds(0)) { 185 // Check whether the time stamp is older than our pruning interval. 186 // If not, do nothing. 187 const auto TimeStampModTime = FileStatus.getLastModificationTime(); 188 auto TimeStampAge = CurrentTime - TimeStampModTime; 189 if (TimeStampAge <= *Policy.Interval) { 190 LLVM_DEBUG(dbgs() << "Timestamp file too recent (" 191 << duration_cast<seconds>(TimeStampAge).count() 192 << "s old), do not prune.\n"); 193 return false; 194 } 195 } 196 // Write a new timestamp file so that nobody else attempts to prune. 197 // There is a benign race condition here, if two processes happen to 198 // notice at the same time that the timestamp is out-of-date. 199 writeTimestampFile(TimestampFile); 200 } 201 202 // Keep track of files to delete to get below the size limit. 203 // Order by time of last use so that recently used files are preserved. 204 std::set<FileInfo> FileInfos; 205 uint64_t TotalSize = 0; 206 207 // Walk the entire directory cache, looking for unused files. 208 std::error_code EC; 209 SmallString<128> CachePathNative; 210 sys::path::native(Path, CachePathNative); 211 // Walk all of the files within this directory. 212 for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd; 213 File != FileEnd && !EC; File.increment(EC)) { 214 // Ignore filenames not beginning with "llvmcache-" or "Thin-". This 215 // includes the timestamp file as well as any files created by the user. 216 // This acts as a safeguard against data loss if the user specifies the 217 // wrong directory as their cache directory. 218 StringRef filename = sys::path::filename(File->path()); 219 if (!filename.startswith("llvmcache-") && !filename.startswith("Thin-")) 220 continue; 221 222 // Look at this file. If we can't stat it, there's nothing interesting 223 // there. 224 ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status(); 225 if (!StatusOrErr) { 226 LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n"); 227 continue; 228 } 229 230 // If the file hasn't been used recently enough, delete it 231 const auto FileAccessTime = StatusOrErr->getLastAccessedTime(); 232 auto FileAge = CurrentTime - FileAccessTime; 233 if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) { 234 LLVM_DEBUG(dbgs() << "Remove " << File->path() << " (" 235 << duration_cast<seconds>(FileAge).count() 236 << "s old)\n"); 237 sys::fs::remove(File->path()); 238 continue; 239 } 240 241 // Leave it here for now, but add it to the list of size-based pruning. 242 TotalSize += StatusOrErr->getSize(); 243 FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()}); 244 } 245 246 auto FileInfo = FileInfos.begin(); 247 size_t NumFiles = FileInfos.size(); 248 249 auto RemoveCacheFile = [&]() { 250 // Remove the file. 251 sys::fs::remove(FileInfo->Path); 252 // Update size 253 TotalSize -= FileInfo->Size; 254 NumFiles--; 255 LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size " 256 << FileInfo->Size << "), new occupancy is " << TotalSize 257 << "%\n"); 258 ++FileInfo; 259 }; 260 261 // Prune for number of files. 262 if (Policy.MaxSizeFiles) 263 while (NumFiles > Policy.MaxSizeFiles) 264 RemoveCacheFile(); 265 266 // Prune for size now if needed 267 if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) { 268 auto ErrOrSpaceInfo = sys::fs::disk_space(Path); 269 if (!ErrOrSpaceInfo) { 270 report_fatal_error("Can't get available size"); 271 } 272 sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get(); 273 auto AvailableSpace = TotalSize + SpaceInfo.free; 274 275 if (Policy.MaxSizePercentageOfAvailableSpace == 0) 276 Policy.MaxSizePercentageOfAvailableSpace = 100; 277 if (Policy.MaxSizeBytes == 0) 278 Policy.MaxSizeBytes = AvailableSpace; 279 auto TotalSizeTarget = std::min<uint64_t>( 280 AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull, 281 Policy.MaxSizeBytes); 282 283 LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace) 284 << "% target is: " 285 << Policy.MaxSizePercentageOfAvailableSpace << "%, " 286 << Policy.MaxSizeBytes << " bytes\n"); 287 288 // Remove the oldest accessed files first, till we get below the threshold. 289 while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end()) 290 RemoveCacheFile(); 291 } 292 return true; 293 } 294