1 //===- Job.cpp - Command to Execute ---------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Driver/Job.h" 10 #include "InputInfo.h" 11 #include "clang/Basic/LLVM.h" 12 #include "clang/Driver/Driver.h" 13 #include "clang/Driver/DriverDiagnostic.h" 14 #include "clang/Driver/Tool.h" 15 #include "clang/Driver/ToolChain.h" 16 #include "llvm/ADT/ArrayRef.h" 17 #include "llvm/ADT/SmallString.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/ADT/StringRef.h" 20 #include "llvm/ADT/StringSet.h" 21 #include "llvm/ADT/StringSwitch.h" 22 #include "llvm/Support/FileSystem.h" 23 #include "llvm/Support/Path.h" 24 #include "llvm/Support/Program.h" 25 #include "llvm/Support/raw_ostream.h" 26 #include <algorithm> 27 #include <cassert> 28 #include <cstddef> 29 #include <string> 30 #include <system_error> 31 #include <utility> 32 33 using namespace clang; 34 using namespace driver; 35 36 Command::Command(const Action &Source, const Tool &Creator, 37 const char *Executable, 38 const llvm::opt::ArgStringList &Arguments, 39 ArrayRef<InputInfo> Inputs) 40 : Source(Source), Creator(Creator), Executable(Executable), 41 Arguments(Arguments) { 42 for (const auto &II : Inputs) 43 if (II.isFilename()) 44 InputFilenames.push_back(II.getFilename()); 45 } 46 47 /// Check if the compiler flag in question should be skipped when 48 /// emitting a reproducer. Also track how many arguments it has and if the 49 /// option is some kind of include path. 50 static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum, 51 bool &IsInclude) { 52 SkipNum = 2; 53 // These flags are all of the form -Flag <Arg> and are treated as two 54 // arguments. Therefore, we need to skip the flag and the next argument. 55 bool ShouldSkip = llvm::StringSwitch<bool>(Flag) 56 .Cases("-MF", "-MT", "-MQ", "-serialize-diagnostic-file", true) 57 .Cases("-o", "-dependency-file", true) 58 .Cases("-fdebug-compilation-dir", "-diagnostic-log-file", true) 59 .Cases("-dwarf-debug-flags", "-ivfsoverlay", true) 60 .Default(false); 61 if (ShouldSkip) 62 return true; 63 64 // Some include flags shouldn't be skipped if we have a crash VFS 65 IsInclude = llvm::StringSwitch<bool>(Flag) 66 .Cases("-include", "-header-include-file", true) 67 .Cases("-idirafter", "-internal-isystem", "-iwithprefix", true) 68 .Cases("-internal-externc-isystem", "-iprefix", true) 69 .Cases("-iwithprefixbefore", "-isystem", "-iquote", true) 70 .Cases("-isysroot", "-I", "-F", "-resource-dir", true) 71 .Cases("-iframework", "-include-pch", true) 72 .Default(false); 73 if (IsInclude) 74 return !HaveCrashVFS; 75 76 // The remaining flags are treated as a single argument. 77 78 // These flags are all of the form -Flag and have no second argument. 79 ShouldSkip = llvm::StringSwitch<bool>(Flag) 80 .Cases("-M", "-MM", "-MG", "-MP", "-MD", true) 81 .Case("-MMD", true) 82 .Default(false); 83 84 // Match found. 85 SkipNum = 1; 86 if (ShouldSkip) 87 return true; 88 89 // These flags are treated as a single argument (e.g., -F<Dir>). 90 StringRef FlagRef(Flag); 91 IsInclude = FlagRef.startswith("-F") || FlagRef.startswith("-I"); 92 if (IsInclude) 93 return !HaveCrashVFS; 94 if (FlagRef.startswith("-fmodules-cache-path=")) 95 return true; 96 97 SkipNum = 0; 98 return false; 99 } 100 101 void Command::printArg(raw_ostream &OS, StringRef Arg, bool Quote) { 102 const bool Escape = Arg.find_first_of(" \"\\$") != StringRef::npos; 103 104 if (!Quote && !Escape) { 105 OS << Arg; 106 return; 107 } 108 109 // Quote and escape. This isn't really complete, but good enough. 110 OS << '"'; 111 for (const auto c : Arg) { 112 if (c == '"' || c == '\\' || c == '$') 113 OS << '\\'; 114 OS << c; 115 } 116 OS << '"'; 117 } 118 119 void Command::writeResponseFile(raw_ostream &OS) const { 120 // In a file list, we only write the set of inputs to the response file 121 if (Creator.getResponseFilesSupport() == Tool::RF_FileList) { 122 for (const auto *Arg : InputFileList) { 123 OS << Arg << '\n'; 124 } 125 return; 126 } 127 128 // In regular response files, we send all arguments to the response file. 129 // Wrapping all arguments in double quotes ensures that both Unix tools and 130 // Windows tools understand the response file. 131 for (const auto *Arg : Arguments) { 132 OS << '"'; 133 134 for (; *Arg != '\0'; Arg++) { 135 if (*Arg == '\"' || *Arg == '\\') { 136 OS << '\\'; 137 } 138 OS << *Arg; 139 } 140 141 OS << "\" "; 142 } 143 } 144 145 void Command::buildArgvForResponseFile( 146 llvm::SmallVectorImpl<const char *> &Out) const { 147 // When not a file list, all arguments are sent to the response file. 148 // This leaves us to set the argv to a single parameter, requesting the tool 149 // to read the response file. 150 if (Creator.getResponseFilesSupport() != Tool::RF_FileList) { 151 Out.push_back(Executable); 152 Out.push_back(ResponseFileFlag.c_str()); 153 return; 154 } 155 156 llvm::StringSet<> Inputs; 157 for (const auto *InputName : InputFileList) 158 Inputs.insert(InputName); 159 Out.push_back(Executable); 160 // In a file list, build args vector ignoring parameters that will go in the 161 // response file (elements of the InputFileList vector) 162 bool FirstInput = true; 163 for (const auto *Arg : Arguments) { 164 if (Inputs.count(Arg) == 0) { 165 Out.push_back(Arg); 166 } else if (FirstInput) { 167 FirstInput = false; 168 Out.push_back(Creator.getResponseFileFlag()); 169 Out.push_back(ResponseFile); 170 } 171 } 172 } 173 174 /// Rewrite relative include-like flag paths to absolute ones. 175 static void 176 rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx, 177 size_t NumArgs, 178 llvm::SmallVectorImpl<llvm::SmallString<128>> &IncFlags) { 179 using namespace llvm; 180 using namespace sys; 181 182 auto getAbsPath = [](StringRef InInc, SmallVectorImpl<char> &OutInc) -> bool { 183 if (path::is_absolute(InInc)) // Nothing to do here... 184 return false; 185 std::error_code EC = fs::current_path(OutInc); 186 if (EC) 187 return false; 188 path::append(OutInc, InInc); 189 return true; 190 }; 191 192 SmallString<128> NewInc; 193 if (NumArgs == 1) { 194 StringRef FlagRef(Args[Idx + NumArgs - 1]); 195 assert((FlagRef.startswith("-F") || FlagRef.startswith("-I")) && 196 "Expecting -I or -F"); 197 StringRef Inc = FlagRef.slice(2, StringRef::npos); 198 if (getAbsPath(Inc, NewInc)) { 199 SmallString<128> NewArg(FlagRef.slice(0, 2)); 200 NewArg += NewInc; 201 IncFlags.push_back(std::move(NewArg)); 202 } 203 return; 204 } 205 206 assert(NumArgs == 2 && "Not expecting more than two arguments"); 207 StringRef Inc(Args[Idx + NumArgs - 1]); 208 if (!getAbsPath(Inc, NewInc)) 209 return; 210 IncFlags.push_back(SmallString<128>(Args[Idx])); 211 IncFlags.push_back(std::move(NewInc)); 212 } 213 214 void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, 215 CrashReportInfo *CrashInfo) const { 216 // Always quote the exe. 217 OS << ' '; 218 printArg(OS, Executable, /*Quote=*/true); 219 220 ArrayRef<const char *> Args = Arguments; 221 SmallVector<const char *, 128> ArgsRespFile; 222 if (ResponseFile != nullptr) { 223 buildArgvForResponseFile(ArgsRespFile); 224 Args = ArrayRef<const char *>(ArgsRespFile).slice(1); // no executable name 225 } 226 227 bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty(); 228 for (size_t i = 0, e = Args.size(); i < e; ++i) { 229 const char *const Arg = Args[i]; 230 231 if (CrashInfo) { 232 int NumArgs = 0; 233 bool IsInclude = false; 234 if (skipArgs(Arg, HaveCrashVFS, NumArgs, IsInclude)) { 235 i += NumArgs - 1; 236 continue; 237 } 238 239 // Relative includes need to be expanded to absolute paths. 240 if (HaveCrashVFS && IsInclude) { 241 SmallVector<SmallString<128>, 2> NewIncFlags; 242 rewriteIncludes(Args, i, NumArgs, NewIncFlags); 243 if (!NewIncFlags.empty()) { 244 for (auto &F : NewIncFlags) { 245 OS << ' '; 246 printArg(OS, F.c_str(), Quote); 247 } 248 i += NumArgs - 1; 249 continue; 250 } 251 } 252 253 auto Found = llvm::find_if(InputFilenames, 254 [&Arg](StringRef IF) { return IF == Arg; }); 255 if (Found != InputFilenames.end() && 256 (i == 0 || StringRef(Args[i - 1]) != "-main-file-name")) { 257 // Replace the input file name with the crashinfo's file name. 258 OS << ' '; 259 StringRef ShortName = llvm::sys::path::filename(CrashInfo->Filename); 260 printArg(OS, ShortName.str(), Quote); 261 continue; 262 } 263 } 264 265 OS << ' '; 266 printArg(OS, Arg, Quote); 267 } 268 269 if (CrashInfo && HaveCrashVFS) { 270 OS << ' '; 271 printArg(OS, "-ivfsoverlay", Quote); 272 OS << ' '; 273 printArg(OS, CrashInfo->VFSPath.str(), Quote); 274 275 // The leftover modules from the crash are stored in 276 // <name>.cache/vfs/modules 277 // Leave it untouched for pcm inspection and provide a clean/empty dir 278 // path to contain the future generated module cache: 279 // <name>.cache/vfs/repro-modules 280 SmallString<128> RelModCacheDir = llvm::sys::path::parent_path( 281 llvm::sys::path::parent_path(CrashInfo->VFSPath)); 282 llvm::sys::path::append(RelModCacheDir, "repro-modules"); 283 284 std::string ModCachePath = "-fmodules-cache-path="; 285 ModCachePath.append(RelModCacheDir.c_str()); 286 287 OS << ' '; 288 printArg(OS, ModCachePath, Quote); 289 } 290 291 if (ResponseFile != nullptr) { 292 OS << "\n Arguments passed via response file:\n"; 293 writeResponseFile(OS); 294 // Avoiding duplicated newline terminator, since FileLists are 295 // newline-separated. 296 if (Creator.getResponseFilesSupport() != Tool::RF_FileList) 297 OS << "\n"; 298 OS << " (end of response file)"; 299 } 300 301 OS << Terminator; 302 } 303 304 void Command::setResponseFile(const char *FileName) { 305 ResponseFile = FileName; 306 ResponseFileFlag = Creator.getResponseFileFlag(); 307 ResponseFileFlag += FileName; 308 } 309 310 void Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { 311 Environment.reserve(NewEnvironment.size() + 1); 312 Environment.assign(NewEnvironment.begin(), NewEnvironment.end()); 313 Environment.push_back(nullptr); 314 } 315 316 int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 317 std::string *ErrMsg, bool *ExecutionFailed) const { 318 if (PrintInputFilenames) { 319 for (const char *Arg : InputFilenames) 320 llvm::outs() << llvm::sys::path::filename(Arg) << "\n"; 321 llvm::outs().flush(); 322 } 323 324 SmallVector<const char*, 128> Argv; 325 326 Optional<ArrayRef<StringRef>> Env; 327 std::vector<StringRef> ArgvVectorStorage; 328 if (!Environment.empty()) { 329 assert(Environment.back() == nullptr && 330 "Environment vector should be null-terminated by now"); 331 ArgvVectorStorage = llvm::toStringRefArray(Environment.data()); 332 Env = makeArrayRef(ArgvVectorStorage); 333 } 334 335 if (ResponseFile == nullptr) { 336 Argv.push_back(Executable); 337 Argv.append(Arguments.begin(), Arguments.end()); 338 Argv.push_back(nullptr); 339 340 auto Args = llvm::toStringRefArray(Argv.data()); 341 return llvm::sys::ExecuteAndWait( 342 Executable, Args, Env, Redirects, /*secondsToWait*/ 0, 343 /*memoryLimit*/ 0, ErrMsg, ExecutionFailed); 344 } 345 346 // We need to put arguments in a response file (command is too large) 347 // Open stream to store the response file contents 348 std::string RespContents; 349 llvm::raw_string_ostream SS(RespContents); 350 351 // Write file contents and build the Argv vector 352 writeResponseFile(SS); 353 buildArgvForResponseFile(Argv); 354 Argv.push_back(nullptr); 355 SS.flush(); 356 357 // Save the response file in the appropriate encoding 358 if (std::error_code EC = writeFileWithEncoding( 359 ResponseFile, RespContents, Creator.getResponseFileEncoding())) { 360 if (ErrMsg) 361 *ErrMsg = EC.message(); 362 if (ExecutionFailed) 363 *ExecutionFailed = true; 364 return -1; 365 } 366 367 auto Args = llvm::toStringRefArray(Argv.data()); 368 return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects, 369 /*secondsToWait*/ 0, 370 /*memoryLimit*/ 0, ErrMsg, ExecutionFailed); 371 } 372 373 FallbackCommand::FallbackCommand(const Action &Source_, const Tool &Creator_, 374 const char *Executable_, 375 const llvm::opt::ArgStringList &Arguments_, 376 ArrayRef<InputInfo> Inputs, 377 std::unique_ptr<Command> Fallback_) 378 : Command(Source_, Creator_, Executable_, Arguments_, Inputs), 379 Fallback(std::move(Fallback_)) {} 380 381 void FallbackCommand::Print(raw_ostream &OS, const char *Terminator, 382 bool Quote, CrashReportInfo *CrashInfo) const { 383 Command::Print(OS, "", Quote, CrashInfo); 384 OS << " ||"; 385 Fallback->Print(OS, Terminator, Quote, CrashInfo); 386 } 387 388 static bool ShouldFallback(int ExitCode) { 389 // FIXME: We really just want to fall back for internal errors, such 390 // as when some symbol cannot be mangled, when we should be able to 391 // parse something but can't, etc. 392 return ExitCode != 0; 393 } 394 395 int FallbackCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 396 std::string *ErrMsg, bool *ExecutionFailed) const { 397 int PrimaryStatus = Command::Execute(Redirects, ErrMsg, ExecutionFailed); 398 if (!ShouldFallback(PrimaryStatus)) 399 return PrimaryStatus; 400 401 // Clear ExecutionFailed and ErrMsg before falling back. 402 if (ErrMsg) 403 ErrMsg->clear(); 404 if (ExecutionFailed) 405 *ExecutionFailed = false; 406 407 const Driver &D = getCreator().getToolChain().getDriver(); 408 D.Diag(diag::warn_drv_invoking_fallback) << Fallback->getExecutable(); 409 410 int SecondaryStatus = Fallback->Execute(Redirects, ErrMsg, ExecutionFailed); 411 return SecondaryStatus; 412 } 413 414 ForceSuccessCommand::ForceSuccessCommand( 415 const Action &Source_, const Tool &Creator_, const char *Executable_, 416 const llvm::opt::ArgStringList &Arguments_, ArrayRef<InputInfo> Inputs) 417 : Command(Source_, Creator_, Executable_, Arguments_, Inputs) {} 418 419 void ForceSuccessCommand::Print(raw_ostream &OS, const char *Terminator, 420 bool Quote, CrashReportInfo *CrashInfo) const { 421 Command::Print(OS, "", Quote, CrashInfo); 422 OS << " || (exit 0)" << Terminator; 423 } 424 425 int ForceSuccessCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 426 std::string *ErrMsg, 427 bool *ExecutionFailed) const { 428 int Status = Command::Execute(Redirects, ErrMsg, ExecutionFailed); 429 (void)Status; 430 if (ExecutionFailed) 431 *ExecutionFailed = false; 432 return 0; 433 } 434 435 void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote, 436 CrashReportInfo *CrashInfo) const { 437 for (const auto &Job : *this) 438 Job.Print(OS, Terminator, Quote, CrashInfo); 439 } 440 441 void JobList::clear() { Jobs.clear(); } 442