1 //===- Job.cpp - Command to Execute ---------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Driver/Job.h" 10 #include "InputInfo.h" 11 #include "clang/Basic/LLVM.h" 12 #include "clang/Driver/Driver.h" 13 #include "clang/Driver/DriverDiagnostic.h" 14 #include "clang/Driver/Tool.h" 15 #include "clang/Driver/ToolChain.h" 16 #include "llvm/ADT/ArrayRef.h" 17 #include "llvm/ADT/SmallString.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/ADT/StringRef.h" 20 #include "llvm/ADT/StringSet.h" 21 #include "llvm/ADT/StringSwitch.h" 22 #include "llvm/Support/CrashRecoveryContext.h" 23 #include "llvm/Support/FileSystem.h" 24 #include "llvm/Support/Path.h" 25 #include "llvm/Support/PrettyStackTrace.h" 26 #include "llvm/Support/Program.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include <algorithm> 29 #include <cassert> 30 #include <cstddef> 31 #include <string> 32 #include <system_error> 33 #include <utility> 34 35 using namespace clang; 36 using namespace driver; 37 38 Command::Command(const Action &Source, const Tool &Creator, 39 ResponseFileSupport ResponseSupport, const char *Executable, 40 const llvm::opt::ArgStringList &Arguments, 41 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs) 42 : Source(Source), Creator(Creator), ResponseSupport(ResponseSupport), 43 Executable(Executable), Arguments(Arguments) { 44 for (const auto &II : Inputs) 45 if (II.isFilename()) 46 InputFilenames.push_back(II.getFilename()); 47 for (const auto &II : Outputs) 48 if (II.isFilename()) 49 OutputFilenames.push_back(II.getFilename()); 50 } 51 52 /// Check if the compiler flag in question should be skipped when 53 /// emitting a reproducer. Also track how many arguments it has and if the 54 /// option is some kind of include path. 55 static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum, 56 bool &IsInclude) { 57 SkipNum = 2; 58 // These flags are all of the form -Flag <Arg> and are treated as two 59 // arguments. Therefore, we need to skip the flag and the next argument. 60 bool ShouldSkip = llvm::StringSwitch<bool>(Flag) 61 .Cases("-MF", "-MT", "-MQ", "-serialize-diagnostic-file", true) 62 .Cases("-o", "-dependency-file", true) 63 .Cases("-fdebug-compilation-dir", "-diagnostic-log-file", true) 64 .Cases("-dwarf-debug-flags", "-ivfsoverlay", true) 65 .Default(false); 66 if (ShouldSkip) 67 return true; 68 69 // Some include flags shouldn't be skipped if we have a crash VFS 70 IsInclude = llvm::StringSwitch<bool>(Flag) 71 .Cases("-include", "-header-include-file", true) 72 .Cases("-idirafter", "-internal-isystem", "-iwithprefix", true) 73 .Cases("-internal-externc-isystem", "-iprefix", true) 74 .Cases("-iwithprefixbefore", "-isystem", "-iquote", true) 75 .Cases("-isysroot", "-I", "-F", "-resource-dir", true) 76 .Cases("-iframework", "-include-pch", true) 77 .Default(false); 78 if (IsInclude) 79 return !HaveCrashVFS; 80 81 // The remaining flags are treated as a single argument. 82 83 // These flags are all of the form -Flag and have no second argument. 84 ShouldSkip = llvm::StringSwitch<bool>(Flag) 85 .Cases("-M", "-MM", "-MG", "-MP", "-MD", true) 86 .Case("-MMD", true) 87 .Default(false); 88 89 // Match found. 90 SkipNum = 1; 91 if (ShouldSkip) 92 return true; 93 94 // These flags are treated as a single argument (e.g., -F<Dir>). 95 StringRef FlagRef(Flag); 96 IsInclude = FlagRef.startswith("-F") || FlagRef.startswith("-I"); 97 if (IsInclude) 98 return !HaveCrashVFS; 99 if (FlagRef.startswith("-fmodules-cache-path=")) 100 return true; 101 102 SkipNum = 0; 103 return false; 104 } 105 106 void Command::writeResponseFile(raw_ostream &OS) const { 107 // In a file list, we only write the set of inputs to the response file 108 if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) { 109 for (const auto *Arg : InputFileList) { 110 OS << Arg << '\n'; 111 } 112 return; 113 } 114 115 // In regular response files, we send all arguments to the response file. 116 // Wrapping all arguments in double quotes ensures that both Unix tools and 117 // Windows tools understand the response file. 118 for (const auto *Arg : Arguments) { 119 OS << '"'; 120 121 for (; *Arg != '\0'; Arg++) { 122 if (*Arg == '\"' || *Arg == '\\') { 123 OS << '\\'; 124 } 125 OS << *Arg; 126 } 127 128 OS << "\" "; 129 } 130 } 131 132 void Command::buildArgvForResponseFile( 133 llvm::SmallVectorImpl<const char *> &Out) const { 134 // When not a file list, all arguments are sent to the response file. 135 // This leaves us to set the argv to a single parameter, requesting the tool 136 // to read the response file. 137 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) { 138 Out.push_back(Executable); 139 Out.push_back(ResponseFileFlag.c_str()); 140 return; 141 } 142 143 llvm::StringSet<> Inputs; 144 for (const auto *InputName : InputFileList) 145 Inputs.insert(InputName); 146 Out.push_back(Executable); 147 // In a file list, build args vector ignoring parameters that will go in the 148 // response file (elements of the InputFileList vector) 149 bool FirstInput = true; 150 for (const auto *Arg : Arguments) { 151 if (Inputs.count(Arg) == 0) { 152 Out.push_back(Arg); 153 } else if (FirstInput) { 154 FirstInput = false; 155 Out.push_back(ResponseSupport.ResponseFlag); 156 Out.push_back(ResponseFile); 157 } 158 } 159 } 160 161 /// Rewrite relative include-like flag paths to absolute ones. 162 static void 163 rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx, 164 size_t NumArgs, 165 llvm::SmallVectorImpl<llvm::SmallString<128>> &IncFlags) { 166 using namespace llvm; 167 using namespace sys; 168 169 auto getAbsPath = [](StringRef InInc, SmallVectorImpl<char> &OutInc) -> bool { 170 if (path::is_absolute(InInc)) // Nothing to do here... 171 return false; 172 std::error_code EC = fs::current_path(OutInc); 173 if (EC) 174 return false; 175 path::append(OutInc, InInc); 176 return true; 177 }; 178 179 SmallString<128> NewInc; 180 if (NumArgs == 1) { 181 StringRef FlagRef(Args[Idx + NumArgs - 1]); 182 assert((FlagRef.startswith("-F") || FlagRef.startswith("-I")) && 183 "Expecting -I or -F"); 184 StringRef Inc = FlagRef.slice(2, StringRef::npos); 185 if (getAbsPath(Inc, NewInc)) { 186 SmallString<128> NewArg(FlagRef.slice(0, 2)); 187 NewArg += NewInc; 188 IncFlags.push_back(std::move(NewArg)); 189 } 190 return; 191 } 192 193 assert(NumArgs == 2 && "Not expecting more than two arguments"); 194 StringRef Inc(Args[Idx + NumArgs - 1]); 195 if (!getAbsPath(Inc, NewInc)) 196 return; 197 IncFlags.push_back(SmallString<128>(Args[Idx])); 198 IncFlags.push_back(std::move(NewInc)); 199 } 200 201 void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, 202 CrashReportInfo *CrashInfo) const { 203 // Always quote the exe. 204 OS << ' '; 205 llvm::sys::printArg(OS, Executable, /*Quote=*/true); 206 207 ArrayRef<const char *> Args = Arguments; 208 SmallVector<const char *, 128> ArgsRespFile; 209 if (ResponseFile != nullptr) { 210 buildArgvForResponseFile(ArgsRespFile); 211 Args = ArrayRef<const char *>(ArgsRespFile).slice(1); // no executable name 212 } 213 214 bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty(); 215 for (size_t i = 0, e = Args.size(); i < e; ++i) { 216 const char *const Arg = Args[i]; 217 218 if (CrashInfo) { 219 int NumArgs = 0; 220 bool IsInclude = false; 221 if (skipArgs(Arg, HaveCrashVFS, NumArgs, IsInclude)) { 222 i += NumArgs - 1; 223 continue; 224 } 225 226 // Relative includes need to be expanded to absolute paths. 227 if (HaveCrashVFS && IsInclude) { 228 SmallVector<SmallString<128>, 2> NewIncFlags; 229 rewriteIncludes(Args, i, NumArgs, NewIncFlags); 230 if (!NewIncFlags.empty()) { 231 for (auto &F : NewIncFlags) { 232 OS << ' '; 233 llvm::sys::printArg(OS, F.c_str(), Quote); 234 } 235 i += NumArgs - 1; 236 continue; 237 } 238 } 239 240 auto Found = llvm::find_if(InputFilenames, 241 [&Arg](StringRef IF) { return IF == Arg; }); 242 if (Found != InputFilenames.end() && 243 (i == 0 || StringRef(Args[i - 1]) != "-main-file-name")) { 244 // Replace the input file name with the crashinfo's file name. 245 OS << ' '; 246 StringRef ShortName = llvm::sys::path::filename(CrashInfo->Filename); 247 llvm::sys::printArg(OS, ShortName.str(), Quote); 248 continue; 249 } 250 } 251 252 OS << ' '; 253 llvm::sys::printArg(OS, Arg, Quote); 254 } 255 256 if (CrashInfo && HaveCrashVFS) { 257 OS << ' '; 258 llvm::sys::printArg(OS, "-ivfsoverlay", Quote); 259 OS << ' '; 260 llvm::sys::printArg(OS, CrashInfo->VFSPath.str(), Quote); 261 262 // The leftover modules from the crash are stored in 263 // <name>.cache/vfs/modules 264 // Leave it untouched for pcm inspection and provide a clean/empty dir 265 // path to contain the future generated module cache: 266 // <name>.cache/vfs/repro-modules 267 SmallString<128> RelModCacheDir = llvm::sys::path::parent_path( 268 llvm::sys::path::parent_path(CrashInfo->VFSPath)); 269 llvm::sys::path::append(RelModCacheDir, "repro-modules"); 270 271 std::string ModCachePath = "-fmodules-cache-path="; 272 ModCachePath.append(RelModCacheDir.c_str()); 273 274 OS << ' '; 275 llvm::sys::printArg(OS, ModCachePath, Quote); 276 } 277 278 if (ResponseFile != nullptr) { 279 OS << "\n Arguments passed via response file:\n"; 280 writeResponseFile(OS); 281 // Avoiding duplicated newline terminator, since FileLists are 282 // newline-separated. 283 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) 284 OS << "\n"; 285 OS << " (end of response file)"; 286 } 287 288 OS << Terminator; 289 } 290 291 void Command::setResponseFile(const char *FileName) { 292 ResponseFile = FileName; 293 ResponseFileFlag = ResponseSupport.ResponseFlag; 294 ResponseFileFlag += FileName; 295 } 296 297 void Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { 298 Environment.reserve(NewEnvironment.size() + 1); 299 Environment.assign(NewEnvironment.begin(), NewEnvironment.end()); 300 Environment.push_back(nullptr); 301 } 302 303 void Command::PrintFileNames() const { 304 if (PrintInputFilenames) { 305 for (const char *Arg : InputFilenames) 306 llvm::outs() << llvm::sys::path::filename(Arg) << "\n"; 307 llvm::outs().flush(); 308 } 309 } 310 311 int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 312 std::string *ErrMsg, bool *ExecutionFailed) const { 313 PrintFileNames(); 314 315 SmallVector<const char *, 128> Argv; 316 if (ResponseFile == nullptr) { 317 Argv.push_back(Executable); 318 Argv.append(Arguments.begin(), Arguments.end()); 319 Argv.push_back(nullptr); 320 } else { 321 // If the command is too large, we need to put arguments in a response file. 322 std::string RespContents; 323 llvm::raw_string_ostream SS(RespContents); 324 325 // Write file contents and build the Argv vector 326 writeResponseFile(SS); 327 buildArgvForResponseFile(Argv); 328 Argv.push_back(nullptr); 329 SS.flush(); 330 331 // Save the response file in the appropriate encoding 332 if (std::error_code EC = writeFileWithEncoding( 333 ResponseFile, RespContents, ResponseSupport.ResponseEncoding)) { 334 if (ErrMsg) 335 *ErrMsg = EC.message(); 336 if (ExecutionFailed) 337 *ExecutionFailed = true; 338 // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to 339 // indicate the requested executable cannot be started. 340 return -1; 341 } 342 } 343 344 Optional<ArrayRef<StringRef>> Env; 345 std::vector<StringRef> ArgvVectorStorage; 346 if (!Environment.empty()) { 347 assert(Environment.back() == nullptr && 348 "Environment vector should be null-terminated by now"); 349 ArgvVectorStorage = llvm::toStringRefArray(Environment.data()); 350 Env = makeArrayRef(ArgvVectorStorage); 351 } 352 353 auto Args = llvm::toStringRefArray(Argv.data()); 354 return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects, 355 /*secondsToWait*/ 0, /*memoryLimit*/ 0, 356 ErrMsg, ExecutionFailed, &ProcStat); 357 } 358 359 CC1Command::CC1Command(const Action &Source, const Tool &Creator, 360 ResponseFileSupport ResponseSupport, 361 const char *Executable, 362 const llvm::opt::ArgStringList &Arguments, 363 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs) 364 : Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs, 365 Outputs) { 366 InProcess = true; 367 } 368 369 void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, 370 CrashReportInfo *CrashInfo) const { 371 if (InProcess) 372 OS << " (in-process)\n"; 373 Command::Print(OS, Terminator, Quote, CrashInfo); 374 } 375 376 int CC1Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 377 std::string *ErrMsg, bool *ExecutionFailed) const { 378 // FIXME: Currently, if there're more than one job, we disable 379 // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to 380 // out-of-process execution. See discussion in https://reviews.llvm.org/D74447 381 if (!InProcess) 382 return Command::Execute(Redirects, ErrMsg, ExecutionFailed); 383 384 PrintFileNames(); 385 386 SmallVector<const char *, 128> Argv; 387 Argv.push_back(getExecutable()); 388 Argv.append(getArguments().begin(), getArguments().end()); 389 Argv.push_back(nullptr); 390 391 // This flag simply indicates that the program couldn't start, which isn't 392 // applicable here. 393 if (ExecutionFailed) 394 *ExecutionFailed = false; 395 396 llvm::CrashRecoveryContext CRC; 397 CRC.DumpStackAndCleanupOnFailure = true; 398 399 const void *PrettyState = llvm::SavePrettyStackState(); 400 const Driver &D = getCreator().getToolChain().getDriver(); 401 402 int R = 0; 403 // Enter ExecuteCC1Tool() instead of starting up a new process 404 if (!CRC.RunSafely([&]() { R = D.CC1Main(Argv); })) { 405 llvm::RestorePrettyStackState(PrettyState); 406 return CRC.RetCode; 407 } 408 return R; 409 } 410 411 void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { 412 // We don't support set a new environment when calling into ExecuteCC1Tool() 413 llvm_unreachable( 414 "The CC1Command doesn't support changing the environment vars!"); 415 } 416 417 FallbackCommand::FallbackCommand(const Action &Source_, const Tool &Creator_, 418 ResponseFileSupport ResponseSupport, 419 const char *Executable_, 420 const llvm::opt::ArgStringList &Arguments_, 421 ArrayRef<InputInfo> Inputs, 422 ArrayRef<InputInfo> Outputs, 423 std::unique_ptr<Command> Fallback_) 424 : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_, 425 Inputs, Outputs), 426 Fallback(std::move(Fallback_)) {} 427 428 void FallbackCommand::Print(raw_ostream &OS, const char *Terminator, 429 bool Quote, CrashReportInfo *CrashInfo) const { 430 Command::Print(OS, "", Quote, CrashInfo); 431 OS << " ||"; 432 Fallback->Print(OS, Terminator, Quote, CrashInfo); 433 } 434 435 static bool ShouldFallback(int ExitCode) { 436 // FIXME: We really just want to fall back for internal errors, such 437 // as when some symbol cannot be mangled, when we should be able to 438 // parse something but can't, etc. 439 return ExitCode != 0; 440 } 441 442 int FallbackCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 443 std::string *ErrMsg, bool *ExecutionFailed) const { 444 int PrimaryStatus = Command::Execute(Redirects, ErrMsg, ExecutionFailed); 445 if (!ShouldFallback(PrimaryStatus)) 446 return PrimaryStatus; 447 448 // Clear ExecutionFailed and ErrMsg before falling back. 449 if (ErrMsg) 450 ErrMsg->clear(); 451 if (ExecutionFailed) 452 *ExecutionFailed = false; 453 454 const Driver &D = getCreator().getToolChain().getDriver(); 455 D.Diag(diag::warn_drv_invoking_fallback) << Fallback->getExecutable(); 456 457 int SecondaryStatus = Fallback->Execute(Redirects, ErrMsg, ExecutionFailed); 458 return SecondaryStatus; 459 } 460 461 ForceSuccessCommand::ForceSuccessCommand( 462 const Action &Source_, const Tool &Creator_, 463 ResponseFileSupport ResponseSupport, const char *Executable_, 464 const llvm::opt::ArgStringList &Arguments_, ArrayRef<InputInfo> Inputs, 465 ArrayRef<InputInfo> Outputs) 466 : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_, 467 Inputs, Outputs) {} 468 469 void ForceSuccessCommand::Print(raw_ostream &OS, const char *Terminator, 470 bool Quote, CrashReportInfo *CrashInfo) const { 471 Command::Print(OS, "", Quote, CrashInfo); 472 OS << " || (exit 0)" << Terminator; 473 } 474 475 int ForceSuccessCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 476 std::string *ErrMsg, 477 bool *ExecutionFailed) const { 478 int Status = Command::Execute(Redirects, ErrMsg, ExecutionFailed); 479 (void)Status; 480 if (ExecutionFailed) 481 *ExecutionFailed = false; 482 return 0; 483 } 484 485 void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote, 486 CrashReportInfo *CrashInfo) const { 487 for (const auto &Job : *this) 488 Job.Print(OS, Terminator, Quote, CrashInfo); 489 } 490 491 void JobList::clear() { Jobs.clear(); } 492