1 //===- Job.cpp - Command to Execute ---------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Driver/Job.h" 10 #include "InputInfo.h" 11 #include "clang/Basic/LLVM.h" 12 #include "clang/Driver/Driver.h" 13 #include "clang/Driver/DriverDiagnostic.h" 14 #include "clang/Driver/Tool.h" 15 #include "clang/Driver/ToolChain.h" 16 #include "llvm/ADT/ArrayRef.h" 17 #include "llvm/ADT/SmallString.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/ADT/StringRef.h" 20 #include "llvm/ADT/StringSet.h" 21 #include "llvm/ADT/StringSwitch.h" 22 #include "llvm/Support/CrashRecoveryContext.h" 23 #include "llvm/Support/FileSystem.h" 24 #include "llvm/Support/Path.h" 25 #include "llvm/Support/PrettyStackTrace.h" 26 #include "llvm/Support/Program.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include <algorithm> 29 #include <cassert> 30 #include <cstddef> 31 #include <string> 32 #include <system_error> 33 #include <utility> 34 35 using namespace clang; 36 using namespace driver; 37 38 Command::Command(const Action &Source, const Tool &Creator, 39 ResponseFileSupport ResponseSupport, const char *Executable, 40 const llvm::opt::ArgStringList &Arguments, 41 ArrayRef<InputInfo> Inputs) 42 : Source(Source), Creator(Creator), ResponseSupport(ResponseSupport), 43 Executable(Executable), Arguments(Arguments) { 44 for (const auto &II : Inputs) 45 if (II.isFilename()) 46 InputFilenames.push_back(II.getFilename()); 47 } 48 49 /// Check if the compiler flag in question should be skipped when 50 /// emitting a reproducer. Also track how many arguments it has and if the 51 /// option is some kind of include path. 52 static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum, 53 bool &IsInclude) { 54 SkipNum = 2; 55 // These flags are all of the form -Flag <Arg> and are treated as two 56 // arguments. Therefore, we need to skip the flag and the next argument. 57 bool ShouldSkip = llvm::StringSwitch<bool>(Flag) 58 .Cases("-MF", "-MT", "-MQ", "-serialize-diagnostic-file", true) 59 .Cases("-o", "-dependency-file", true) 60 .Cases("-fdebug-compilation-dir", "-diagnostic-log-file", true) 61 .Cases("-dwarf-debug-flags", "-ivfsoverlay", true) 62 .Default(false); 63 if (ShouldSkip) 64 return true; 65 66 // Some include flags shouldn't be skipped if we have a crash VFS 67 IsInclude = llvm::StringSwitch<bool>(Flag) 68 .Cases("-include", "-header-include-file", true) 69 .Cases("-idirafter", "-internal-isystem", "-iwithprefix", true) 70 .Cases("-internal-externc-isystem", "-iprefix", true) 71 .Cases("-iwithprefixbefore", "-isystem", "-iquote", true) 72 .Cases("-isysroot", "-I", "-F", "-resource-dir", true) 73 .Cases("-iframework", "-include-pch", true) 74 .Default(false); 75 if (IsInclude) 76 return !HaveCrashVFS; 77 78 // The remaining flags are treated as a single argument. 79 80 // These flags are all of the form -Flag and have no second argument. 81 ShouldSkip = llvm::StringSwitch<bool>(Flag) 82 .Cases("-M", "-MM", "-MG", "-MP", "-MD", true) 83 .Case("-MMD", true) 84 .Default(false); 85 86 // Match found. 87 SkipNum = 1; 88 if (ShouldSkip) 89 return true; 90 91 // These flags are treated as a single argument (e.g., -F<Dir>). 92 StringRef FlagRef(Flag); 93 IsInclude = FlagRef.startswith("-F") || FlagRef.startswith("-I"); 94 if (IsInclude) 95 return !HaveCrashVFS; 96 if (FlagRef.startswith("-fmodules-cache-path=")) 97 return true; 98 99 SkipNum = 0; 100 return false; 101 } 102 103 void Command::writeResponseFile(raw_ostream &OS) const { 104 // In a file list, we only write the set of inputs to the response file 105 if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) { 106 for (const auto *Arg : InputFileList) { 107 OS << Arg << '\n'; 108 } 109 return; 110 } 111 112 // In regular response files, we send all arguments to the response file. 113 // Wrapping all arguments in double quotes ensures that both Unix tools and 114 // Windows tools understand the response file. 115 for (const auto *Arg : Arguments) { 116 OS << '"'; 117 118 for (; *Arg != '\0'; Arg++) { 119 if (*Arg == '\"' || *Arg == '\\') { 120 OS << '\\'; 121 } 122 OS << *Arg; 123 } 124 125 OS << "\" "; 126 } 127 } 128 129 void Command::buildArgvForResponseFile( 130 llvm::SmallVectorImpl<const char *> &Out) const { 131 // When not a file list, all arguments are sent to the response file. 132 // This leaves us to set the argv to a single parameter, requesting the tool 133 // to read the response file. 134 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) { 135 Out.push_back(Executable); 136 Out.push_back(ResponseFileFlag.c_str()); 137 return; 138 } 139 140 llvm::StringSet<> Inputs; 141 for (const auto *InputName : InputFileList) 142 Inputs.insert(InputName); 143 Out.push_back(Executable); 144 // In a file list, build args vector ignoring parameters that will go in the 145 // response file (elements of the InputFileList vector) 146 bool FirstInput = true; 147 for (const auto *Arg : Arguments) { 148 if (Inputs.count(Arg) == 0) { 149 Out.push_back(Arg); 150 } else if (FirstInput) { 151 FirstInput = false; 152 Out.push_back(ResponseSupport.ResponseFlag); 153 Out.push_back(ResponseFile); 154 } 155 } 156 } 157 158 /// Rewrite relative include-like flag paths to absolute ones. 159 static void 160 rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx, 161 size_t NumArgs, 162 llvm::SmallVectorImpl<llvm::SmallString<128>> &IncFlags) { 163 using namespace llvm; 164 using namespace sys; 165 166 auto getAbsPath = [](StringRef InInc, SmallVectorImpl<char> &OutInc) -> bool { 167 if (path::is_absolute(InInc)) // Nothing to do here... 168 return false; 169 std::error_code EC = fs::current_path(OutInc); 170 if (EC) 171 return false; 172 path::append(OutInc, InInc); 173 return true; 174 }; 175 176 SmallString<128> NewInc; 177 if (NumArgs == 1) { 178 StringRef FlagRef(Args[Idx + NumArgs - 1]); 179 assert((FlagRef.startswith("-F") || FlagRef.startswith("-I")) && 180 "Expecting -I or -F"); 181 StringRef Inc = FlagRef.slice(2, StringRef::npos); 182 if (getAbsPath(Inc, NewInc)) { 183 SmallString<128> NewArg(FlagRef.slice(0, 2)); 184 NewArg += NewInc; 185 IncFlags.push_back(std::move(NewArg)); 186 } 187 return; 188 } 189 190 assert(NumArgs == 2 && "Not expecting more than two arguments"); 191 StringRef Inc(Args[Idx + NumArgs - 1]); 192 if (!getAbsPath(Inc, NewInc)) 193 return; 194 IncFlags.push_back(SmallString<128>(Args[Idx])); 195 IncFlags.push_back(std::move(NewInc)); 196 } 197 198 void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, 199 CrashReportInfo *CrashInfo) const { 200 // Always quote the exe. 201 OS << ' '; 202 llvm::sys::printArg(OS, Executable, /*Quote=*/true); 203 204 ArrayRef<const char *> Args = Arguments; 205 SmallVector<const char *, 128> ArgsRespFile; 206 if (ResponseFile != nullptr) { 207 buildArgvForResponseFile(ArgsRespFile); 208 Args = ArrayRef<const char *>(ArgsRespFile).slice(1); // no executable name 209 } 210 211 bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty(); 212 for (size_t i = 0, e = Args.size(); i < e; ++i) { 213 const char *const Arg = Args[i]; 214 215 if (CrashInfo) { 216 int NumArgs = 0; 217 bool IsInclude = false; 218 if (skipArgs(Arg, HaveCrashVFS, NumArgs, IsInclude)) { 219 i += NumArgs - 1; 220 continue; 221 } 222 223 // Relative includes need to be expanded to absolute paths. 224 if (HaveCrashVFS && IsInclude) { 225 SmallVector<SmallString<128>, 2> NewIncFlags; 226 rewriteIncludes(Args, i, NumArgs, NewIncFlags); 227 if (!NewIncFlags.empty()) { 228 for (auto &F : NewIncFlags) { 229 OS << ' '; 230 llvm::sys::printArg(OS, F.c_str(), Quote); 231 } 232 i += NumArgs - 1; 233 continue; 234 } 235 } 236 237 auto Found = llvm::find_if(InputFilenames, 238 [&Arg](StringRef IF) { return IF == Arg; }); 239 if (Found != InputFilenames.end() && 240 (i == 0 || StringRef(Args[i - 1]) != "-main-file-name")) { 241 // Replace the input file name with the crashinfo's file name. 242 OS << ' '; 243 StringRef ShortName = llvm::sys::path::filename(CrashInfo->Filename); 244 llvm::sys::printArg(OS, ShortName.str(), Quote); 245 continue; 246 } 247 } 248 249 OS << ' '; 250 llvm::sys::printArg(OS, Arg, Quote); 251 } 252 253 if (CrashInfo && HaveCrashVFS) { 254 OS << ' '; 255 llvm::sys::printArg(OS, "-ivfsoverlay", Quote); 256 OS << ' '; 257 llvm::sys::printArg(OS, CrashInfo->VFSPath.str(), Quote); 258 259 // The leftover modules from the crash are stored in 260 // <name>.cache/vfs/modules 261 // Leave it untouched for pcm inspection and provide a clean/empty dir 262 // path to contain the future generated module cache: 263 // <name>.cache/vfs/repro-modules 264 SmallString<128> RelModCacheDir = llvm::sys::path::parent_path( 265 llvm::sys::path::parent_path(CrashInfo->VFSPath)); 266 llvm::sys::path::append(RelModCacheDir, "repro-modules"); 267 268 std::string ModCachePath = "-fmodules-cache-path="; 269 ModCachePath.append(RelModCacheDir.c_str()); 270 271 OS << ' '; 272 llvm::sys::printArg(OS, ModCachePath, Quote); 273 } 274 275 if (ResponseFile != nullptr) { 276 OS << "\n Arguments passed via response file:\n"; 277 writeResponseFile(OS); 278 // Avoiding duplicated newline terminator, since FileLists are 279 // newline-separated. 280 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) 281 OS << "\n"; 282 OS << " (end of response file)"; 283 } 284 285 OS << Terminator; 286 } 287 288 void Command::setResponseFile(const char *FileName) { 289 ResponseFile = FileName; 290 ResponseFileFlag = ResponseSupport.ResponseFlag; 291 ResponseFileFlag += FileName; 292 } 293 294 void Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { 295 Environment.reserve(NewEnvironment.size() + 1); 296 Environment.assign(NewEnvironment.begin(), NewEnvironment.end()); 297 Environment.push_back(nullptr); 298 } 299 300 void Command::PrintFileNames() const { 301 if (PrintInputFilenames) { 302 for (const char *Arg : InputFilenames) 303 llvm::outs() << llvm::sys::path::filename(Arg) << "\n"; 304 llvm::outs().flush(); 305 } 306 } 307 308 int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 309 std::string *ErrMsg, bool *ExecutionFailed) const { 310 PrintFileNames(); 311 312 SmallVector<const char *, 128> Argv; 313 if (ResponseFile == nullptr) { 314 Argv.push_back(Executable); 315 Argv.append(Arguments.begin(), Arguments.end()); 316 Argv.push_back(nullptr); 317 } else { 318 // If the command is too large, we need to put arguments in a response file. 319 std::string RespContents; 320 llvm::raw_string_ostream SS(RespContents); 321 322 // Write file contents and build the Argv vector 323 writeResponseFile(SS); 324 buildArgvForResponseFile(Argv); 325 Argv.push_back(nullptr); 326 SS.flush(); 327 328 // Save the response file in the appropriate encoding 329 if (std::error_code EC = writeFileWithEncoding( 330 ResponseFile, RespContents, ResponseSupport.ResponseEncoding)) { 331 if (ErrMsg) 332 *ErrMsg = EC.message(); 333 if (ExecutionFailed) 334 *ExecutionFailed = true; 335 // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to 336 // indicate the requested executable cannot be started. 337 return -1; 338 } 339 } 340 341 Optional<ArrayRef<StringRef>> Env; 342 std::vector<StringRef> ArgvVectorStorage; 343 if (!Environment.empty()) { 344 assert(Environment.back() == nullptr && 345 "Environment vector should be null-terminated by now"); 346 ArgvVectorStorage = llvm::toStringRefArray(Environment.data()); 347 Env = makeArrayRef(ArgvVectorStorage); 348 } 349 350 auto Args = llvm::toStringRefArray(Argv.data()); 351 return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects, 352 /*secondsToWait*/ 0, 353 /*memoryLimit*/ 0, ErrMsg, ExecutionFailed); 354 } 355 356 CC1Command::CC1Command(const Action &Source, const Tool &Creator, 357 ResponseFileSupport ResponseSupport, 358 const char *Executable, 359 const llvm::opt::ArgStringList &Arguments, 360 ArrayRef<InputInfo> Inputs) 361 : Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs) { 362 InProcess = true; 363 } 364 365 void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, 366 CrashReportInfo *CrashInfo) const { 367 if (InProcess) 368 OS << " (in-process)\n"; 369 Command::Print(OS, Terminator, Quote, CrashInfo); 370 } 371 372 int CC1Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 373 std::string *ErrMsg, bool *ExecutionFailed) const { 374 // FIXME: Currently, if there're more than one job, we disable 375 // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to 376 // out-of-process execution. See discussion in https://reviews.llvm.org/D74447 377 if (!InProcess) 378 return Command::Execute(Redirects, ErrMsg, ExecutionFailed); 379 380 PrintFileNames(); 381 382 SmallVector<const char *, 128> Argv; 383 Argv.push_back(getExecutable()); 384 Argv.append(getArguments().begin(), getArguments().end()); 385 Argv.push_back(nullptr); 386 387 // This flag simply indicates that the program couldn't start, which isn't 388 // applicable here. 389 if (ExecutionFailed) 390 *ExecutionFailed = false; 391 392 llvm::CrashRecoveryContext CRC; 393 CRC.DumpStackAndCleanupOnFailure = true; 394 395 const void *PrettyState = llvm::SavePrettyStackState(); 396 const Driver &D = getCreator().getToolChain().getDriver(); 397 398 int R = 0; 399 // Enter ExecuteCC1Tool() instead of starting up a new process 400 if (!CRC.RunSafely([&]() { R = D.CC1Main(Argv); })) { 401 llvm::RestorePrettyStackState(PrettyState); 402 return CRC.RetCode; 403 } 404 return R; 405 } 406 407 void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { 408 // We don't support set a new environment when calling into ExecuteCC1Tool() 409 llvm_unreachable( 410 "The CC1Command doesn't support changing the environment vars!"); 411 } 412 413 FallbackCommand::FallbackCommand(const Action &Source_, const Tool &Creator_, 414 ResponseFileSupport ResponseSupport, 415 const char *Executable_, 416 const llvm::opt::ArgStringList &Arguments_, 417 ArrayRef<InputInfo> Inputs, 418 std::unique_ptr<Command> Fallback_) 419 : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_, 420 Inputs), 421 Fallback(std::move(Fallback_)) {} 422 423 void FallbackCommand::Print(raw_ostream &OS, const char *Terminator, 424 bool Quote, CrashReportInfo *CrashInfo) const { 425 Command::Print(OS, "", Quote, CrashInfo); 426 OS << " ||"; 427 Fallback->Print(OS, Terminator, Quote, CrashInfo); 428 } 429 430 static bool ShouldFallback(int ExitCode) { 431 // FIXME: We really just want to fall back for internal errors, such 432 // as when some symbol cannot be mangled, when we should be able to 433 // parse something but can't, etc. 434 return ExitCode != 0; 435 } 436 437 int FallbackCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 438 std::string *ErrMsg, bool *ExecutionFailed) const { 439 int PrimaryStatus = Command::Execute(Redirects, ErrMsg, ExecutionFailed); 440 if (!ShouldFallback(PrimaryStatus)) 441 return PrimaryStatus; 442 443 // Clear ExecutionFailed and ErrMsg before falling back. 444 if (ErrMsg) 445 ErrMsg->clear(); 446 if (ExecutionFailed) 447 *ExecutionFailed = false; 448 449 const Driver &D = getCreator().getToolChain().getDriver(); 450 D.Diag(diag::warn_drv_invoking_fallback) << Fallback->getExecutable(); 451 452 int SecondaryStatus = Fallback->Execute(Redirects, ErrMsg, ExecutionFailed); 453 return SecondaryStatus; 454 } 455 456 ForceSuccessCommand::ForceSuccessCommand( 457 const Action &Source_, const Tool &Creator_, 458 ResponseFileSupport ResponseSupport, const char *Executable_, 459 const llvm::opt::ArgStringList &Arguments_, ArrayRef<InputInfo> Inputs) 460 : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_, 461 Inputs) {} 462 463 void ForceSuccessCommand::Print(raw_ostream &OS, const char *Terminator, 464 bool Quote, CrashReportInfo *CrashInfo) const { 465 Command::Print(OS, "", Quote, CrashInfo); 466 OS << " || (exit 0)" << Terminator; 467 } 468 469 int ForceSuccessCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 470 std::string *ErrMsg, 471 bool *ExecutionFailed) const { 472 int Status = Command::Execute(Redirects, ErrMsg, ExecutionFailed); 473 (void)Status; 474 if (ExecutionFailed) 475 *ExecutionFailed = false; 476 return 0; 477 } 478 479 void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote, 480 CrashReportInfo *CrashInfo) const { 481 for (const auto &Job : *this) 482 Job.Print(OS, Terminator, Quote, CrashInfo); 483 } 484 485 void JobList::clear() { Jobs.clear(); } 486