1 //===- Job.cpp - Command to Execute ---------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Driver/Job.h" 10 #include "clang/Basic/LLVM.h" 11 #include "clang/Driver/Driver.h" 12 #include "clang/Driver/DriverDiagnostic.h" 13 #include "clang/Driver/InputInfo.h" 14 #include "clang/Driver/Tool.h" 15 #include "clang/Driver/ToolChain.h" 16 #include "llvm/ADT/ArrayRef.h" 17 #include "llvm/ADT/SmallString.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/ADT/StringRef.h" 20 #include "llvm/ADT/StringSet.h" 21 #include "llvm/ADT/StringSwitch.h" 22 #include "llvm/Support/CrashRecoveryContext.h" 23 #include "llvm/Support/FileSystem.h" 24 #include "llvm/Support/Path.h" 25 #include "llvm/Support/PrettyStackTrace.h" 26 #include "llvm/Support/Program.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include <algorithm> 29 #include <cassert> 30 #include <cstddef> 31 #include <string> 32 #include <system_error> 33 #include <utility> 34 35 using namespace clang; 36 using namespace driver; 37 38 Command::Command(const Action &Source, const Tool &Creator, 39 ResponseFileSupport ResponseSupport, const char *Executable, 40 const llvm::opt::ArgStringList &Arguments, 41 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs) 42 : Source(Source), Creator(Creator), ResponseSupport(ResponseSupport), 43 Executable(Executable), Arguments(Arguments) { 44 for (const auto &II : Inputs) 45 if (II.isFilename()) 46 InputInfoList.push_back(II); 47 for (const auto &II : Outputs) 48 if (II.isFilename()) 49 OutputFilenames.push_back(II.getFilename()); 50 } 51 52 /// Check if the compiler flag in question should be skipped when 53 /// emitting a reproducer. Also track how many arguments it has and if the 54 /// option is some kind of include path. 55 static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum, 56 bool &IsInclude) { 57 SkipNum = 2; 58 // These flags are all of the form -Flag <Arg> and are treated as two 59 // arguments. Therefore, we need to skip the flag and the next argument. 60 bool ShouldSkip = llvm::StringSwitch<bool>(Flag) 61 .Cases("-MF", "-MT", "-MQ", "-serialize-diagnostic-file", true) 62 .Cases("-o", "-dependency-file", true) 63 .Cases("-fdebug-compilation-dir", "-diagnostic-log-file", true) 64 .Cases("-dwarf-debug-flags", "-ivfsoverlay", true) 65 .Default(false); 66 if (ShouldSkip) 67 return true; 68 69 // Some include flags shouldn't be skipped if we have a crash VFS 70 IsInclude = llvm::StringSwitch<bool>(Flag) 71 .Cases("-include", "-header-include-file", true) 72 .Cases("-idirafter", "-internal-isystem", "-iwithprefix", true) 73 .Cases("-internal-externc-isystem", "-iprefix", true) 74 .Cases("-iwithprefixbefore", "-isystem", "-iquote", true) 75 .Cases("-isysroot", "-I", "-F", "-resource-dir", true) 76 .Cases("-iframework", "-include-pch", true) 77 .Default(false); 78 if (IsInclude) 79 return !HaveCrashVFS; 80 81 // The remaining flags are treated as a single argument. 82 83 // These flags are all of the form -Flag and have no second argument. 84 ShouldSkip = llvm::StringSwitch<bool>(Flag) 85 .Cases("-M", "-MM", "-MG", "-MP", "-MD", true) 86 .Case("-MMD", true) 87 .Default(false); 88 89 // Match found. 90 SkipNum = 1; 91 if (ShouldSkip) 92 return true; 93 94 // These flags are treated as a single argument (e.g., -F<Dir>). 95 StringRef FlagRef(Flag); 96 IsInclude = FlagRef.startswith("-F") || FlagRef.startswith("-I"); 97 if (IsInclude) 98 return !HaveCrashVFS; 99 if (FlagRef.startswith("-fmodules-cache-path=")) 100 return true; 101 102 SkipNum = 0; 103 return false; 104 } 105 106 void Command::writeResponseFile(raw_ostream &OS) const { 107 // In a file list, we only write the set of inputs to the response file 108 if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) { 109 for (const auto *Arg : InputFileList) { 110 OS << Arg << '\n'; 111 } 112 return; 113 } 114 115 // In regular response files, we send all arguments to the response file. 116 // Wrapping all arguments in double quotes ensures that both Unix tools and 117 // Windows tools understand the response file. 118 for (const auto *Arg : Arguments) { 119 OS << '"'; 120 121 for (; *Arg != '\0'; Arg++) { 122 if (*Arg == '\"' || *Arg == '\\') { 123 OS << '\\'; 124 } 125 OS << *Arg; 126 } 127 128 OS << "\" "; 129 } 130 } 131 132 void Command::buildArgvForResponseFile( 133 llvm::SmallVectorImpl<const char *> &Out) const { 134 // When not a file list, all arguments are sent to the response file. 135 // This leaves us to set the argv to a single parameter, requesting the tool 136 // to read the response file. 137 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) { 138 Out.push_back(Executable); 139 Out.push_back(ResponseFileFlag.c_str()); 140 return; 141 } 142 143 llvm::StringSet<> Inputs; 144 for (const auto *InputName : InputFileList) 145 Inputs.insert(InputName); 146 Out.push_back(Executable); 147 // In a file list, build args vector ignoring parameters that will go in the 148 // response file (elements of the InputFileList vector) 149 bool FirstInput = true; 150 for (const auto *Arg : Arguments) { 151 if (Inputs.count(Arg) == 0) { 152 Out.push_back(Arg); 153 } else if (FirstInput) { 154 FirstInput = false; 155 Out.push_back(ResponseSupport.ResponseFlag); 156 Out.push_back(ResponseFile); 157 } 158 } 159 } 160 161 /// Rewrite relative include-like flag paths to absolute ones. 162 static void 163 rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx, 164 size_t NumArgs, 165 llvm::SmallVectorImpl<llvm::SmallString<128>> &IncFlags) { 166 using namespace llvm; 167 using namespace sys; 168 169 auto getAbsPath = [](StringRef InInc, SmallVectorImpl<char> &OutInc) -> bool { 170 if (path::is_absolute(InInc)) // Nothing to do here... 171 return false; 172 std::error_code EC = fs::current_path(OutInc); 173 if (EC) 174 return false; 175 path::append(OutInc, InInc); 176 return true; 177 }; 178 179 SmallString<128> NewInc; 180 if (NumArgs == 1) { 181 StringRef FlagRef(Args[Idx + NumArgs - 1]); 182 assert((FlagRef.startswith("-F") || FlagRef.startswith("-I")) && 183 "Expecting -I or -F"); 184 StringRef Inc = FlagRef.slice(2, StringRef::npos); 185 if (getAbsPath(Inc, NewInc)) { 186 SmallString<128> NewArg(FlagRef.slice(0, 2)); 187 NewArg += NewInc; 188 IncFlags.push_back(std::move(NewArg)); 189 } 190 return; 191 } 192 193 assert(NumArgs == 2 && "Not expecting more than two arguments"); 194 StringRef Inc(Args[Idx + NumArgs - 1]); 195 if (!getAbsPath(Inc, NewInc)) 196 return; 197 IncFlags.push_back(SmallString<128>(Args[Idx])); 198 IncFlags.push_back(std::move(NewInc)); 199 } 200 201 void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, 202 CrashReportInfo *CrashInfo) const { 203 // Always quote the exe. 204 OS << ' '; 205 llvm::sys::printArg(OS, Executable, /*Quote=*/true); 206 207 ArrayRef<const char *> Args = Arguments; 208 SmallVector<const char *, 128> ArgsRespFile; 209 if (ResponseFile != nullptr) { 210 buildArgvForResponseFile(ArgsRespFile); 211 Args = ArrayRef<const char *>(ArgsRespFile).slice(1); // no executable name 212 } 213 214 bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty(); 215 for (size_t i = 0, e = Args.size(); i < e; ++i) { 216 const char *const Arg = Args[i]; 217 218 if (CrashInfo) { 219 int NumArgs = 0; 220 bool IsInclude = false; 221 if (skipArgs(Arg, HaveCrashVFS, NumArgs, IsInclude)) { 222 i += NumArgs - 1; 223 continue; 224 } 225 226 // Relative includes need to be expanded to absolute paths. 227 if (HaveCrashVFS && IsInclude) { 228 SmallVector<SmallString<128>, 2> NewIncFlags; 229 rewriteIncludes(Args, i, NumArgs, NewIncFlags); 230 if (!NewIncFlags.empty()) { 231 for (auto &F : NewIncFlags) { 232 OS << ' '; 233 llvm::sys::printArg(OS, F.c_str(), Quote); 234 } 235 i += NumArgs - 1; 236 continue; 237 } 238 } 239 240 auto Found = llvm::find_if(InputInfoList, [&Arg](const InputInfo &II) { 241 return II.getFilename() == Arg; 242 }); 243 if (Found != InputInfoList.end() && 244 (i == 0 || StringRef(Args[i - 1]) != "-main-file-name")) { 245 // Replace the input file name with the crashinfo's file name. 246 OS << ' '; 247 StringRef ShortName = llvm::sys::path::filename(CrashInfo->Filename); 248 llvm::sys::printArg(OS, ShortName.str(), Quote); 249 continue; 250 } 251 } 252 253 OS << ' '; 254 llvm::sys::printArg(OS, Arg, Quote); 255 } 256 257 if (CrashInfo && HaveCrashVFS) { 258 OS << ' '; 259 llvm::sys::printArg(OS, "-ivfsoverlay", Quote); 260 OS << ' '; 261 llvm::sys::printArg(OS, CrashInfo->VFSPath.str(), Quote); 262 263 // The leftover modules from the crash are stored in 264 // <name>.cache/vfs/modules 265 // Leave it untouched for pcm inspection and provide a clean/empty dir 266 // path to contain the future generated module cache: 267 // <name>.cache/vfs/repro-modules 268 SmallString<128> RelModCacheDir = llvm::sys::path::parent_path( 269 llvm::sys::path::parent_path(CrashInfo->VFSPath)); 270 llvm::sys::path::append(RelModCacheDir, "repro-modules"); 271 272 std::string ModCachePath = "-fmodules-cache-path="; 273 ModCachePath.append(RelModCacheDir.c_str()); 274 275 OS << ' '; 276 llvm::sys::printArg(OS, ModCachePath, Quote); 277 } 278 279 if (ResponseFile != nullptr) { 280 OS << "\n Arguments passed via response file:\n"; 281 writeResponseFile(OS); 282 // Avoiding duplicated newline terminator, since FileLists are 283 // newline-separated. 284 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) 285 OS << "\n"; 286 OS << " (end of response file)"; 287 } 288 289 OS << Terminator; 290 } 291 292 void Command::setResponseFile(const char *FileName) { 293 ResponseFile = FileName; 294 ResponseFileFlag = ResponseSupport.ResponseFlag; 295 ResponseFileFlag += FileName; 296 } 297 298 void Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { 299 Environment.reserve(NewEnvironment.size() + 1); 300 Environment.assign(NewEnvironment.begin(), NewEnvironment.end()); 301 Environment.push_back(nullptr); 302 } 303 304 void Command::PrintFileNames() const { 305 if (PrintInputFilenames) { 306 for (const auto &Arg : InputInfoList) 307 llvm::outs() << llvm::sys::path::filename(Arg.getFilename()) << "\n"; 308 llvm::outs().flush(); 309 } 310 } 311 312 int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 313 std::string *ErrMsg, bool *ExecutionFailed) const { 314 PrintFileNames(); 315 316 SmallVector<const char *, 128> Argv; 317 if (ResponseFile == nullptr) { 318 Argv.push_back(Executable); 319 Argv.append(Arguments.begin(), Arguments.end()); 320 Argv.push_back(nullptr); 321 } else { 322 // If the command is too large, we need to put arguments in a response file. 323 std::string RespContents; 324 llvm::raw_string_ostream SS(RespContents); 325 326 // Write file contents and build the Argv vector 327 writeResponseFile(SS); 328 buildArgvForResponseFile(Argv); 329 Argv.push_back(nullptr); 330 SS.flush(); 331 332 // Save the response file in the appropriate encoding 333 if (std::error_code EC = writeFileWithEncoding( 334 ResponseFile, RespContents, ResponseSupport.ResponseEncoding)) { 335 if (ErrMsg) 336 *ErrMsg = EC.message(); 337 if (ExecutionFailed) 338 *ExecutionFailed = true; 339 // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to 340 // indicate the requested executable cannot be started. 341 return -1; 342 } 343 } 344 345 Optional<ArrayRef<StringRef>> Env; 346 std::vector<StringRef> ArgvVectorStorage; 347 if (!Environment.empty()) { 348 assert(Environment.back() == nullptr && 349 "Environment vector should be null-terminated by now"); 350 ArgvVectorStorage = llvm::toStringRefArray(Environment.data()); 351 Env = makeArrayRef(ArgvVectorStorage); 352 } 353 354 auto Args = llvm::toStringRefArray(Argv.data()); 355 return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects, 356 /*secondsToWait*/ 0, /*memoryLimit*/ 0, 357 ErrMsg, ExecutionFailed, &ProcStat); 358 } 359 360 CC1Command::CC1Command(const Action &Source, const Tool &Creator, 361 ResponseFileSupport ResponseSupport, 362 const char *Executable, 363 const llvm::opt::ArgStringList &Arguments, 364 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs) 365 : Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs, 366 Outputs) { 367 InProcess = true; 368 } 369 370 void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote, 371 CrashReportInfo *CrashInfo) const { 372 if (InProcess) 373 OS << " (in-process)\n"; 374 Command::Print(OS, Terminator, Quote, CrashInfo); 375 } 376 377 int CC1Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 378 std::string *ErrMsg, bool *ExecutionFailed) const { 379 // FIXME: Currently, if there're more than one job, we disable 380 // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to 381 // out-of-process execution. See discussion in https://reviews.llvm.org/D74447 382 if (!InProcess) 383 return Command::Execute(Redirects, ErrMsg, ExecutionFailed); 384 385 PrintFileNames(); 386 387 SmallVector<const char *, 128> Argv; 388 Argv.push_back(getExecutable()); 389 Argv.append(getArguments().begin(), getArguments().end()); 390 Argv.push_back(nullptr); 391 Argv.pop_back(); // The terminating null element shall not be part of the 392 // slice (main() behavior). 393 394 // This flag simply indicates that the program couldn't start, which isn't 395 // applicable here. 396 if (ExecutionFailed) 397 *ExecutionFailed = false; 398 399 llvm::CrashRecoveryContext CRC; 400 CRC.DumpStackAndCleanupOnFailure = true; 401 402 const void *PrettyState = llvm::SavePrettyStackState(); 403 const Driver &D = getCreator().getToolChain().getDriver(); 404 405 int R = 0; 406 // Enter ExecuteCC1Tool() instead of starting up a new process 407 if (!CRC.RunSafely([&]() { R = D.CC1Main(Argv); })) { 408 llvm::RestorePrettyStackState(PrettyState); 409 return CRC.RetCode; 410 } 411 return R; 412 } 413 414 void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) { 415 // We don't support set a new environment when calling into ExecuteCC1Tool() 416 llvm_unreachable( 417 "The CC1Command doesn't support changing the environment vars!"); 418 } 419 420 ForceSuccessCommand::ForceSuccessCommand( 421 const Action &Source_, const Tool &Creator_, 422 ResponseFileSupport ResponseSupport, const char *Executable_, 423 const llvm::opt::ArgStringList &Arguments_, ArrayRef<InputInfo> Inputs, 424 ArrayRef<InputInfo> Outputs) 425 : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_, 426 Inputs, Outputs) {} 427 428 void ForceSuccessCommand::Print(raw_ostream &OS, const char *Terminator, 429 bool Quote, CrashReportInfo *CrashInfo) const { 430 Command::Print(OS, "", Quote, CrashInfo); 431 OS << " || (exit 0)" << Terminator; 432 } 433 434 int ForceSuccessCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects, 435 std::string *ErrMsg, 436 bool *ExecutionFailed) const { 437 int Status = Command::Execute(Redirects, ErrMsg, ExecutionFailed); 438 (void)Status; 439 if (ExecutionFailed) 440 *ExecutionFailed = false; 441 return 0; 442 } 443 444 void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote, 445 CrashReportInfo *CrashInfo) const { 446 for (const auto &Job : *this) 447 Job.Print(OS, Terminator, Quote, CrashInfo); 448 } 449 450 void JobList::clear() { Jobs.clear(); } 451