1 //===--- Cuda.cpp - Cuda Tool and ToolChain Implementations -----*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Cuda.h" 10 #include "CommonArgs.h" 11 #include "clang/Basic/Cuda.h" 12 #include "clang/Config/config.h" 13 #include "clang/Driver/Compilation.h" 14 #include "clang/Driver/Distro.h" 15 #include "clang/Driver/Driver.h" 16 #include "clang/Driver/DriverDiagnostic.h" 17 #include "clang/Driver/InputInfo.h" 18 #include "clang/Driver/Options.h" 19 #include "llvm/ADT/StringExtras.h" 20 #include "llvm/Option/ArgList.h" 21 #include "llvm/Support/FileSystem.h" 22 #include "llvm/Support/FormatAdapters.h" 23 #include "llvm/Support/FormatVariadic.h" 24 #include "llvm/Support/Path.h" 25 #include "llvm/Support/Process.h" 26 #include "llvm/Support/Program.h" 27 #include "llvm/Support/VirtualFileSystem.h" 28 #include "llvm/TargetParser/Host.h" 29 #include "llvm/TargetParser/TargetParser.h" 30 #include <system_error> 31 32 using namespace clang::driver; 33 using namespace clang::driver::toolchains; 34 using namespace clang::driver::tools; 35 using namespace clang; 36 using namespace llvm::opt; 37 38 namespace { 39 40 CudaVersion getCudaVersion(uint32_t raw_version) { 41 if (raw_version < 7050) 42 return CudaVersion::CUDA_70; 43 if (raw_version < 8000) 44 return CudaVersion::CUDA_75; 45 if (raw_version < 9000) 46 return CudaVersion::CUDA_80; 47 if (raw_version < 9010) 48 return CudaVersion::CUDA_90; 49 if (raw_version < 9020) 50 return CudaVersion::CUDA_91; 51 if (raw_version < 10000) 52 return CudaVersion::CUDA_92; 53 if (raw_version < 10010) 54 return CudaVersion::CUDA_100; 55 if (raw_version < 10020) 56 return CudaVersion::CUDA_101; 57 if (raw_version < 11000) 58 return CudaVersion::CUDA_102; 59 if (raw_version < 11010) 60 return CudaVersion::CUDA_110; 61 if (raw_version < 11020) 62 return CudaVersion::CUDA_111; 63 if (raw_version < 11030) 64 return CudaVersion::CUDA_112; 65 if (raw_version < 11040) 66 return CudaVersion::CUDA_113; 67 if (raw_version < 11050) 68 return CudaVersion::CUDA_114; 69 if (raw_version < 11060) 70 return CudaVersion::CUDA_115; 71 if (raw_version < 11070) 72 return CudaVersion::CUDA_116; 73 if (raw_version < 11080) 74 return CudaVersion::CUDA_117; 75 if (raw_version < 11090) 76 return CudaVersion::CUDA_118; 77 if (raw_version < 12010) 78 return CudaVersion::CUDA_120; 79 if (raw_version < 12020) 80 return CudaVersion::CUDA_121; 81 if (raw_version < 12030) 82 return CudaVersion::CUDA_122; 83 if (raw_version < 12040) 84 return CudaVersion::CUDA_123; 85 return CudaVersion::NEW; 86 } 87 88 CudaVersion parseCudaHFile(llvm::StringRef Input) { 89 // Helper lambda which skips the words if the line starts with them or returns 90 // std::nullopt otherwise. 91 auto StartsWithWords = 92 [](llvm::StringRef Line, 93 const SmallVector<StringRef, 3> words) -> std::optional<StringRef> { 94 for (StringRef word : words) { 95 if (!Line.consume_front(word)) 96 return {}; 97 Line = Line.ltrim(); 98 } 99 return Line; 100 }; 101 102 Input = Input.ltrim(); 103 while (!Input.empty()) { 104 if (auto Line = 105 StartsWithWords(Input.ltrim(), {"#", "define", "CUDA_VERSION"})) { 106 uint32_t RawVersion; 107 Line->consumeInteger(10, RawVersion); 108 return getCudaVersion(RawVersion); 109 } 110 // Find next non-empty line. 111 Input = Input.drop_front(Input.find_first_of("\n\r")).ltrim(); 112 } 113 return CudaVersion::UNKNOWN; 114 } 115 } // namespace 116 117 void CudaInstallationDetector::WarnIfUnsupportedVersion() { 118 if (Version > CudaVersion::PARTIALLY_SUPPORTED) { 119 std::string VersionString = CudaVersionToString(Version); 120 if (!VersionString.empty()) 121 VersionString.insert(0, " "); 122 D.Diag(diag::warn_drv_new_cuda_version) 123 << VersionString 124 << (CudaVersion::PARTIALLY_SUPPORTED != CudaVersion::FULLY_SUPPORTED) 125 << CudaVersionToString(CudaVersion::PARTIALLY_SUPPORTED); 126 } else if (Version > CudaVersion::FULLY_SUPPORTED) 127 D.Diag(diag::warn_drv_partially_supported_cuda_version) 128 << CudaVersionToString(Version); 129 } 130 131 CudaInstallationDetector::CudaInstallationDetector( 132 const Driver &D, const llvm::Triple &HostTriple, 133 const llvm::opt::ArgList &Args) 134 : D(D) { 135 struct Candidate { 136 std::string Path; 137 bool StrictChecking; 138 139 Candidate(std::string Path, bool StrictChecking = false) 140 : Path(Path), StrictChecking(StrictChecking) {} 141 }; 142 SmallVector<Candidate, 4> Candidates; 143 144 // In decreasing order so we prefer newer versions to older versions. 145 std::initializer_list<const char *> Versions = {"8.0", "7.5", "7.0"}; 146 auto &FS = D.getVFS(); 147 148 if (Args.hasArg(clang::driver::options::OPT_cuda_path_EQ)) { 149 Candidates.emplace_back( 150 Args.getLastArgValue(clang::driver::options::OPT_cuda_path_EQ).str()); 151 } else if (HostTriple.isOSWindows()) { 152 for (const char *Ver : Versions) 153 Candidates.emplace_back( 154 D.SysRoot + "/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v" + 155 Ver); 156 } else { 157 if (!Args.hasArg(clang::driver::options::OPT_cuda_path_ignore_env)) { 158 // Try to find ptxas binary. If the executable is located in a directory 159 // called 'bin/', its parent directory might be a good guess for a valid 160 // CUDA installation. 161 // However, some distributions might installs 'ptxas' to /usr/bin. In that 162 // case the candidate would be '/usr' which passes the following checks 163 // because '/usr/include' exists as well. To avoid this case, we always 164 // check for the directory potentially containing files for libdevice, 165 // even if the user passes -nocudalib. 166 if (llvm::ErrorOr<std::string> ptxas = 167 llvm::sys::findProgramByName("ptxas")) { 168 SmallString<256> ptxasAbsolutePath; 169 llvm::sys::fs::real_path(*ptxas, ptxasAbsolutePath); 170 171 StringRef ptxasDir = llvm::sys::path::parent_path(ptxasAbsolutePath); 172 if (llvm::sys::path::filename(ptxasDir) == "bin") 173 Candidates.emplace_back( 174 std::string(llvm::sys::path::parent_path(ptxasDir)), 175 /*StrictChecking=*/true); 176 } 177 } 178 179 Candidates.emplace_back(D.SysRoot + "/usr/local/cuda"); 180 for (const char *Ver : Versions) 181 Candidates.emplace_back(D.SysRoot + "/usr/local/cuda-" + Ver); 182 183 Distro Dist(FS, llvm::Triple(llvm::sys::getProcessTriple())); 184 if (Dist.IsDebian() || Dist.IsUbuntu()) 185 // Special case for Debian to have nvidia-cuda-toolkit work 186 // out of the box. More info on http://bugs.debian.org/882505 187 Candidates.emplace_back(D.SysRoot + "/usr/lib/cuda"); 188 } 189 190 bool NoCudaLib = Args.hasArg(options::OPT_nogpulib); 191 192 for (const auto &Candidate : Candidates) { 193 InstallPath = Candidate.Path; 194 if (InstallPath.empty() || !FS.exists(InstallPath)) 195 continue; 196 197 BinPath = InstallPath + "/bin"; 198 IncludePath = InstallPath + "/include"; 199 LibDevicePath = InstallPath + "/nvvm/libdevice"; 200 201 if (!(FS.exists(IncludePath) && FS.exists(BinPath))) 202 continue; 203 bool CheckLibDevice = (!NoCudaLib || Candidate.StrictChecking); 204 if (CheckLibDevice && !FS.exists(LibDevicePath)) 205 continue; 206 207 Version = CudaVersion::UNKNOWN; 208 if (auto CudaHFile = FS.getBufferForFile(InstallPath + "/include/cuda.h")) 209 Version = parseCudaHFile((*CudaHFile)->getBuffer()); 210 // As the last resort, make an educated guess between CUDA-7.0, which had 211 // old-style libdevice bitcode, and an unknown recent CUDA version. 212 if (Version == CudaVersion::UNKNOWN) { 213 Version = FS.exists(LibDevicePath + "/libdevice.10.bc") 214 ? CudaVersion::NEW 215 : CudaVersion::CUDA_70; 216 } 217 218 if (Version >= CudaVersion::CUDA_90) { 219 // CUDA-9+ uses single libdevice file for all GPU variants. 220 std::string FilePath = LibDevicePath + "/libdevice.10.bc"; 221 if (FS.exists(FilePath)) { 222 for (int Arch = (int)CudaArch::SM_30, E = (int)CudaArch::LAST; Arch < E; 223 ++Arch) { 224 CudaArch GpuArch = static_cast<CudaArch>(Arch); 225 if (!IsNVIDIAGpuArch(GpuArch)) 226 continue; 227 std::string GpuArchName(CudaArchToString(GpuArch)); 228 LibDeviceMap[GpuArchName] = FilePath; 229 } 230 } 231 } else { 232 std::error_code EC; 233 for (llvm::vfs::directory_iterator LI = FS.dir_begin(LibDevicePath, EC), 234 LE; 235 !EC && LI != LE; LI = LI.increment(EC)) { 236 StringRef FilePath = LI->path(); 237 StringRef FileName = llvm::sys::path::filename(FilePath); 238 // Process all bitcode filenames that look like 239 // libdevice.compute_XX.YY.bc 240 const StringRef LibDeviceName = "libdevice."; 241 if (!(FileName.starts_with(LibDeviceName) && FileName.ends_with(".bc"))) 242 continue; 243 StringRef GpuArch = FileName.slice( 244 LibDeviceName.size(), FileName.find('.', LibDeviceName.size())); 245 LibDeviceMap[GpuArch] = FilePath.str(); 246 // Insert map entries for specific devices with this compute 247 // capability. NVCC's choice of the libdevice library version is 248 // rather peculiar and depends on the CUDA version. 249 if (GpuArch == "compute_20") { 250 LibDeviceMap["sm_20"] = std::string(FilePath); 251 LibDeviceMap["sm_21"] = std::string(FilePath); 252 LibDeviceMap["sm_32"] = std::string(FilePath); 253 } else if (GpuArch == "compute_30") { 254 LibDeviceMap["sm_30"] = std::string(FilePath); 255 if (Version < CudaVersion::CUDA_80) { 256 LibDeviceMap["sm_50"] = std::string(FilePath); 257 LibDeviceMap["sm_52"] = std::string(FilePath); 258 LibDeviceMap["sm_53"] = std::string(FilePath); 259 } 260 LibDeviceMap["sm_60"] = std::string(FilePath); 261 LibDeviceMap["sm_61"] = std::string(FilePath); 262 LibDeviceMap["sm_62"] = std::string(FilePath); 263 } else if (GpuArch == "compute_35") { 264 LibDeviceMap["sm_35"] = std::string(FilePath); 265 LibDeviceMap["sm_37"] = std::string(FilePath); 266 } else if (GpuArch == "compute_50") { 267 if (Version >= CudaVersion::CUDA_80) { 268 LibDeviceMap["sm_50"] = std::string(FilePath); 269 LibDeviceMap["sm_52"] = std::string(FilePath); 270 LibDeviceMap["sm_53"] = std::string(FilePath); 271 } 272 } 273 } 274 } 275 276 // Check that we have found at least one libdevice that we can link in if 277 // -nocudalib hasn't been specified. 278 if (LibDeviceMap.empty() && !NoCudaLib) 279 continue; 280 281 IsValid = true; 282 break; 283 } 284 } 285 286 void CudaInstallationDetector::AddCudaIncludeArgs( 287 const ArgList &DriverArgs, ArgStringList &CC1Args) const { 288 if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) { 289 // Add cuda_wrappers/* to our system include path. This lets us wrap 290 // standard library headers. 291 SmallString<128> P(D.ResourceDir); 292 llvm::sys::path::append(P, "include"); 293 llvm::sys::path::append(P, "cuda_wrappers"); 294 CC1Args.push_back("-internal-isystem"); 295 CC1Args.push_back(DriverArgs.MakeArgString(P)); 296 } 297 298 if (DriverArgs.hasArg(options::OPT_nogpuinc)) 299 return; 300 301 if (!isValid()) { 302 D.Diag(diag::err_drv_no_cuda_installation); 303 return; 304 } 305 306 CC1Args.push_back("-include"); 307 CC1Args.push_back("__clang_cuda_runtime_wrapper.h"); 308 } 309 310 void CudaInstallationDetector::CheckCudaVersionSupportsArch( 311 CudaArch Arch) const { 312 if (Arch == CudaArch::UNKNOWN || Version == CudaVersion::UNKNOWN || 313 ArchsWithBadVersion[(int)Arch]) 314 return; 315 316 auto MinVersion = MinVersionForCudaArch(Arch); 317 auto MaxVersion = MaxVersionForCudaArch(Arch); 318 if (Version < MinVersion || Version > MaxVersion) { 319 ArchsWithBadVersion[(int)Arch] = true; 320 D.Diag(diag::err_drv_cuda_version_unsupported) 321 << CudaArchToString(Arch) << CudaVersionToString(MinVersion) 322 << CudaVersionToString(MaxVersion) << InstallPath 323 << CudaVersionToString(Version); 324 } 325 } 326 327 void CudaInstallationDetector::print(raw_ostream &OS) const { 328 if (isValid()) 329 OS << "Found CUDA installation: " << InstallPath << ", version " 330 << CudaVersionToString(Version) << "\n"; 331 } 332 333 namespace { 334 /// Debug info level for the NVPTX devices. We may need to emit different debug 335 /// info level for the host and for the device itselfi. This type controls 336 /// emission of the debug info for the devices. It either prohibits disable info 337 /// emission completely, or emits debug directives only, or emits same debug 338 /// info as for the host. 339 enum DeviceDebugInfoLevel { 340 DisableDebugInfo, /// Do not emit debug info for the devices. 341 DebugDirectivesOnly, /// Emit only debug directives. 342 EmitSameDebugInfoAsHost, /// Use the same debug info level just like for the 343 /// host. 344 }; 345 } // anonymous namespace 346 347 /// Define debug info level for the NVPTX devices. If the debug info for both 348 /// the host and device are disabled (-g0/-ggdb0 or no debug options at all). If 349 /// only debug directives are requested for the both host and device 350 /// (-gline-directvies-only), or the debug info only for the device is disabled 351 /// (optimization is on and --cuda-noopt-device-debug was not specified), the 352 /// debug directves only must be emitted for the device. Otherwise, use the same 353 /// debug info level just like for the host (with the limitations of only 354 /// supported DWARF2 standard). 355 static DeviceDebugInfoLevel mustEmitDebugInfo(const ArgList &Args) { 356 const Arg *A = Args.getLastArg(options::OPT_O_Group); 357 bool IsDebugEnabled = !A || A->getOption().matches(options::OPT_O0) || 358 Args.hasFlag(options::OPT_cuda_noopt_device_debug, 359 options::OPT_no_cuda_noopt_device_debug, 360 /*Default=*/false); 361 if (const Arg *A = Args.getLastArg(options::OPT_g_Group)) { 362 const Option &Opt = A->getOption(); 363 if (Opt.matches(options::OPT_gN_Group)) { 364 if (Opt.matches(options::OPT_g0) || Opt.matches(options::OPT_ggdb0)) 365 return DisableDebugInfo; 366 if (Opt.matches(options::OPT_gline_directives_only)) 367 return DebugDirectivesOnly; 368 } 369 return IsDebugEnabled ? EmitSameDebugInfoAsHost : DebugDirectivesOnly; 370 } 371 return willEmitRemarks(Args) ? DebugDirectivesOnly : DisableDebugInfo; 372 } 373 374 void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA, 375 const InputInfo &Output, 376 const InputInfoList &Inputs, 377 const ArgList &Args, 378 const char *LinkingOutput) const { 379 const auto &TC = 380 static_cast<const toolchains::NVPTXToolChain &>(getToolChain()); 381 assert(TC.getTriple().isNVPTX() && "Wrong platform"); 382 383 StringRef GPUArchName; 384 // If this is a CUDA action we need to extract the device architecture 385 // from the Job's associated architecture, otherwise use the -march=arch 386 // option. This option may come from -Xopenmp-target flag or the default 387 // value. 388 if (JA.isDeviceOffloading(Action::OFK_Cuda)) { 389 GPUArchName = JA.getOffloadingArch(); 390 } else { 391 GPUArchName = Args.getLastArgValue(options::OPT_march_EQ); 392 assert(!GPUArchName.empty() && "Must have an architecture passed in."); 393 } 394 395 // Obtain architecture from the action. 396 CudaArch gpu_arch = StringToCudaArch(GPUArchName); 397 assert(gpu_arch != CudaArch::UNKNOWN && 398 "Device action expected to have an architecture."); 399 400 // Check that our installation's ptxas supports gpu_arch. 401 if (!Args.hasArg(options::OPT_no_cuda_version_check)) { 402 TC.CudaInstallation.CheckCudaVersionSupportsArch(gpu_arch); 403 } 404 405 ArgStringList CmdArgs; 406 CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-m64" : "-m32"); 407 DeviceDebugInfoLevel DIKind = mustEmitDebugInfo(Args); 408 if (DIKind == EmitSameDebugInfoAsHost) { 409 // ptxas does not accept -g option if optimization is enabled, so 410 // we ignore the compiler's -O* options if we want debug info. 411 CmdArgs.push_back("-g"); 412 CmdArgs.push_back("--dont-merge-basicblocks"); 413 CmdArgs.push_back("--return-at-end"); 414 } else if (Arg *A = Args.getLastArg(options::OPT_O_Group)) { 415 // Map the -O we received to -O{0,1,2,3}. 416 // 417 // TODO: Perhaps we should map host -O2 to ptxas -O3. -O3 is ptxas's 418 // default, so it may correspond more closely to the spirit of clang -O2. 419 420 // -O3 seems like the least-bad option when -Osomething is specified to 421 // clang but it isn't handled below. 422 StringRef OOpt = "3"; 423 if (A->getOption().matches(options::OPT_O4) || 424 A->getOption().matches(options::OPT_Ofast)) 425 OOpt = "3"; 426 else if (A->getOption().matches(options::OPT_O0)) 427 OOpt = "0"; 428 else if (A->getOption().matches(options::OPT_O)) { 429 // -Os, -Oz, and -O(anything else) map to -O2, for lack of better options. 430 OOpt = llvm::StringSwitch<const char *>(A->getValue()) 431 .Case("1", "1") 432 .Case("2", "2") 433 .Case("3", "3") 434 .Case("s", "2") 435 .Case("z", "2") 436 .Default("2"); 437 } 438 CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-O") + OOpt)); 439 } else { 440 // If no -O was passed, pass -O0 to ptxas -- no opt flag should correspond 441 // to no optimizations, but ptxas's default is -O3. 442 CmdArgs.push_back("-O0"); 443 } 444 if (DIKind == DebugDirectivesOnly) 445 CmdArgs.push_back("-lineinfo"); 446 447 // Pass -v to ptxas if it was passed to the driver. 448 if (Args.hasArg(options::OPT_v)) 449 CmdArgs.push_back("-v"); 450 451 CmdArgs.push_back("--gpu-name"); 452 CmdArgs.push_back(Args.MakeArgString(CudaArchToString(gpu_arch))); 453 CmdArgs.push_back("--output-file"); 454 std::string OutputFileName = TC.getInputFilename(Output); 455 456 // If we are invoking `nvlink` internally we need to output a `.cubin` file. 457 // FIXME: This should hopefully be removed if NVIDIA updates their tooling. 458 if (!C.getInputArgs().getLastArg(options::OPT_c)) { 459 SmallString<256> Filename(Output.getFilename()); 460 llvm::sys::path::replace_extension(Filename, "cubin"); 461 OutputFileName = Filename.str(); 462 } 463 if (Output.isFilename() && OutputFileName != Output.getFilename()) 464 C.addTempFile(Args.MakeArgString(OutputFileName)); 465 466 CmdArgs.push_back(Args.MakeArgString(OutputFileName)); 467 for (const auto &II : Inputs) 468 CmdArgs.push_back(Args.MakeArgString(II.getFilename())); 469 470 for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_ptxas)) 471 CmdArgs.push_back(Args.MakeArgString(A)); 472 473 bool Relocatable; 474 if (JA.isOffloading(Action::OFK_OpenMP)) 475 // In OpenMP we need to generate relocatable code. 476 Relocatable = Args.hasFlag(options::OPT_fopenmp_relocatable_target, 477 options::OPT_fnoopenmp_relocatable_target, 478 /*Default=*/true); 479 else if (JA.isOffloading(Action::OFK_Cuda)) 480 // In CUDA we generate relocatable code by default. 481 Relocatable = Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, 482 /*Default=*/false); 483 else 484 // Otherwise, we are compiling directly and should create linkable output. 485 Relocatable = true; 486 487 if (Relocatable) 488 CmdArgs.push_back("-c"); 489 490 const char *Exec; 491 if (Arg *A = Args.getLastArg(options::OPT_ptxas_path_EQ)) 492 Exec = A->getValue(); 493 else 494 Exec = Args.MakeArgString(TC.GetProgramPath("ptxas")); 495 C.addCommand(std::make_unique<Command>( 496 JA, *this, 497 ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8, 498 "--options-file"}, 499 Exec, CmdArgs, Inputs, Output)); 500 } 501 502 static bool shouldIncludePTX(const ArgList &Args, const char *gpu_arch) { 503 bool includePTX = true; 504 for (Arg *A : Args) { 505 if (!(A->getOption().matches(options::OPT_cuda_include_ptx_EQ) || 506 A->getOption().matches(options::OPT_no_cuda_include_ptx_EQ))) 507 continue; 508 A->claim(); 509 const StringRef ArchStr = A->getValue(); 510 if (ArchStr == "all" || ArchStr == gpu_arch) { 511 includePTX = A->getOption().matches(options::OPT_cuda_include_ptx_EQ); 512 continue; 513 } 514 } 515 return includePTX; 516 } 517 518 // All inputs to this linker must be from CudaDeviceActions, as we need to look 519 // at the Inputs' Actions in order to figure out which GPU architecture they 520 // correspond to. 521 void NVPTX::FatBinary::ConstructJob(Compilation &C, const JobAction &JA, 522 const InputInfo &Output, 523 const InputInfoList &Inputs, 524 const ArgList &Args, 525 const char *LinkingOutput) const { 526 const auto &TC = 527 static_cast<const toolchains::CudaToolChain &>(getToolChain()); 528 assert(TC.getTriple().isNVPTX() && "Wrong platform"); 529 530 ArgStringList CmdArgs; 531 if (TC.CudaInstallation.version() <= CudaVersion::CUDA_100) 532 CmdArgs.push_back("--cuda"); 533 CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-64" : "-32"); 534 CmdArgs.push_back(Args.MakeArgString("--create")); 535 CmdArgs.push_back(Args.MakeArgString(Output.getFilename())); 536 if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost) 537 CmdArgs.push_back("-g"); 538 539 for (const auto &II : Inputs) { 540 auto *A = II.getAction(); 541 assert(A->getInputs().size() == 1 && 542 "Device offload action is expected to have a single input"); 543 const char *gpu_arch_str = A->getOffloadingArch(); 544 assert(gpu_arch_str && 545 "Device action expected to have associated a GPU architecture!"); 546 CudaArch gpu_arch = StringToCudaArch(gpu_arch_str); 547 548 if (II.getType() == types::TY_PP_Asm && 549 !shouldIncludePTX(Args, gpu_arch_str)) 550 continue; 551 // We need to pass an Arch of the form "sm_XX" for cubin files and 552 // "compute_XX" for ptx. 553 const char *Arch = (II.getType() == types::TY_PP_Asm) 554 ? CudaArchToVirtualArchString(gpu_arch) 555 : gpu_arch_str; 556 CmdArgs.push_back( 557 Args.MakeArgString(llvm::Twine("--image=profile=") + Arch + 558 ",file=" + getToolChain().getInputFilename(II))); 559 } 560 561 for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_fatbinary)) 562 CmdArgs.push_back(Args.MakeArgString(A)); 563 564 const char *Exec = Args.MakeArgString(TC.GetProgramPath("fatbinary")); 565 C.addCommand(std::make_unique<Command>( 566 JA, *this, 567 ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8, 568 "--options-file"}, 569 Exec, CmdArgs, Inputs, Output)); 570 } 571 572 void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA, 573 const InputInfo &Output, 574 const InputInfoList &Inputs, 575 const ArgList &Args, 576 const char *LinkingOutput) const { 577 const auto &TC = 578 static_cast<const toolchains::NVPTXToolChain &>(getToolChain()); 579 ArgStringList CmdArgs; 580 581 assert(TC.getTriple().isNVPTX() && "Wrong platform"); 582 583 assert((Output.isFilename() || Output.isNothing()) && "Invalid output."); 584 if (Output.isFilename()) { 585 CmdArgs.push_back("-o"); 586 CmdArgs.push_back(Output.getFilename()); 587 } 588 589 if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost) 590 CmdArgs.push_back("-g"); 591 592 if (Args.hasArg(options::OPT_v)) 593 CmdArgs.push_back("-v"); 594 595 StringRef GPUArch = Args.getLastArgValue(options::OPT_march_EQ); 596 assert(!GPUArch.empty() && "At least one GPU Arch required for nvlink."); 597 598 CmdArgs.push_back("-arch"); 599 CmdArgs.push_back(Args.MakeArgString(GPUArch)); 600 601 // Add paths specified in LIBRARY_PATH environment variable as -L options. 602 addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH"); 603 604 // Add paths for the default clang library path. 605 SmallString<256> DefaultLibPath = 606 llvm::sys::path::parent_path(TC.getDriver().Dir); 607 llvm::sys::path::append(DefaultLibPath, CLANG_INSTALL_LIBDIR_BASENAME); 608 CmdArgs.push_back(Args.MakeArgString(Twine("-L") + DefaultLibPath)); 609 610 for (const auto &II : Inputs) { 611 if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR || 612 II.getType() == types::TY_LTO_BC || II.getType() == types::TY_LLVM_BC) { 613 C.getDriver().Diag(diag::err_drv_no_linker_llvm_support) 614 << getToolChain().getTripleString(); 615 continue; 616 } 617 618 // Currently, we only pass the input files to the linker, we do not pass 619 // any libraries that may be valid only for the host. 620 if (!II.isFilename()) 621 continue; 622 623 // The 'nvlink' application performs RDC-mode linking when given a '.o' 624 // file and device linking when given a '.cubin' file. We always want to 625 // perform device linking, so just rename any '.o' files. 626 // FIXME: This should hopefully be removed if NVIDIA updates their tooling. 627 auto InputFile = getToolChain().getInputFilename(II); 628 if (llvm::sys::path::extension(InputFile) != ".cubin") { 629 // If there are no actions above this one then this is direct input and we 630 // can copy it. Otherwise the input is internal so a `.cubin` file should 631 // exist. 632 if (II.getAction() && II.getAction()->getInputs().size() == 0) { 633 const char *CubinF = 634 Args.MakeArgString(getToolChain().getDriver().GetTemporaryPath( 635 llvm::sys::path::stem(InputFile), "cubin")); 636 if (llvm::sys::fs::copy_file(InputFile, C.addTempFile(CubinF))) 637 continue; 638 639 CmdArgs.push_back(CubinF); 640 } else { 641 SmallString<256> Filename(InputFile); 642 llvm::sys::path::replace_extension(Filename, "cubin"); 643 CmdArgs.push_back(Args.MakeArgString(Filename)); 644 } 645 } else { 646 CmdArgs.push_back(Args.MakeArgString(InputFile)); 647 } 648 } 649 650 C.addCommand(std::make_unique<Command>( 651 JA, *this, 652 ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8, 653 "--options-file"}, 654 Args.MakeArgString(getToolChain().GetProgramPath("nvlink")), CmdArgs, 655 Inputs, Output)); 656 } 657 658 void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple, 659 const llvm::opt::ArgList &Args, 660 std::vector<StringRef> &Features) { 661 if (Args.hasArg(options::OPT_cuda_feature_EQ)) { 662 StringRef PtxFeature = 663 Args.getLastArgValue(options::OPT_cuda_feature_EQ, "+ptx42"); 664 Features.push_back(Args.MakeArgString(PtxFeature)); 665 return; 666 } 667 CudaInstallationDetector CudaInstallation(D, Triple, Args); 668 669 // New CUDA versions often introduce new instructions that are only supported 670 // by new PTX version, so we need to raise PTX level to enable them in NVPTX 671 // back-end. 672 const char *PtxFeature = nullptr; 673 switch (CudaInstallation.version()) { 674 #define CASE_CUDA_VERSION(CUDA_VER, PTX_VER) \ 675 case CudaVersion::CUDA_##CUDA_VER: \ 676 PtxFeature = "+ptx" #PTX_VER; \ 677 break; 678 CASE_CUDA_VERSION(123, 83); 679 CASE_CUDA_VERSION(122, 82); 680 CASE_CUDA_VERSION(121, 81); 681 CASE_CUDA_VERSION(120, 80); 682 CASE_CUDA_VERSION(118, 78); 683 CASE_CUDA_VERSION(117, 77); 684 CASE_CUDA_VERSION(116, 76); 685 CASE_CUDA_VERSION(115, 75); 686 CASE_CUDA_VERSION(114, 74); 687 CASE_CUDA_VERSION(113, 73); 688 CASE_CUDA_VERSION(112, 72); 689 CASE_CUDA_VERSION(111, 71); 690 CASE_CUDA_VERSION(110, 70); 691 CASE_CUDA_VERSION(102, 65); 692 CASE_CUDA_VERSION(101, 64); 693 CASE_CUDA_VERSION(100, 63); 694 CASE_CUDA_VERSION(92, 61); 695 CASE_CUDA_VERSION(91, 61); 696 CASE_CUDA_VERSION(90, 60); 697 #undef CASE_CUDA_VERSION 698 default: 699 PtxFeature = "+ptx42"; 700 } 701 Features.push_back(PtxFeature); 702 } 703 704 /// NVPTX toolchain. Our assembler is ptxas, and our linker is nvlink. This 705 /// operates as a stand-alone version of the NVPTX tools without the host 706 /// toolchain. 707 NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple, 708 const llvm::Triple &HostTriple, 709 const ArgList &Args, bool Freestanding = false) 710 : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args), 711 Freestanding(Freestanding) { 712 if (CudaInstallation.isValid()) 713 getProgramPaths().push_back(std::string(CudaInstallation.getBinPath())); 714 // Lookup binaries into the driver directory, this is used to 715 // discover the 'nvptx-arch' executable. 716 getProgramPaths().push_back(getDriver().Dir); 717 } 718 719 /// We only need the host triple to locate the CUDA binary utilities, use the 720 /// system's default triple if not provided. 721 NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple, 722 const ArgList &Args) 723 : NVPTXToolChain(D, Triple, llvm::Triple(LLVM_HOST_TRIPLE), Args, 724 /*Freestanding=*/true) {} 725 726 llvm::opt::DerivedArgList * 727 NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args, 728 StringRef BoundArch, 729 Action::OffloadKind DeviceOffloadKind) const { 730 DerivedArgList *DAL = 731 ToolChain::TranslateArgs(Args, BoundArch, DeviceOffloadKind); 732 if (!DAL) 733 DAL = new DerivedArgList(Args.getBaseArgs()); 734 735 const OptTable &Opts = getDriver().getOpts(); 736 737 for (Arg *A : Args) 738 if (!llvm::is_contained(*DAL, A)) 739 DAL->append(A); 740 741 if (!DAL->hasArg(options::OPT_march_EQ)) 742 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ), 743 CudaArchToString(CudaArch::CudaDefault)); 744 745 return DAL; 746 } 747 748 void NVPTXToolChain::addClangTargetOptions( 749 const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args, 750 Action::OffloadKind DeviceOffloadingKind) const { 751 // If we are compiling with a standalone NVPTX toolchain we want to try to 752 // mimic a standard environment as much as possible. So we enable lowering 753 // ctor / dtor functions to global symbols that can be registered. 754 if (Freestanding) 755 CC1Args.append({"-mllvm", "--nvptx-lower-global-ctor-dtor"}); 756 } 757 758 bool NVPTXToolChain::supportsDebugInfoOption(const llvm::opt::Arg *A) const { 759 const Option &O = A->getOption(); 760 return (O.matches(options::OPT_gN_Group) && 761 !O.matches(options::OPT_gmodules)) || 762 O.matches(options::OPT_g_Flag) || 763 O.matches(options::OPT_ggdbN_Group) || O.matches(options::OPT_ggdb) || 764 O.matches(options::OPT_gdwarf) || O.matches(options::OPT_gdwarf_2) || 765 O.matches(options::OPT_gdwarf_3) || O.matches(options::OPT_gdwarf_4) || 766 O.matches(options::OPT_gdwarf_5) || 767 O.matches(options::OPT_gcolumn_info); 768 } 769 770 void NVPTXToolChain::adjustDebugInfoKind( 771 llvm::codegenoptions::DebugInfoKind &DebugInfoKind, 772 const ArgList &Args) const { 773 switch (mustEmitDebugInfo(Args)) { 774 case DisableDebugInfo: 775 DebugInfoKind = llvm::codegenoptions::NoDebugInfo; 776 break; 777 case DebugDirectivesOnly: 778 DebugInfoKind = llvm::codegenoptions::DebugDirectivesOnly; 779 break; 780 case EmitSameDebugInfoAsHost: 781 // Use same debug info level as the host. 782 break; 783 } 784 } 785 786 /// CUDA toolchain. Our assembler is ptxas, and our "linker" is fatbinary, 787 /// which isn't properly a linker but nonetheless performs the step of stitching 788 /// together object files from the assembler into a single blob. 789 790 CudaToolChain::CudaToolChain(const Driver &D, const llvm::Triple &Triple, 791 const ToolChain &HostTC, const ArgList &Args) 792 : NVPTXToolChain(D, Triple, HostTC.getTriple(), Args), HostTC(HostTC) {} 793 794 void CudaToolChain::addClangTargetOptions( 795 const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args, 796 Action::OffloadKind DeviceOffloadingKind) const { 797 HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind); 798 799 StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ); 800 assert(!GpuArch.empty() && "Must have an explicit GPU arch."); 801 assert((DeviceOffloadingKind == Action::OFK_OpenMP || 802 DeviceOffloadingKind == Action::OFK_Cuda) && 803 "Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs."); 804 805 if (DeviceOffloadingKind == Action::OFK_Cuda) { 806 CC1Args.append( 807 {"-fcuda-is-device", "-mllvm", "-enable-memcpyopt-without-libcalls"}); 808 809 // Unsized function arguments used for variadics were introduced in CUDA-9.0 810 // We still do not support generating code that actually uses variadic 811 // arguments yet, but we do need to allow parsing them as recent CUDA 812 // headers rely on that. https://github.com/llvm/llvm-project/issues/58410 813 if (CudaInstallation.version() >= CudaVersion::CUDA_90) 814 CC1Args.push_back("-fcuda-allow-variadic-functions"); 815 } 816 817 if (DriverArgs.hasArg(options::OPT_nogpulib)) 818 return; 819 820 if (DeviceOffloadingKind == Action::OFK_OpenMP && 821 DriverArgs.hasArg(options::OPT_S)) 822 return; 823 824 std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch); 825 if (LibDeviceFile.empty()) { 826 getDriver().Diag(diag::err_drv_no_cuda_libdevice) << GpuArch; 827 return; 828 } 829 830 CC1Args.push_back("-mlink-builtin-bitcode"); 831 CC1Args.push_back(DriverArgs.MakeArgString(LibDeviceFile)); 832 833 clang::CudaVersion CudaInstallationVersion = CudaInstallation.version(); 834 835 if (DriverArgs.hasFlag(options::OPT_fcuda_short_ptr, 836 options::OPT_fno_cuda_short_ptr, false)) 837 CC1Args.append({"-mllvm", "--nvptx-short-ptr"}); 838 839 if (CudaInstallationVersion >= CudaVersion::UNKNOWN) 840 CC1Args.push_back( 841 DriverArgs.MakeArgString(Twine("-target-sdk-version=") + 842 CudaVersionToString(CudaInstallationVersion))); 843 844 if (DeviceOffloadingKind == Action::OFK_OpenMP) { 845 if (CudaInstallationVersion < CudaVersion::CUDA_92) { 846 getDriver().Diag( 847 diag::err_drv_omp_offload_target_cuda_version_not_support) 848 << CudaVersionToString(CudaInstallationVersion); 849 return; 850 } 851 852 // Link the bitcode library late if we're using device LTO. 853 if (getDriver().isUsingLTO(/* IsOffload */ true)) 854 return; 855 856 addOpenMPDeviceRTL(getDriver(), DriverArgs, CC1Args, GpuArch.str(), 857 getTriple()); 858 } 859 } 860 861 llvm::DenormalMode CudaToolChain::getDefaultDenormalModeForType( 862 const llvm::opt::ArgList &DriverArgs, const JobAction &JA, 863 const llvm::fltSemantics *FPType) const { 864 if (JA.getOffloadingDeviceKind() == Action::OFK_Cuda) { 865 if (FPType && FPType == &llvm::APFloat::IEEEsingle() && 866 DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero, 867 options::OPT_fno_gpu_flush_denormals_to_zero, false)) 868 return llvm::DenormalMode::getPreserveSign(); 869 } 870 871 assert(JA.getOffloadingDeviceKind() != Action::OFK_Host); 872 return llvm::DenormalMode::getIEEE(); 873 } 874 875 void CudaToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs, 876 ArgStringList &CC1Args) const { 877 // Check our CUDA version if we're going to include the CUDA headers. 878 if (!DriverArgs.hasArg(options::OPT_nogpuinc) && 879 !DriverArgs.hasArg(options::OPT_no_cuda_version_check)) { 880 StringRef Arch = DriverArgs.getLastArgValue(options::OPT_march_EQ); 881 assert(!Arch.empty() && "Must have an explicit GPU arch."); 882 CudaInstallation.CheckCudaVersionSupportsArch(StringToCudaArch(Arch)); 883 } 884 CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args); 885 } 886 887 std::string CudaToolChain::getInputFilename(const InputInfo &Input) const { 888 // Only object files are changed, for example assembly files keep their .s 889 // extensions. If the user requested device-only compilation don't change it. 890 if (Input.getType() != types::TY_Object || getDriver().offloadDeviceOnly()) 891 return ToolChain::getInputFilename(Input); 892 893 // Replace extension for object files with cubin because nvlink relies on 894 // these particular file names. 895 SmallString<256> Filename(ToolChain::getInputFilename(Input)); 896 llvm::sys::path::replace_extension(Filename, "cubin"); 897 return std::string(Filename); 898 } 899 900 llvm::opt::DerivedArgList * 901 CudaToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args, 902 StringRef BoundArch, 903 Action::OffloadKind DeviceOffloadKind) const { 904 DerivedArgList *DAL = 905 HostTC.TranslateArgs(Args, BoundArch, DeviceOffloadKind); 906 if (!DAL) 907 DAL = new DerivedArgList(Args.getBaseArgs()); 908 909 const OptTable &Opts = getDriver().getOpts(); 910 911 // For OpenMP device offloading, append derived arguments. Make sure 912 // flags are not duplicated. 913 // Also append the compute capability. 914 if (DeviceOffloadKind == Action::OFK_OpenMP) { 915 for (Arg *A : Args) 916 if (!llvm::is_contained(*DAL, A)) 917 DAL->append(A); 918 919 if (!DAL->hasArg(options::OPT_march_EQ)) { 920 StringRef Arch = BoundArch; 921 if (Arch.empty()) { 922 auto ArchsOrErr = getSystemGPUArchs(Args); 923 if (!ArchsOrErr) { 924 std::string ErrMsg = 925 llvm::formatv("{0}", llvm::fmt_consume(ArchsOrErr.takeError())); 926 getDriver().Diag(diag::err_drv_undetermined_gpu_arch) 927 << llvm::Triple::getArchTypeName(getArch()) << ErrMsg << "-march"; 928 Arch = CudaArchToString(CudaArch::CudaDefault); 929 } else { 930 Arch = Args.MakeArgString(ArchsOrErr->front()); 931 } 932 } 933 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ), Arch); 934 } 935 936 return DAL; 937 } 938 939 for (Arg *A : Args) { 940 DAL->append(A); 941 } 942 943 if (!BoundArch.empty()) { 944 DAL->eraseArg(options::OPT_march_EQ); 945 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ), 946 BoundArch); 947 } 948 return DAL; 949 } 950 951 Expected<SmallVector<std::string>> 952 CudaToolChain::getSystemGPUArchs(const ArgList &Args) const { 953 // Detect NVIDIA GPUs availible on the system. 954 std::string Program; 955 if (Arg *A = Args.getLastArg(options::OPT_nvptx_arch_tool_EQ)) 956 Program = A->getValue(); 957 else 958 Program = GetProgramPath("nvptx-arch"); 959 960 auto StdoutOrErr = executeToolChainProgram(Program); 961 if (!StdoutOrErr) 962 return StdoutOrErr.takeError(); 963 964 SmallVector<std::string, 1> GPUArchs; 965 for (StringRef Arch : llvm::split((*StdoutOrErr)->getBuffer(), "\n")) 966 if (!Arch.empty()) 967 GPUArchs.push_back(Arch.str()); 968 969 if (GPUArchs.empty()) 970 return llvm::createStringError(std::error_code(), 971 "No NVIDIA GPU detected in the system"); 972 973 return std::move(GPUArchs); 974 } 975 976 Tool *NVPTXToolChain::buildAssembler() const { 977 return new tools::NVPTX::Assembler(*this); 978 } 979 980 Tool *NVPTXToolChain::buildLinker() const { 981 return new tools::NVPTX::Linker(*this); 982 } 983 984 Tool *CudaToolChain::buildAssembler() const { 985 return new tools::NVPTX::Assembler(*this); 986 } 987 988 Tool *CudaToolChain::buildLinker() const { 989 return new tools::NVPTX::FatBinary(*this); 990 } 991 992 void CudaToolChain::addClangWarningOptions(ArgStringList &CC1Args) const { 993 HostTC.addClangWarningOptions(CC1Args); 994 } 995 996 ToolChain::CXXStdlibType 997 CudaToolChain::GetCXXStdlibType(const ArgList &Args) const { 998 return HostTC.GetCXXStdlibType(Args); 999 } 1000 1001 void CudaToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs, 1002 ArgStringList &CC1Args) const { 1003 HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args); 1004 1005 if (!DriverArgs.hasArg(options::OPT_nogpuinc) && CudaInstallation.isValid()) 1006 CC1Args.append( 1007 {"-internal-isystem", 1008 DriverArgs.MakeArgString(CudaInstallation.getIncludePath())}); 1009 } 1010 1011 void CudaToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &Args, 1012 ArgStringList &CC1Args) const { 1013 HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args); 1014 } 1015 1016 void CudaToolChain::AddIAMCUIncludeArgs(const ArgList &Args, 1017 ArgStringList &CC1Args) const { 1018 HostTC.AddIAMCUIncludeArgs(Args, CC1Args); 1019 } 1020 1021 SanitizerMask CudaToolChain::getSupportedSanitizers() const { 1022 // The CudaToolChain only supports sanitizers in the sense that it allows 1023 // sanitizer arguments on the command line if they are supported by the host 1024 // toolchain. The CudaToolChain will actually ignore any command line 1025 // arguments for any of these "supported" sanitizers. That means that no 1026 // sanitization of device code is actually supported at this time. 1027 // 1028 // This behavior is necessary because the host and device toolchains 1029 // invocations often share the command line, so the device toolchain must 1030 // tolerate flags meant only for the host toolchain. 1031 return HostTC.getSupportedSanitizers(); 1032 } 1033 1034 VersionTuple CudaToolChain::computeMSVCVersion(const Driver *D, 1035 const ArgList &Args) const { 1036 return HostTC.computeMSVCVersion(D, Args); 1037 } 1038