1 //===--- AMDGPU.cpp - AMDGPU ToolChain Implementations ----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPU.h" 10 #include "CommonArgs.h" 11 #include "clang/Basic/TargetID.h" 12 #include "clang/Config/config.h" 13 #include "clang/Driver/Compilation.h" 14 #include "clang/Driver/DriverDiagnostic.h" 15 #include "clang/Driver/InputInfo.h" 16 #include "clang/Driver/Options.h" 17 #include "llvm/ADT/StringExtras.h" 18 #include "llvm/Option/ArgList.h" 19 #include "llvm/Support/Error.h" 20 #include "llvm/Support/LineIterator.h" 21 #include "llvm/Support/Path.h" 22 #include "llvm/Support/Process.h" 23 #include "llvm/Support/VirtualFileSystem.h" 24 #include "llvm/TargetParser/Host.h" 25 #include <optional> 26 #include <system_error> 27 28 using namespace clang::driver; 29 using namespace clang::driver::tools; 30 using namespace clang::driver::toolchains; 31 using namespace clang; 32 using namespace llvm::opt; 33 34 // Look for sub-directory starts with PackageName under ROCm candidate path. 35 // If there is one and only one matching sub-directory found, append the 36 // sub-directory to Path. If there is no matching sub-directory or there are 37 // more than one matching sub-directories, diagnose them. Returns the full 38 // path of the package if there is only one matching sub-directory, otherwise 39 // returns an empty string. 40 llvm::SmallString<0> 41 RocmInstallationDetector::findSPACKPackage(const Candidate &Cand, 42 StringRef PackageName) { 43 if (!Cand.isSPACK()) 44 return {}; 45 std::error_code EC; 46 std::string Prefix = Twine(PackageName + "-" + Cand.SPACKReleaseStr).str(); 47 llvm::SmallVector<llvm::SmallString<0>> SubDirs; 48 for (llvm::vfs::directory_iterator File = D.getVFS().dir_begin(Cand.Path, EC), 49 FileEnd; 50 File != FileEnd && !EC; File.increment(EC)) { 51 llvm::StringRef FileName = llvm::sys::path::filename(File->path()); 52 if (FileName.starts_with(Prefix)) { 53 SubDirs.push_back(FileName); 54 if (SubDirs.size() > 1) 55 break; 56 } 57 } 58 if (SubDirs.size() == 1) { 59 auto PackagePath = Cand.Path; 60 llvm::sys::path::append(PackagePath, SubDirs[0]); 61 return PackagePath; 62 } 63 if (SubDirs.size() == 0 && Verbose) { 64 llvm::errs() << "SPACK package " << Prefix << " not found at " << Cand.Path 65 << '\n'; 66 return {}; 67 } 68 69 if (SubDirs.size() > 1 && Verbose) { 70 llvm::errs() << "Cannot use SPACK package " << Prefix << " at " << Cand.Path 71 << " due to multiple installations for the same version\n"; 72 } 73 return {}; 74 } 75 76 void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) { 77 assert(!Path.empty()); 78 79 const StringRef Suffix(".bc"); 80 const StringRef Suffix2(".amdgcn.bc"); 81 82 std::error_code EC; 83 for (llvm::vfs::directory_iterator LI = D.getVFS().dir_begin(Path, EC), LE; 84 !EC && LI != LE; LI = LI.increment(EC)) { 85 StringRef FilePath = LI->path(); 86 StringRef FileName = llvm::sys::path::filename(FilePath); 87 if (!FileName.ends_with(Suffix)) 88 continue; 89 90 StringRef BaseName; 91 if (FileName.ends_with(Suffix2)) 92 BaseName = FileName.drop_back(Suffix2.size()); 93 else if (FileName.ends_with(Suffix)) 94 BaseName = FileName.drop_back(Suffix.size()); 95 96 const StringRef ABIVersionPrefix = "oclc_abi_version_"; 97 if (BaseName == "ocml") { 98 OCML = FilePath; 99 } else if (BaseName == "ockl") { 100 OCKL = FilePath; 101 } else if (BaseName == "opencl") { 102 OpenCL = FilePath; 103 } else if (BaseName == "hip") { 104 HIP = FilePath; 105 } else if (BaseName == "asanrtl") { 106 AsanRTL = FilePath; 107 } else if (BaseName == "oclc_finite_only_off") { 108 FiniteOnly.Off = FilePath; 109 } else if (BaseName == "oclc_finite_only_on") { 110 FiniteOnly.On = FilePath; 111 } else if (BaseName == "oclc_daz_opt_on") { 112 DenormalsAreZero.On = FilePath; 113 } else if (BaseName == "oclc_daz_opt_off") { 114 DenormalsAreZero.Off = FilePath; 115 } else if (BaseName == "oclc_correctly_rounded_sqrt_on") { 116 CorrectlyRoundedSqrt.On = FilePath; 117 } else if (BaseName == "oclc_correctly_rounded_sqrt_off") { 118 CorrectlyRoundedSqrt.Off = FilePath; 119 } else if (BaseName == "oclc_unsafe_math_on") { 120 UnsafeMath.On = FilePath; 121 } else if (BaseName == "oclc_unsafe_math_off") { 122 UnsafeMath.Off = FilePath; 123 } else if (BaseName == "oclc_wavefrontsize64_on") { 124 WavefrontSize64.On = FilePath; 125 } else if (BaseName == "oclc_wavefrontsize64_off") { 126 WavefrontSize64.Off = FilePath; 127 } else if (BaseName.starts_with(ABIVersionPrefix)) { 128 unsigned ABIVersionNumber; 129 if (BaseName.drop_front(ABIVersionPrefix.size()) 130 .getAsInteger(/*Redex=*/0, ABIVersionNumber)) 131 continue; 132 ABIVersionMap[ABIVersionNumber] = FilePath.str(); 133 } else { 134 // Process all bitcode filenames that look like 135 // ocl_isa_version_XXX.amdgcn.bc 136 const StringRef DeviceLibPrefix = "oclc_isa_version_"; 137 if (!BaseName.starts_with(DeviceLibPrefix)) 138 continue; 139 140 StringRef IsaVersionNumber = 141 BaseName.drop_front(DeviceLibPrefix.size()); 142 143 llvm::Twine GfxName = Twine("gfx") + IsaVersionNumber; 144 SmallString<8> Tmp; 145 LibDeviceMap.insert( 146 std::make_pair(GfxName.toStringRef(Tmp), FilePath.str())); 147 } 148 } 149 } 150 151 // Parse and extract version numbers from `.hipVersion`. Return `true` if 152 // the parsing fails. 153 bool RocmInstallationDetector::parseHIPVersionFile(llvm::StringRef V) { 154 SmallVector<StringRef, 4> VersionParts; 155 V.split(VersionParts, '\n'); 156 unsigned Major = ~0U; 157 unsigned Minor = ~0U; 158 for (auto Part : VersionParts) { 159 auto Splits = Part.rtrim().split('='); 160 if (Splits.first == "HIP_VERSION_MAJOR") { 161 if (Splits.second.getAsInteger(0, Major)) 162 return true; 163 } else if (Splits.first == "HIP_VERSION_MINOR") { 164 if (Splits.second.getAsInteger(0, Minor)) 165 return true; 166 } else if (Splits.first == "HIP_VERSION_PATCH") 167 VersionPatch = Splits.second.str(); 168 } 169 if (Major == ~0U || Minor == ~0U) 170 return true; 171 VersionMajorMinor = llvm::VersionTuple(Major, Minor); 172 DetectedVersion = 173 (Twine(Major) + "." + Twine(Minor) + "." + VersionPatch).str(); 174 return false; 175 } 176 177 /// \returns a list of candidate directories for ROCm installation, which is 178 /// cached and populated only once. 179 const SmallVectorImpl<RocmInstallationDetector::Candidate> & 180 RocmInstallationDetector::getInstallationPathCandidates() { 181 182 // Return the cached candidate list if it has already been populated. 183 if (!ROCmSearchDirs.empty()) 184 return ROCmSearchDirs; 185 186 auto DoPrintROCmSearchDirs = [&]() { 187 if (PrintROCmSearchDirs) 188 for (auto Cand : ROCmSearchDirs) { 189 llvm::errs() << "ROCm installation search path"; 190 if (Cand.isSPACK()) 191 llvm::errs() << " (Spack " << Cand.SPACKReleaseStr << ")"; 192 llvm::errs() << ": " << Cand.Path << '\n'; 193 } 194 }; 195 196 // For candidate specified by --rocm-path we do not do strict check, i.e., 197 // checking existence of HIP version file and device library files. 198 if (!RocmPathArg.empty()) { 199 ROCmSearchDirs.emplace_back(RocmPathArg.str()); 200 DoPrintROCmSearchDirs(); 201 return ROCmSearchDirs; 202 } else if (std::optional<std::string> RocmPathEnv = 203 llvm::sys::Process::GetEnv("ROCM_PATH")) { 204 if (!RocmPathEnv->empty()) { 205 ROCmSearchDirs.emplace_back(std::move(*RocmPathEnv)); 206 DoPrintROCmSearchDirs(); 207 return ROCmSearchDirs; 208 } 209 } 210 211 // Try to find relative to the compiler binary. 212 const char *InstallDir = D.getInstalledDir(); 213 214 // Check both a normal Unix prefix position of the clang binary, as well as 215 // the Windows-esque layout the ROCm packages use with the host architecture 216 // subdirectory of bin. 217 auto DeduceROCmPath = [](StringRef ClangPath) { 218 // Strip off directory (usually bin) 219 StringRef ParentDir = llvm::sys::path::parent_path(ClangPath); 220 StringRef ParentName = llvm::sys::path::filename(ParentDir); 221 222 // Some builds use bin/{host arch}, so go up again. 223 if (ParentName == "bin") { 224 ParentDir = llvm::sys::path::parent_path(ParentDir); 225 ParentName = llvm::sys::path::filename(ParentDir); 226 } 227 228 // Detect ROCm packages built with SPACK. 229 // clang is installed at 230 // <rocm_root>/llvm-amdgpu-<rocm_release_string>-<hash>/bin directory. 231 // We only consider the parent directory of llvm-amdgpu package as ROCm 232 // installation candidate for SPACK. 233 if (ParentName.starts_with("llvm-amdgpu-")) { 234 auto SPACKPostfix = 235 ParentName.drop_front(strlen("llvm-amdgpu-")).split('-'); 236 auto SPACKReleaseStr = SPACKPostfix.first; 237 if (!SPACKReleaseStr.empty()) { 238 ParentDir = llvm::sys::path::parent_path(ParentDir); 239 return Candidate(ParentDir.str(), /*StrictChecking=*/true, 240 SPACKReleaseStr); 241 } 242 } 243 244 // Some versions of the rocm llvm package install to /opt/rocm/llvm/bin 245 // Some versions of the aomp package install to /opt/rocm/aomp/bin 246 if (ParentName == "llvm" || ParentName.starts_with("aomp")) 247 ParentDir = llvm::sys::path::parent_path(ParentDir); 248 249 return Candidate(ParentDir.str(), /*StrictChecking=*/true); 250 }; 251 252 // Deduce ROCm path by the path used to invoke clang. Do not resolve symbolic 253 // link of clang itself. 254 ROCmSearchDirs.emplace_back(DeduceROCmPath(InstallDir)); 255 256 // Deduce ROCm path by the real path of the invoked clang, resolving symbolic 257 // link of clang itself. 258 llvm::SmallString<256> RealClangPath; 259 llvm::sys::fs::real_path(D.getClangProgramPath(), RealClangPath); 260 auto ParentPath = llvm::sys::path::parent_path(RealClangPath); 261 if (ParentPath != InstallDir) 262 ROCmSearchDirs.emplace_back(DeduceROCmPath(ParentPath)); 263 264 // Device library may be installed in clang or resource directory. 265 auto ClangRoot = llvm::sys::path::parent_path(InstallDir); 266 auto RealClangRoot = llvm::sys::path::parent_path(ParentPath); 267 ROCmSearchDirs.emplace_back(ClangRoot.str(), /*StrictChecking=*/true); 268 if (RealClangRoot != ClangRoot) 269 ROCmSearchDirs.emplace_back(RealClangRoot.str(), /*StrictChecking=*/true); 270 ROCmSearchDirs.emplace_back(D.ResourceDir, 271 /*StrictChecking=*/true); 272 273 ROCmSearchDirs.emplace_back(D.SysRoot + "/opt/rocm", 274 /*StrictChecking=*/true); 275 276 // Find the latest /opt/rocm-{release} directory. 277 std::error_code EC; 278 std::string LatestROCm; 279 llvm::VersionTuple LatestVer; 280 // Get ROCm version from ROCm directory name. 281 auto GetROCmVersion = [](StringRef DirName) { 282 llvm::VersionTuple V; 283 std::string VerStr = DirName.drop_front(strlen("rocm-")).str(); 284 // The ROCm directory name follows the format of 285 // rocm-{major}.{minor}.{subMinor}[-{build}] 286 std::replace(VerStr.begin(), VerStr.end(), '-', '.'); 287 V.tryParse(VerStr); 288 return V; 289 }; 290 for (llvm::vfs::directory_iterator 291 File = D.getVFS().dir_begin(D.SysRoot + "/opt", EC), 292 FileEnd; 293 File != FileEnd && !EC; File.increment(EC)) { 294 llvm::StringRef FileName = llvm::sys::path::filename(File->path()); 295 if (!FileName.starts_with("rocm-")) 296 continue; 297 if (LatestROCm.empty()) { 298 LatestROCm = FileName.str(); 299 LatestVer = GetROCmVersion(LatestROCm); 300 continue; 301 } 302 auto Ver = GetROCmVersion(FileName); 303 if (LatestVer < Ver) { 304 LatestROCm = FileName.str(); 305 LatestVer = Ver; 306 } 307 } 308 if (!LatestROCm.empty()) 309 ROCmSearchDirs.emplace_back(D.SysRoot + "/opt/" + LatestROCm, 310 /*StrictChecking=*/true); 311 312 ROCmSearchDirs.emplace_back(D.SysRoot + "/usr/local", 313 /*StrictChecking=*/true); 314 ROCmSearchDirs.emplace_back(D.SysRoot + "/usr", 315 /*StrictChecking=*/true); 316 317 DoPrintROCmSearchDirs(); 318 return ROCmSearchDirs; 319 } 320 321 RocmInstallationDetector::RocmInstallationDetector( 322 const Driver &D, const llvm::Triple &HostTriple, 323 const llvm::opt::ArgList &Args, bool DetectHIPRuntime, bool DetectDeviceLib) 324 : D(D) { 325 Verbose = Args.hasArg(options::OPT_v); 326 RocmPathArg = Args.getLastArgValue(clang::driver::options::OPT_rocm_path_EQ); 327 PrintROCmSearchDirs = 328 Args.hasArg(clang::driver::options::OPT_print_rocm_search_dirs); 329 RocmDeviceLibPathArg = 330 Args.getAllArgValues(clang::driver::options::OPT_rocm_device_lib_path_EQ); 331 HIPPathArg = Args.getLastArgValue(clang::driver::options::OPT_hip_path_EQ); 332 HIPStdParPathArg = 333 Args.getLastArgValue(clang::driver::options::OPT_hipstdpar_path_EQ); 334 HasHIPStdParLibrary = 335 !HIPStdParPathArg.empty() && D.getVFS().exists(HIPStdParPathArg + 336 "/hipstdpar_lib.hpp"); 337 HIPRocThrustPathArg = 338 Args.getLastArgValue(clang::driver::options::OPT_hipstdpar_thrust_path_EQ); 339 HasRocThrustLibrary = !HIPRocThrustPathArg.empty() && 340 D.getVFS().exists(HIPRocThrustPathArg + "/thrust"); 341 HIPRocPrimPathArg = 342 Args.getLastArgValue(clang::driver::options::OPT_hipstdpar_prim_path_EQ); 343 HasRocPrimLibrary = !HIPRocPrimPathArg.empty() && 344 D.getVFS().exists(HIPRocPrimPathArg + "/rocprim"); 345 346 if (auto *A = Args.getLastArg(clang::driver::options::OPT_hip_version_EQ)) { 347 HIPVersionArg = A->getValue(); 348 unsigned Major = ~0U; 349 unsigned Minor = ~0U; 350 SmallVector<StringRef, 3> Parts; 351 HIPVersionArg.split(Parts, '.'); 352 if (Parts.size()) 353 Parts[0].getAsInteger(0, Major); 354 if (Parts.size() > 1) 355 Parts[1].getAsInteger(0, Minor); 356 if (Parts.size() > 2) 357 VersionPatch = Parts[2].str(); 358 if (VersionPatch.empty()) 359 VersionPatch = "0"; 360 if (Major != ~0U && Minor == ~0U) 361 Minor = 0; 362 if (Major == ~0U || Minor == ~0U) 363 D.Diag(diag::err_drv_invalid_value) 364 << A->getAsString(Args) << HIPVersionArg; 365 366 VersionMajorMinor = llvm::VersionTuple(Major, Minor); 367 DetectedVersion = 368 (Twine(Major) + "." + Twine(Minor) + "." + VersionPatch).str(); 369 } else { 370 VersionPatch = DefaultVersionPatch; 371 VersionMajorMinor = 372 llvm::VersionTuple(DefaultVersionMajor, DefaultVersionMinor); 373 DetectedVersion = (Twine(DefaultVersionMajor) + "." + 374 Twine(DefaultVersionMinor) + "." + VersionPatch) 375 .str(); 376 } 377 378 if (DetectHIPRuntime) 379 detectHIPRuntime(); 380 if (DetectDeviceLib) 381 detectDeviceLibrary(); 382 } 383 384 void RocmInstallationDetector::detectDeviceLibrary() { 385 assert(LibDevicePath.empty()); 386 387 if (!RocmDeviceLibPathArg.empty()) 388 LibDevicePath = RocmDeviceLibPathArg[RocmDeviceLibPathArg.size() - 1]; 389 else if (std::optional<std::string> LibPathEnv = 390 llvm::sys::Process::GetEnv("HIP_DEVICE_LIB_PATH")) 391 LibDevicePath = std::move(*LibPathEnv); 392 393 auto &FS = D.getVFS(); 394 if (!LibDevicePath.empty()) { 395 // Maintain compatability with HIP flag/envvar pointing directly at the 396 // bitcode library directory. This points directly at the library path instead 397 // of the rocm root installation. 398 if (!FS.exists(LibDevicePath)) 399 return; 400 401 scanLibDevicePath(LibDevicePath); 402 HasDeviceLibrary = allGenericLibsValid() && !LibDeviceMap.empty(); 403 return; 404 } 405 406 // Check device library exists at the given path. 407 auto CheckDeviceLib = [&](StringRef Path, bool StrictChecking) { 408 bool CheckLibDevice = (!NoBuiltinLibs || StrictChecking); 409 if (CheckLibDevice && !FS.exists(Path)) 410 return false; 411 412 scanLibDevicePath(Path); 413 414 if (!NoBuiltinLibs) { 415 // Check that the required non-target libraries are all available. 416 if (!allGenericLibsValid()) 417 return false; 418 419 // Check that we have found at least one libdevice that we can link in 420 // if -nobuiltinlib hasn't been specified. 421 if (LibDeviceMap.empty()) 422 return false; 423 } 424 return true; 425 }; 426 427 // Find device libraries in <LLVM_DIR>/lib/clang/<ver>/lib/amdgcn/bitcode 428 LibDevicePath = D.ResourceDir; 429 llvm::sys::path::append(LibDevicePath, CLANG_INSTALL_LIBDIR_BASENAME, 430 "amdgcn", "bitcode"); 431 HasDeviceLibrary = CheckDeviceLib(LibDevicePath, true); 432 if (HasDeviceLibrary) 433 return; 434 435 // Find device libraries in a legacy ROCm directory structure 436 // ${ROCM_ROOT}/amdgcn/bitcode/* 437 auto &ROCmDirs = getInstallationPathCandidates(); 438 for (const auto &Candidate : ROCmDirs) { 439 LibDevicePath = Candidate.Path; 440 llvm::sys::path::append(LibDevicePath, "amdgcn", "bitcode"); 441 HasDeviceLibrary = CheckDeviceLib(LibDevicePath, Candidate.StrictChecking); 442 if (HasDeviceLibrary) 443 return; 444 } 445 } 446 447 void RocmInstallationDetector::detectHIPRuntime() { 448 SmallVector<Candidate, 4> HIPSearchDirs; 449 if (!HIPPathArg.empty()) 450 HIPSearchDirs.emplace_back(HIPPathArg.str()); 451 else if (std::optional<std::string> HIPPathEnv = 452 llvm::sys::Process::GetEnv("HIP_PATH")) { 453 if (!HIPPathEnv->empty()) 454 HIPSearchDirs.emplace_back(std::move(*HIPPathEnv)); 455 } 456 if (HIPSearchDirs.empty()) 457 HIPSearchDirs.append(getInstallationPathCandidates()); 458 auto &FS = D.getVFS(); 459 460 for (const auto &Candidate : HIPSearchDirs) { 461 InstallPath = Candidate.Path; 462 if (InstallPath.empty() || !FS.exists(InstallPath)) 463 continue; 464 // HIP runtime built by SPACK is installed to 465 // <rocm_root>/hip-<rocm_release_string>-<hash> directory. 466 auto SPACKPath = findSPACKPackage(Candidate, "hip"); 467 InstallPath = SPACKPath.empty() ? InstallPath : SPACKPath; 468 469 BinPath = InstallPath; 470 llvm::sys::path::append(BinPath, "bin"); 471 IncludePath = InstallPath; 472 llvm::sys::path::append(IncludePath, "include"); 473 LibPath = InstallPath; 474 llvm::sys::path::append(LibPath, "lib"); 475 SharePath = InstallPath; 476 llvm::sys::path::append(SharePath, "share"); 477 478 // Get parent of InstallPath and append "share" 479 SmallString<0> ParentSharePath = llvm::sys::path::parent_path(InstallPath); 480 llvm::sys::path::append(ParentSharePath, "share"); 481 482 auto Append = [](SmallString<0> &path, const Twine &a, const Twine &b = "", 483 const Twine &c = "", const Twine &d = "") { 484 SmallString<0> newpath = path; 485 llvm::sys::path::append(newpath, a, b, c, d); 486 return newpath; 487 }; 488 // If HIP version file can be found and parsed, use HIP version from there. 489 for (const auto &VersionFilePath : 490 {Append(SharePath, "hip", "version"), 491 Append(ParentSharePath, "hip", "version"), 492 Append(BinPath, ".hipVersion")}) { 493 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> VersionFile = 494 FS.getBufferForFile(VersionFilePath); 495 if (!VersionFile) 496 continue; 497 if (HIPVersionArg.empty() && VersionFile) 498 if (parseHIPVersionFile((*VersionFile)->getBuffer())) 499 continue; 500 501 HasHIPRuntime = true; 502 return; 503 } 504 // Otherwise, if -rocm-path is specified (no strict checking), use the 505 // default HIP version or specified by --hip-version. 506 if (!Candidate.StrictChecking) { 507 HasHIPRuntime = true; 508 return; 509 } 510 } 511 HasHIPRuntime = false; 512 } 513 514 void RocmInstallationDetector::print(raw_ostream &OS) const { 515 if (hasHIPRuntime()) 516 OS << "Found HIP installation: " << InstallPath << ", version " 517 << DetectedVersion << '\n'; 518 } 519 520 void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs, 521 ArgStringList &CC1Args) const { 522 bool UsesRuntimeWrapper = VersionMajorMinor > llvm::VersionTuple(3, 5) && 523 !DriverArgs.hasArg(options::OPT_nohipwrapperinc); 524 bool HasHipStdPar = DriverArgs.hasArg(options::OPT_hipstdpar); 525 526 if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) { 527 // HIP header includes standard library wrapper headers under clang 528 // cuda_wrappers directory. Since these wrapper headers include_next 529 // standard C++ headers, whereas libc++ headers include_next other clang 530 // headers. The include paths have to follow this order: 531 // - wrapper include path 532 // - standard C++ include path 533 // - other clang include path 534 // Since standard C++ and other clang include paths are added in other 535 // places after this function, here we only need to make sure wrapper 536 // include path is added. 537 // 538 // ROCm 3.5 does not fully support the wrapper headers. Therefore it needs 539 // a workaround. 540 SmallString<128> P(D.ResourceDir); 541 if (UsesRuntimeWrapper) 542 llvm::sys::path::append(P, "include", "cuda_wrappers"); 543 CC1Args.push_back("-internal-isystem"); 544 CC1Args.push_back(DriverArgs.MakeArgString(P)); 545 } 546 547 const auto HandleHipStdPar = [=, &DriverArgs, &CC1Args]() { 548 StringRef Inc = getIncludePath(); 549 auto &FS = D.getVFS(); 550 551 if (!hasHIPStdParLibrary()) 552 if (!HIPStdParPathArg.empty() || 553 !FS.exists(Inc + "/thrust/system/hip/hipstdpar/hipstdpar_lib.hpp")) { 554 D.Diag(diag::err_drv_no_hipstdpar_lib); 555 return; 556 } 557 if (!HasRocThrustLibrary && !FS.exists(Inc + "/thrust")) { 558 D.Diag(diag::err_drv_no_hipstdpar_thrust_lib); 559 return; 560 } 561 if (!HasRocPrimLibrary && !FS.exists(Inc + "/rocprim")) { 562 D.Diag(diag::err_drv_no_hipstdpar_prim_lib); 563 return; 564 } 565 const char *ThrustPath; 566 if (HasRocThrustLibrary) 567 ThrustPath = DriverArgs.MakeArgString(HIPRocThrustPathArg); 568 else 569 ThrustPath = DriverArgs.MakeArgString(Inc + "/thrust"); 570 571 const char *HIPStdParPath; 572 if (hasHIPStdParLibrary()) 573 HIPStdParPath = DriverArgs.MakeArgString(HIPStdParPathArg); 574 else 575 HIPStdParPath = DriverArgs.MakeArgString(StringRef(ThrustPath) + 576 "/system/hip/hipstdpar"); 577 578 const char *PrimPath; 579 if (HasRocPrimLibrary) 580 PrimPath = DriverArgs.MakeArgString(HIPRocPrimPathArg); 581 else 582 PrimPath = DriverArgs.MakeArgString(getIncludePath() + "/rocprim"); 583 584 CC1Args.append({"-idirafter", ThrustPath, "-idirafter", PrimPath, 585 "-idirafter", HIPStdParPath, "-include", 586 "hipstdpar_lib.hpp"}); 587 }; 588 589 if (DriverArgs.hasArg(options::OPT_nogpuinc)) { 590 if (HasHipStdPar) 591 HandleHipStdPar(); 592 593 return; 594 } 595 596 if (!hasHIPRuntime()) { 597 D.Diag(diag::err_drv_no_hip_runtime); 598 return; 599 } 600 601 CC1Args.push_back("-idirafter"); 602 CC1Args.push_back(DriverArgs.MakeArgString(getIncludePath())); 603 if (UsesRuntimeWrapper) 604 CC1Args.append({"-include", "__clang_hip_runtime_wrapper.h"}); 605 if (HasHipStdPar) 606 HandleHipStdPar(); 607 } 608 609 void amdgpu::Linker::ConstructJob(Compilation &C, const JobAction &JA, 610 const InputInfo &Output, 611 const InputInfoList &Inputs, 612 const ArgList &Args, 613 const char *LinkingOutput) const { 614 615 std::string Linker = getToolChain().GetProgramPath(getShortName()); 616 ArgStringList CmdArgs; 617 CmdArgs.push_back("--no-undefined"); 618 CmdArgs.push_back("-shared"); 619 620 addLinkerCompressDebugSectionsOption(getToolChain(), Args, CmdArgs); 621 Args.AddAllArgs(CmdArgs, options::OPT_L); 622 AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA); 623 if (C.getDriver().isUsingLTO()) 624 addLTOOptions(getToolChain(), Args, CmdArgs, Output, Inputs[0], 625 C.getDriver().getLTOMode() == LTOK_Thin); 626 else if (Args.hasArg(options::OPT_mcpu_EQ)) 627 CmdArgs.push_back(Args.MakeArgString( 628 "-plugin-opt=mcpu=" + Args.getLastArgValue(options::OPT_mcpu_EQ))); 629 CmdArgs.push_back("-o"); 630 CmdArgs.push_back(Output.getFilename()); 631 C.addCommand(std::make_unique<Command>( 632 JA, *this, ResponseFileSupport::AtFileCurCP(), Args.MakeArgString(Linker), 633 CmdArgs, Inputs, Output)); 634 } 635 636 void amdgpu::getAMDGPUTargetFeatures(const Driver &D, 637 const llvm::Triple &Triple, 638 const llvm::opt::ArgList &Args, 639 std::vector<StringRef> &Features) { 640 // Add target ID features to -target-feature options. No diagnostics should 641 // be emitted here since invalid target ID is diagnosed at other places. 642 StringRef TargetID = Args.getLastArgValue(options::OPT_mcpu_EQ); 643 if (!TargetID.empty()) { 644 llvm::StringMap<bool> FeatureMap; 645 auto OptionalGpuArch = parseTargetID(Triple, TargetID, &FeatureMap); 646 if (OptionalGpuArch) { 647 StringRef GpuArch = *OptionalGpuArch; 648 // Iterate through all possible target ID features for the given GPU. 649 // If it is mapped to true, add +feature. 650 // If it is mapped to false, add -feature. 651 // If it is not in the map (default), do not add it 652 for (auto &&Feature : getAllPossibleTargetIDFeatures(Triple, GpuArch)) { 653 auto Pos = FeatureMap.find(Feature); 654 if (Pos == FeatureMap.end()) 655 continue; 656 Features.push_back(Args.MakeArgStringRef( 657 (Twine(Pos->second ? "+" : "-") + Feature).str())); 658 } 659 } 660 } 661 662 if (Args.hasFlag(options::OPT_mwavefrontsize64, 663 options::OPT_mno_wavefrontsize64, false)) 664 Features.push_back("+wavefrontsize64"); 665 666 handleTargetFeaturesGroup(D, Triple, Args, Features, 667 options::OPT_m_amdgpu_Features_Group); 668 } 669 670 /// AMDGPU Toolchain 671 AMDGPUToolChain::AMDGPUToolChain(const Driver &D, const llvm::Triple &Triple, 672 const ArgList &Args) 673 : Generic_ELF(D, Triple, Args), 674 OptionsDefault( 675 {{options::OPT_O, "3"}, {options::OPT_cl_std_EQ, "CL1.2"}}) { 676 // Check code object version options. Emit warnings for legacy options 677 // and errors for the last invalid code object version options. 678 // It is done here to avoid repeated warning or error messages for 679 // each tool invocation. 680 checkAMDGPUCodeObjectVersion(D, Args); 681 } 682 683 Tool *AMDGPUToolChain::buildLinker() const { 684 return new tools::amdgpu::Linker(*this); 685 } 686 687 DerivedArgList * 688 AMDGPUToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch, 689 Action::OffloadKind DeviceOffloadKind) const { 690 691 DerivedArgList *DAL = 692 Generic_ELF::TranslateArgs(Args, BoundArch, DeviceOffloadKind); 693 694 const OptTable &Opts = getDriver().getOpts(); 695 696 if (!DAL) 697 DAL = new DerivedArgList(Args.getBaseArgs()); 698 699 for (Arg *A : Args) 700 DAL->append(A); 701 702 // Replace -mcpu=native with detected GPU. 703 Arg *LastMCPUArg = DAL->getLastArg(options::OPT_mcpu_EQ); 704 if (LastMCPUArg && StringRef(LastMCPUArg->getValue()) == "native") { 705 DAL->eraseArg(options::OPT_mcpu_EQ); 706 auto GPUsOrErr = getSystemGPUArchs(Args); 707 if (!GPUsOrErr) { 708 getDriver().Diag(diag::err_drv_undetermined_gpu_arch) 709 << llvm::Triple::getArchTypeName(getArch()) 710 << llvm::toString(GPUsOrErr.takeError()) << "-mcpu"; 711 } else { 712 auto &GPUs = *GPUsOrErr; 713 if (GPUs.size() > 1) { 714 getDriver().Diag(diag::warn_drv_multi_gpu_arch) 715 << llvm::Triple::getArchTypeName(getArch()) 716 << llvm::join(GPUs, ", ") << "-mcpu"; 717 } 718 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_mcpu_EQ), 719 Args.MakeArgString(GPUs.front())); 720 } 721 } 722 723 checkTargetID(*DAL); 724 725 if (!Args.getLastArgValue(options::OPT_x).equals("cl")) 726 return DAL; 727 728 // Phase 1 (.cl -> .bc) 729 if (Args.hasArg(options::OPT_c) && Args.hasArg(options::OPT_emit_llvm)) { 730 DAL->AddFlagArg(nullptr, Opts.getOption(getTriple().isArch64Bit() 731 ? options::OPT_m64 732 : options::OPT_m32)); 733 734 // Have to check OPT_O4, OPT_O0 & OPT_Ofast separately 735 // as they defined that way in Options.td 736 if (!Args.hasArg(options::OPT_O, options::OPT_O0, options::OPT_O4, 737 options::OPT_Ofast)) 738 DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_O), 739 getOptionDefault(options::OPT_O)); 740 } 741 742 return DAL; 743 } 744 745 bool AMDGPUToolChain::getDefaultDenormsAreZeroForTarget( 746 llvm::AMDGPU::GPUKind Kind) { 747 748 // Assume nothing without a specific target. 749 if (Kind == llvm::AMDGPU::GK_NONE) 750 return false; 751 752 const unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(Kind); 753 754 // Default to enabling f32 denormals by default on subtargets where fma is 755 // fast with denormals 756 const bool BothDenormAndFMAFast = 757 (ArchAttr & llvm::AMDGPU::FEATURE_FAST_FMA_F32) && 758 (ArchAttr & llvm::AMDGPU::FEATURE_FAST_DENORMAL_F32); 759 return !BothDenormAndFMAFast; 760 } 761 762 llvm::DenormalMode AMDGPUToolChain::getDefaultDenormalModeForType( 763 const llvm::opt::ArgList &DriverArgs, const JobAction &JA, 764 const llvm::fltSemantics *FPType) const { 765 // Denormals should always be enabled for f16 and f64. 766 if (!FPType || FPType != &llvm::APFloat::IEEEsingle()) 767 return llvm::DenormalMode::getIEEE(); 768 769 if (JA.getOffloadingDeviceKind() == Action::OFK_HIP || 770 JA.getOffloadingDeviceKind() == Action::OFK_Cuda) { 771 auto Arch = getProcessorFromTargetID(getTriple(), JA.getOffloadingArch()); 772 auto Kind = llvm::AMDGPU::parseArchAMDGCN(Arch); 773 if (FPType && FPType == &llvm::APFloat::IEEEsingle() && 774 DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero, 775 options::OPT_fno_gpu_flush_denormals_to_zero, 776 getDefaultDenormsAreZeroForTarget(Kind))) 777 return llvm::DenormalMode::getPreserveSign(); 778 779 return llvm::DenormalMode::getIEEE(); 780 } 781 782 const StringRef GpuArch = getGPUArch(DriverArgs); 783 auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch); 784 785 // TODO: There are way too many flags that change this. Do we need to check 786 // them all? 787 bool DAZ = DriverArgs.hasArg(options::OPT_cl_denorms_are_zero) || 788 getDefaultDenormsAreZeroForTarget(Kind); 789 790 // Outputs are flushed to zero (FTZ), preserving sign. Denormal inputs are 791 // also implicit treated as zero (DAZ). 792 return DAZ ? llvm::DenormalMode::getPreserveSign() : 793 llvm::DenormalMode::getIEEE(); 794 } 795 796 bool AMDGPUToolChain::isWave64(const llvm::opt::ArgList &DriverArgs, 797 llvm::AMDGPU::GPUKind Kind) { 798 const unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(Kind); 799 bool HasWave32 = (ArchAttr & llvm::AMDGPU::FEATURE_WAVE32); 800 801 return !HasWave32 || DriverArgs.hasFlag( 802 options::OPT_mwavefrontsize64, options::OPT_mno_wavefrontsize64, false); 803 } 804 805 806 /// ROCM Toolchain 807 ROCMToolChain::ROCMToolChain(const Driver &D, const llvm::Triple &Triple, 808 const ArgList &Args) 809 : AMDGPUToolChain(D, Triple, Args) { 810 RocmInstallation->detectDeviceLibrary(); 811 } 812 813 void AMDGPUToolChain::addClangTargetOptions( 814 const llvm::opt::ArgList &DriverArgs, 815 llvm::opt::ArgStringList &CC1Args, 816 Action::OffloadKind DeviceOffloadingKind) const { 817 // Default to "hidden" visibility, as object level linking will not be 818 // supported for the foreseeable future. 819 if (!DriverArgs.hasArg(options::OPT_fvisibility_EQ, 820 options::OPT_fvisibility_ms_compat)) { 821 CC1Args.push_back("-fvisibility=hidden"); 822 CC1Args.push_back("-fapply-global-visibility-to-externs"); 823 } 824 } 825 826 StringRef 827 AMDGPUToolChain::getGPUArch(const llvm::opt::ArgList &DriverArgs) const { 828 return getProcessorFromTargetID( 829 getTriple(), DriverArgs.getLastArgValue(options::OPT_mcpu_EQ)); 830 } 831 832 AMDGPUToolChain::ParsedTargetIDType 833 AMDGPUToolChain::getParsedTargetID(const llvm::opt::ArgList &DriverArgs) const { 834 StringRef TargetID = DriverArgs.getLastArgValue(options::OPT_mcpu_EQ); 835 if (TargetID.empty()) 836 return {std::nullopt, std::nullopt, std::nullopt}; 837 838 llvm::StringMap<bool> FeatureMap; 839 auto OptionalGpuArch = parseTargetID(getTriple(), TargetID, &FeatureMap); 840 if (!OptionalGpuArch) 841 return {TargetID.str(), std::nullopt, std::nullopt}; 842 843 return {TargetID.str(), OptionalGpuArch->str(), FeatureMap}; 844 } 845 846 void AMDGPUToolChain::checkTargetID( 847 const llvm::opt::ArgList &DriverArgs) const { 848 auto PTID = getParsedTargetID(DriverArgs); 849 if (PTID.OptionalTargetID && !PTID.OptionalGPUArch) { 850 getDriver().Diag(clang::diag::err_drv_bad_target_id) 851 << *PTID.OptionalTargetID; 852 } 853 } 854 855 Expected<SmallVector<std::string>> 856 AMDGPUToolChain::getSystemGPUArchs(const ArgList &Args) const { 857 // Detect AMD GPUs availible on the system. 858 std::string Program; 859 if (Arg *A = Args.getLastArg(options::OPT_amdgpu_arch_tool_EQ)) 860 Program = A->getValue(); 861 else 862 Program = GetProgramPath("amdgpu-arch"); 863 864 auto StdoutOrErr = executeToolChainProgram(Program); 865 if (!StdoutOrErr) 866 return StdoutOrErr.takeError(); 867 868 SmallVector<std::string, 1> GPUArchs; 869 for (StringRef Arch : llvm::split((*StdoutOrErr)->getBuffer(), "\n")) 870 if (!Arch.empty()) 871 GPUArchs.push_back(Arch.str()); 872 873 if (GPUArchs.empty()) 874 return llvm::createStringError(std::error_code(), 875 "No AMD GPU detected in the system"); 876 877 return std::move(GPUArchs); 878 } 879 880 void ROCMToolChain::addClangTargetOptions( 881 const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args, 882 Action::OffloadKind DeviceOffloadingKind) const { 883 AMDGPUToolChain::addClangTargetOptions(DriverArgs, CC1Args, 884 DeviceOffloadingKind); 885 886 // For the OpenCL case where there is no offload target, accept -nostdlib to 887 // disable bitcode linking. 888 if (DeviceOffloadingKind == Action::OFK_None && 889 DriverArgs.hasArg(options::OPT_nostdlib)) 890 return; 891 892 if (DriverArgs.hasArg(options::OPT_nogpulib)) 893 return; 894 895 // Get the device name and canonicalize it 896 const StringRef GpuArch = getGPUArch(DriverArgs); 897 auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch); 898 const StringRef CanonArch = llvm::AMDGPU::getArchNameAMDGCN(Kind); 899 StringRef LibDeviceFile = RocmInstallation->getLibDeviceFile(CanonArch); 900 auto ABIVer = DeviceLibABIVersion::fromCodeObjectVersion( 901 getAMDGPUCodeObjectVersion(getDriver(), DriverArgs)); 902 if (!RocmInstallation->checkCommonBitcodeLibs(CanonArch, LibDeviceFile, 903 ABIVer)) 904 return; 905 906 bool Wave64 = isWave64(DriverArgs, Kind); 907 908 // TODO: There are way too many flags that change this. Do we need to check 909 // them all? 910 bool DAZ = DriverArgs.hasArg(options::OPT_cl_denorms_are_zero) || 911 getDefaultDenormsAreZeroForTarget(Kind); 912 bool FiniteOnly = DriverArgs.hasArg(options::OPT_cl_finite_math_only); 913 914 bool UnsafeMathOpt = 915 DriverArgs.hasArg(options::OPT_cl_unsafe_math_optimizations); 916 bool FastRelaxedMath = DriverArgs.hasArg(options::OPT_cl_fast_relaxed_math); 917 bool CorrectSqrt = 918 DriverArgs.hasArg(options::OPT_cl_fp32_correctly_rounded_divide_sqrt); 919 920 // Add the OpenCL specific bitcode library. 921 llvm::SmallVector<std::string, 12> BCLibs; 922 BCLibs.push_back(RocmInstallation->getOpenCLPath().str()); 923 924 // Add the generic set of libraries. 925 BCLibs.append(RocmInstallation->getCommonBitcodeLibs( 926 DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt, 927 FastRelaxedMath, CorrectSqrt, ABIVer, false)); 928 929 for (StringRef BCFile : BCLibs) { 930 CC1Args.push_back("-mlink-builtin-bitcode"); 931 CC1Args.push_back(DriverArgs.MakeArgString(BCFile)); 932 } 933 } 934 935 bool RocmInstallationDetector::checkCommonBitcodeLibs( 936 StringRef GPUArch, StringRef LibDeviceFile, 937 DeviceLibABIVersion ABIVer) const { 938 if (!hasDeviceLibrary()) { 939 D.Diag(diag::err_drv_no_rocm_device_lib) << 0; 940 return false; 941 } 942 if (LibDeviceFile.empty()) { 943 D.Diag(diag::err_drv_no_rocm_device_lib) << 1 << GPUArch; 944 return false; 945 } 946 if (ABIVer.requiresLibrary() && getABIVersionPath(ABIVer).empty()) { 947 D.Diag(diag::err_drv_no_rocm_device_lib) << 2 << ABIVer.toString(); 948 return false; 949 } 950 return true; 951 } 952 953 llvm::SmallVector<std::string, 12> 954 RocmInstallationDetector::getCommonBitcodeLibs( 955 const llvm::opt::ArgList &DriverArgs, StringRef LibDeviceFile, bool Wave64, 956 bool DAZ, bool FiniteOnly, bool UnsafeMathOpt, bool FastRelaxedMath, 957 bool CorrectSqrt, DeviceLibABIVersion ABIVer, bool isOpenMP = false) const { 958 llvm::SmallVector<std::string, 12> BCLibs; 959 960 auto AddBCLib = [&](StringRef BCFile) { BCLibs.push_back(BCFile.str()); }; 961 962 AddBCLib(getOCMLPath()); 963 if (!isOpenMP) 964 AddBCLib(getOCKLPath()); 965 AddBCLib(getDenormalsAreZeroPath(DAZ)); 966 AddBCLib(getUnsafeMathPath(UnsafeMathOpt || FastRelaxedMath)); 967 AddBCLib(getFiniteOnlyPath(FiniteOnly || FastRelaxedMath)); 968 AddBCLib(getCorrectlyRoundedSqrtPath(CorrectSqrt)); 969 AddBCLib(getWavefrontSize64Path(Wave64)); 970 AddBCLib(LibDeviceFile); 971 auto ABIVerPath = getABIVersionPath(ABIVer); 972 if (!ABIVerPath.empty()) 973 AddBCLib(ABIVerPath); 974 975 return BCLibs; 976 } 977 978 llvm::SmallVector<std::string, 12> 979 ROCMToolChain::getCommonDeviceLibNames(const llvm::opt::ArgList &DriverArgs, 980 const std::string &GPUArch, 981 bool isOpenMP) const { 982 auto Kind = llvm::AMDGPU::parseArchAMDGCN(GPUArch); 983 const StringRef CanonArch = llvm::AMDGPU::getArchNameAMDGCN(Kind); 984 985 StringRef LibDeviceFile = RocmInstallation->getLibDeviceFile(CanonArch); 986 auto ABIVer = DeviceLibABIVersion::fromCodeObjectVersion( 987 getAMDGPUCodeObjectVersion(getDriver(), DriverArgs)); 988 if (!RocmInstallation->checkCommonBitcodeLibs(CanonArch, LibDeviceFile, 989 ABIVer)) 990 return {}; 991 992 // If --hip-device-lib is not set, add the default bitcode libraries. 993 // TODO: There are way too many flags that change this. Do we need to check 994 // them all? 995 bool DAZ = DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero, 996 options::OPT_fno_gpu_flush_denormals_to_zero, 997 getDefaultDenormsAreZeroForTarget(Kind)); 998 bool FiniteOnly = DriverArgs.hasFlag( 999 options::OPT_ffinite_math_only, options::OPT_fno_finite_math_only, false); 1000 bool UnsafeMathOpt = 1001 DriverArgs.hasFlag(options::OPT_funsafe_math_optimizations, 1002 options::OPT_fno_unsafe_math_optimizations, false); 1003 bool FastRelaxedMath = DriverArgs.hasFlag(options::OPT_ffast_math, 1004 options::OPT_fno_fast_math, false); 1005 bool CorrectSqrt = DriverArgs.hasFlag( 1006 options::OPT_fhip_fp32_correctly_rounded_divide_sqrt, 1007 options::OPT_fno_hip_fp32_correctly_rounded_divide_sqrt, true); 1008 bool Wave64 = isWave64(DriverArgs, Kind); 1009 1010 return RocmInstallation->getCommonBitcodeLibs( 1011 DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt, 1012 FastRelaxedMath, CorrectSqrt, ABIVer, isOpenMP); 1013 } 1014