1 //===-- NVPTXTargetMachine.cpp - Define TargetMachine for NVPTX -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Top-level implementation for the NVPTX target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "NVPTXTargetMachine.h" 14 #include "NVPTX.h" 15 #include "NVPTXAllocaHoisting.h" 16 #include "NVPTXLowerAggrCopies.h" 17 #include "NVPTXTargetObjectFile.h" 18 #include "NVPTXTargetTransformInfo.h" 19 #include "TargetInfo/NVPTXTargetInfo.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/Triple.h" 22 #include "llvm/Analysis/TargetTransformInfo.h" 23 #include "llvm/CodeGen/Passes.h" 24 #include "llvm/CodeGen/TargetPassConfig.h" 25 #include "llvm/IR/LegacyPassManager.h" 26 #include "llvm/Pass.h" 27 #include "llvm/Support/CommandLine.h" 28 #include "llvm/Support/TargetRegistry.h" 29 #include "llvm/Target/TargetMachine.h" 30 #include "llvm/Target/TargetOptions.h" 31 #include "llvm/Transforms/IPO/PassManagerBuilder.h" 32 #include "llvm/Transforms/Scalar.h" 33 #include "llvm/Transforms/Scalar/GVN.h" 34 #include "llvm/Transforms/Vectorize.h" 35 #include <cassert> 36 #include <string> 37 38 using namespace llvm; 39 40 // LSV is still relatively new; this switch lets us turn it off in case we 41 // encounter (or suspect) a bug. 42 static cl::opt<bool> 43 DisableLoadStoreVectorizer("disable-nvptx-load-store-vectorizer", 44 cl::desc("Disable load/store vectorizer"), 45 cl::init(false), cl::Hidden); 46 47 // TODO: Remove this flag when we are confident with no regressions. 48 static cl::opt<bool> DisableRequireStructuredCFG( 49 "disable-nvptx-require-structured-cfg", 50 cl::desc("Transitional flag to turn off NVPTX's requirement on preserving " 51 "structured CFG. The requirement should be disabled only when " 52 "unexpected regressions happen."), 53 cl::init(false), cl::Hidden); 54 55 static cl::opt<bool> UseShortPointersOpt( 56 "nvptx-short-ptr", 57 cl::desc( 58 "Use 32-bit pointers for accessing const/local/shared address spaces."), 59 cl::init(false), cl::Hidden); 60 61 namespace llvm { 62 63 void initializeNVVMIntrRangePass(PassRegistry&); 64 void initializeNVVMReflectPass(PassRegistry&); 65 void initializeGenericToNVVMPass(PassRegistry&); 66 void initializeNVPTXAllocaHoistingPass(PassRegistry &); 67 void initializeNVPTXAssignValidGlobalNamesPass(PassRegistry&); 68 void initializeNVPTXLowerAggrCopiesPass(PassRegistry &); 69 void initializeNVPTXLowerArgsPass(PassRegistry &); 70 void initializeNVPTXLowerAllocaPass(PassRegistry &); 71 void initializeNVPTXProxyRegErasurePass(PassRegistry &); 72 73 } // end namespace llvm 74 75 extern "C" void LLVMInitializeNVPTXTarget() { 76 // Register the target. 77 RegisterTargetMachine<NVPTXTargetMachine32> X(getTheNVPTXTarget32()); 78 RegisterTargetMachine<NVPTXTargetMachine64> Y(getTheNVPTXTarget64()); 79 80 // FIXME: This pass is really intended to be invoked during IR optimization, 81 // but it's very NVPTX-specific. 82 PassRegistry &PR = *PassRegistry::getPassRegistry(); 83 initializeNVVMReflectPass(PR); 84 initializeNVVMIntrRangePass(PR); 85 initializeGenericToNVVMPass(PR); 86 initializeNVPTXAllocaHoistingPass(PR); 87 initializeNVPTXAssignValidGlobalNamesPass(PR); 88 initializeNVPTXLowerArgsPass(PR); 89 initializeNVPTXLowerAllocaPass(PR); 90 initializeNVPTXLowerAggrCopiesPass(PR); 91 initializeNVPTXProxyRegErasurePass(PR); 92 } 93 94 static std::string computeDataLayout(bool is64Bit, bool UseShortPointers) { 95 std::string Ret = "e"; 96 97 if (!is64Bit) 98 Ret += "-p:32:32"; 99 else if (UseShortPointers) 100 Ret += "-p3:32:32-p4:32:32-p5:32:32"; 101 102 Ret += "-i64:64-i128:128-v16:16-v32:32-n16:32:64"; 103 104 return Ret; 105 } 106 107 NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, const Triple &TT, 108 StringRef CPU, StringRef FS, 109 const TargetOptions &Options, 110 Optional<Reloc::Model> RM, 111 Optional<CodeModel::Model> CM, 112 CodeGenOpt::Level OL, bool is64bit) 113 // The pic relocation model is used regardless of what the client has 114 // specified, as it is the only relocation model currently supported. 115 : LLVMTargetMachine(T, computeDataLayout(is64bit, UseShortPointersOpt), TT, 116 CPU, FS, Options, Reloc::PIC_, 117 getEffectiveCodeModel(CM, CodeModel::Small), OL), 118 is64bit(is64bit), UseShortPointers(UseShortPointersOpt), 119 TLOF(std::make_unique<NVPTXTargetObjectFile>()), 120 Subtarget(TT, CPU, FS, *this) { 121 if (TT.getOS() == Triple::NVCL) 122 drvInterface = NVPTX::NVCL; 123 else 124 drvInterface = NVPTX::CUDA; 125 if (!DisableRequireStructuredCFG) 126 setRequiresStructuredCFG(true); 127 initAsmInfo(); 128 } 129 130 NVPTXTargetMachine::~NVPTXTargetMachine() = default; 131 132 void NVPTXTargetMachine32::anchor() {} 133 134 NVPTXTargetMachine32::NVPTXTargetMachine32(const Target &T, const Triple &TT, 135 StringRef CPU, StringRef FS, 136 const TargetOptions &Options, 137 Optional<Reloc::Model> RM, 138 Optional<CodeModel::Model> CM, 139 CodeGenOpt::Level OL, bool JIT) 140 : NVPTXTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {} 141 142 void NVPTXTargetMachine64::anchor() {} 143 144 NVPTXTargetMachine64::NVPTXTargetMachine64(const Target &T, const Triple &TT, 145 StringRef CPU, StringRef FS, 146 const TargetOptions &Options, 147 Optional<Reloc::Model> RM, 148 Optional<CodeModel::Model> CM, 149 CodeGenOpt::Level OL, bool JIT) 150 : NVPTXTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {} 151 152 namespace { 153 154 class NVPTXPassConfig : public TargetPassConfig { 155 public: 156 NVPTXPassConfig(NVPTXTargetMachine &TM, PassManagerBase &PM) 157 : TargetPassConfig(TM, PM) {} 158 159 NVPTXTargetMachine &getNVPTXTargetMachine() const { 160 return getTM<NVPTXTargetMachine>(); 161 } 162 163 void addIRPasses() override; 164 bool addInstSelector() override; 165 void addPreRegAlloc() override; 166 void addPostRegAlloc() override; 167 void addMachineSSAOptimization() override; 168 169 FunctionPass *createTargetRegisterAllocator(bool) override; 170 void addFastRegAlloc() override; 171 void addOptimizedRegAlloc() override; 172 173 bool addRegAssignmentFast() override { 174 llvm_unreachable("should not be used"); 175 } 176 177 bool addRegAssignmentOptimized() override { 178 llvm_unreachable("should not be used"); 179 } 180 181 private: 182 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE. This 183 // function is only called in opt mode. 184 void addEarlyCSEOrGVNPass(); 185 186 // Add passes that propagate special memory spaces. 187 void addAddressSpaceInferencePasses(); 188 189 // Add passes that perform straight-line scalar optimizations. 190 void addStraightLineScalarOptimizationPasses(); 191 }; 192 193 } // end anonymous namespace 194 195 TargetPassConfig *NVPTXTargetMachine::createPassConfig(PassManagerBase &PM) { 196 return new NVPTXPassConfig(*this, PM); 197 } 198 199 void NVPTXTargetMachine::adjustPassManager(PassManagerBuilder &Builder) { 200 Builder.addExtension( 201 PassManagerBuilder::EP_EarlyAsPossible, 202 [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { 203 PM.add(createNVVMReflectPass(Subtarget.getSmVersion())); 204 PM.add(createNVVMIntrRangePass(Subtarget.getSmVersion())); 205 }); 206 } 207 208 TargetTransformInfo 209 NVPTXTargetMachine::getTargetTransformInfo(const Function &F) { 210 return TargetTransformInfo(NVPTXTTIImpl(this, F)); 211 } 212 213 void NVPTXPassConfig::addEarlyCSEOrGVNPass() { 214 if (getOptLevel() == CodeGenOpt::Aggressive) 215 addPass(createGVNPass()); 216 else 217 addPass(createEarlyCSEPass()); 218 } 219 220 void NVPTXPassConfig::addAddressSpaceInferencePasses() { 221 // NVPTXLowerArgs emits alloca for byval parameters which can often 222 // be eliminated by SROA. 223 addPass(createSROAPass()); 224 addPass(createNVPTXLowerAllocaPass()); 225 addPass(createInferAddressSpacesPass()); 226 } 227 228 void NVPTXPassConfig::addStraightLineScalarOptimizationPasses() { 229 addPass(createSeparateConstOffsetFromGEPPass()); 230 addPass(createSpeculativeExecutionPass()); 231 // ReassociateGEPs exposes more opportunites for SLSR. See 232 // the example in reassociate-geps-and-slsr.ll. 233 addPass(createStraightLineStrengthReducePass()); 234 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or 235 // EarlyCSE can reuse. GVN generates significantly better code than EarlyCSE 236 // for some of our benchmarks. 237 addEarlyCSEOrGVNPass(); 238 // Run NaryReassociate after EarlyCSE/GVN to be more effective. 239 addPass(createNaryReassociatePass()); 240 // NaryReassociate on GEPs creates redundant common expressions, so run 241 // EarlyCSE after it. 242 addPass(createEarlyCSEPass()); 243 } 244 245 void NVPTXPassConfig::addIRPasses() { 246 // The following passes are known to not play well with virtual regs hanging 247 // around after register allocation (which in our case, is *all* registers). 248 // We explicitly disable them here. We do, however, need some functionality 249 // of the PrologEpilogCodeInserter pass, so we emulate that behavior in the 250 // NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp). 251 disablePass(&PrologEpilogCodeInserterID); 252 disablePass(&MachineCopyPropagationID); 253 disablePass(&TailDuplicateID); 254 disablePass(&StackMapLivenessID); 255 disablePass(&LiveDebugValuesID); 256 disablePass(&PostRAMachineSinkingID); 257 disablePass(&PostRASchedulerID); 258 disablePass(&FuncletLayoutID); 259 disablePass(&PatchableFunctionID); 260 disablePass(&ShrinkWrapID); 261 262 // NVVMReflectPass is added in addEarlyAsPossiblePasses, so hopefully running 263 // it here does nothing. But since we need it for correctness when lowering 264 // to NVPTX, run it here too, in case whoever built our pass pipeline didn't 265 // call addEarlyAsPossiblePasses. 266 const NVPTXSubtarget &ST = *getTM<NVPTXTargetMachine>().getSubtargetImpl(); 267 addPass(createNVVMReflectPass(ST.getSmVersion())); 268 269 if (getOptLevel() != CodeGenOpt::None) 270 addPass(createNVPTXImageOptimizerPass()); 271 addPass(createNVPTXAssignValidGlobalNamesPass()); 272 addPass(createGenericToNVVMPass()); 273 274 // NVPTXLowerArgs is required for correctness and should be run right 275 // before the address space inference passes. 276 addPass(createNVPTXLowerArgsPass(&getNVPTXTargetMachine())); 277 if (getOptLevel() != CodeGenOpt::None) { 278 addAddressSpaceInferencePasses(); 279 if (!DisableLoadStoreVectorizer) 280 addPass(createLoadStoreVectorizerPass()); 281 addStraightLineScalarOptimizationPasses(); 282 } 283 284 // === LSR and other generic IR passes === 285 TargetPassConfig::addIRPasses(); 286 // EarlyCSE is not always strong enough to clean up what LSR produces. For 287 // example, GVN can combine 288 // 289 // %0 = add %a, %b 290 // %1 = add %b, %a 291 // 292 // and 293 // 294 // %0 = shl nsw %a, 2 295 // %1 = shl %a, 2 296 // 297 // but EarlyCSE can do neither of them. 298 if (getOptLevel() != CodeGenOpt::None) 299 addEarlyCSEOrGVNPass(); 300 } 301 302 bool NVPTXPassConfig::addInstSelector() { 303 const NVPTXSubtarget &ST = *getTM<NVPTXTargetMachine>().getSubtargetImpl(); 304 305 addPass(createLowerAggrCopies()); 306 addPass(createAllocaHoisting()); 307 addPass(createNVPTXISelDag(getNVPTXTargetMachine(), getOptLevel())); 308 309 if (!ST.hasImageHandles()) 310 addPass(createNVPTXReplaceImageHandlesPass()); 311 312 return false; 313 } 314 315 void NVPTXPassConfig::addPreRegAlloc() { 316 // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive. 317 addPass(createNVPTXProxyRegErasurePass()); 318 } 319 320 void NVPTXPassConfig::addPostRegAlloc() { 321 addPass(createNVPTXPrologEpilogPass(), false); 322 if (getOptLevel() != CodeGenOpt::None) { 323 // NVPTXPrologEpilogPass calculates frame object offset and replace frame 324 // index with VRFrame register. NVPTXPeephole need to be run after that and 325 // will replace VRFrame with VRFrameLocal when possible. 326 addPass(createNVPTXPeephole()); 327 } 328 } 329 330 FunctionPass *NVPTXPassConfig::createTargetRegisterAllocator(bool) { 331 return nullptr; // No reg alloc 332 } 333 334 void NVPTXPassConfig::addFastRegAlloc() { 335 addPass(&PHIEliminationID); 336 addPass(&TwoAddressInstructionPassID); 337 } 338 339 void NVPTXPassConfig::addOptimizedRegAlloc() { 340 addPass(&ProcessImplicitDefsID); 341 addPass(&LiveVariablesID); 342 addPass(&MachineLoopInfoID); 343 addPass(&PHIEliminationID); 344 345 addPass(&TwoAddressInstructionPassID); 346 addPass(&RegisterCoalescerID); 347 348 // PreRA instruction scheduling. 349 if (addPass(&MachineSchedulerID)) 350 printAndVerify("After Machine Scheduling"); 351 352 353 addPass(&StackSlotColoringID); 354 355 // FIXME: Needs physical registers 356 //addPass(&MachineLICMID); 357 358 printAndVerify("After StackSlotColoring"); 359 } 360 361 void NVPTXPassConfig::addMachineSSAOptimization() { 362 // Pre-ra tail duplication. 363 if (addPass(&EarlyTailDuplicateID)) 364 printAndVerify("After Pre-RegAlloc TailDuplicate"); 365 366 // Optimize PHIs before DCE: removing dead PHI cycles may make more 367 // instructions dead. 368 addPass(&OptimizePHIsID); 369 370 // This pass merges large allocas. StackSlotColoring is a different pass 371 // which merges spill slots. 372 addPass(&StackColoringID); 373 374 // If the target requests it, assign local variables to stack slots relative 375 // to one another and simplify frame index references where possible. 376 addPass(&LocalStackSlotAllocationID); 377 378 // With optimization, dead code should already be eliminated. However 379 // there is one known exception: lowered code for arguments that are only 380 // used by tail calls, where the tail calls reuse the incoming stack 381 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). 382 addPass(&DeadMachineInstructionElimID); 383 printAndVerify("After codegen DCE pass"); 384 385 // Allow targets to insert passes that improve instruction level parallelism, 386 // like if-conversion. Such passes will typically need dominator trees and 387 // loop info, just like LICM and CSE below. 388 if (addILPOpts()) 389 printAndVerify("After ILP optimizations"); 390 391 addPass(&EarlyMachineLICMID); 392 addPass(&MachineCSEID); 393 394 addPass(&MachineSinkingID); 395 printAndVerify("After Machine LICM, CSE and Sinking passes"); 396 397 addPass(&PeepholeOptimizerID); 398 printAndVerify("After codegen peephole optimization pass"); 399 } 400