1 //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the X86 specific subclass of TargetMachine. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "X86TargetMachine.h" 14 #include "MCTargetDesc/X86MCTargetDesc.h" 15 #include "TargetInfo/X86TargetInfo.h" 16 #include "X86.h" 17 #include "X86CallLowering.h" 18 #include "X86LegalizerInfo.h" 19 #include "X86MacroFusion.h" 20 #include "X86Subtarget.h" 21 #include "X86TargetObjectFile.h" 22 #include "X86TargetTransformInfo.h" 23 #include "llvm/ADT/Optional.h" 24 #include "llvm/ADT/STLExtras.h" 25 #include "llvm/ADT/SmallString.h" 26 #include "llvm/ADT/StringRef.h" 27 #include "llvm/ADT/Triple.h" 28 #include "llvm/Analysis/TargetTransformInfo.h" 29 #include "llvm/CodeGen/ExecutionDomainFix.h" 30 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 31 #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 32 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 33 #include "llvm/CodeGen/GlobalISel/Legalizer.h" 34 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 35 #include "llvm/CodeGen/MachineScheduler.h" 36 #include "llvm/CodeGen/Passes.h" 37 #include "llvm/CodeGen/TargetPassConfig.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/MC/MCAsmInfo.h" 42 #include "llvm/Pass.h" 43 #include "llvm/Support/CodeGen.h" 44 #include "llvm/Support/CommandLine.h" 45 #include "llvm/Support/ErrorHandling.h" 46 #include "llvm/Support/TargetRegistry.h" 47 #include "llvm/Target/TargetLoweringObjectFile.h" 48 #include "llvm/Target/TargetOptions.h" 49 #include <memory> 50 #include <string> 51 52 using namespace llvm; 53 54 static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner", 55 cl::desc("Enable the machine combiner pass"), 56 cl::init(true), cl::Hidden); 57 58 static cl::opt<bool> EnableCondBrFoldingPass("x86-condbr-folding", 59 cl::desc("Enable the conditional branch " 60 "folding pass"), 61 cl::init(false), cl::Hidden); 62 63 extern "C" void LLVMInitializeX86Target() { 64 // Register the target. 65 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target()); 66 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target()); 67 68 PassRegistry &PR = *PassRegistry::getPassRegistry(); 69 initializeGlobalISel(PR); 70 initializeWinEHStatePassPass(PR); 71 initializeFixupBWInstPassPass(PR); 72 initializeEvexToVexInstPassPass(PR); 73 initializeFixupLEAPassPass(PR); 74 initializeFPSPass(PR); 75 initializeX86CallFrameOptimizationPass(PR); 76 initializeX86CmovConverterPassPass(PR); 77 initializeX86ExpandPseudoPass(PR); 78 initializeX86ExecutionDomainFixPass(PR); 79 initializeX86DomainReassignmentPass(PR); 80 initializeX86AvoidSFBPassPass(PR); 81 initializeX86SpeculativeLoadHardeningPassPass(PR); 82 initializeX86FlagsCopyLoweringPassPass(PR); 83 initializeX86CondBrFoldingPassPass(PR); 84 } 85 86 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 87 if (TT.isOSBinFormatMachO()) { 88 if (TT.getArch() == Triple::x86_64) 89 return llvm::make_unique<X86_64MachoTargetObjectFile>(); 90 return llvm::make_unique<TargetLoweringObjectFileMachO>(); 91 } 92 93 if (TT.isOSFreeBSD()) 94 return llvm::make_unique<X86FreeBSDTargetObjectFile>(); 95 if (TT.isOSLinux() || TT.isOSNaCl() || TT.isOSIAMCU()) 96 return llvm::make_unique<X86LinuxNaClTargetObjectFile>(); 97 if (TT.isOSSolaris()) 98 return llvm::make_unique<X86SolarisTargetObjectFile>(); 99 if (TT.isOSFuchsia()) 100 return llvm::make_unique<X86FuchsiaTargetObjectFile>(); 101 if (TT.isOSBinFormatELF()) 102 return llvm::make_unique<X86ELFTargetObjectFile>(); 103 if (TT.isOSBinFormatCOFF()) 104 return llvm::make_unique<TargetLoweringObjectFileCOFF>(); 105 llvm_unreachable("unknown subtarget type"); 106 } 107 108 static std::string computeDataLayout(const Triple &TT) { 109 // X86 is little endian 110 std::string Ret = "e"; 111 112 Ret += DataLayout::getManglingComponent(TT); 113 // X86 and x32 have 32 bit pointers. 114 if ((TT.isArch64Bit() && 115 (TT.getEnvironment() == Triple::GNUX32 || TT.isOSNaCl())) || 116 !TT.isArch64Bit()) 117 Ret += "-p:32:32"; 118 119 // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. 120 if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl()) 121 Ret += "-i64:64"; 122 else if (TT.isOSIAMCU()) 123 Ret += "-i64:32-f64:32"; 124 else 125 Ret += "-f64:32:64"; 126 127 // Some ABIs align long double to 128 bits, others to 32. 128 if (TT.isOSNaCl() || TT.isOSIAMCU()) 129 ; // No f80 130 else if (TT.isArch64Bit() || TT.isOSDarwin()) 131 Ret += "-f80:128"; 132 else 133 Ret += "-f80:32"; 134 135 if (TT.isOSIAMCU()) 136 Ret += "-f128:32"; 137 138 // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. 139 if (TT.isArch64Bit()) 140 Ret += "-n8:16:32:64"; 141 else 142 Ret += "-n8:16:32"; 143 144 // The stack is aligned to 32 bits on some ABIs and 128 bits on others. 145 if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU()) 146 Ret += "-a:0:32-S32"; 147 else 148 Ret += "-S128"; 149 150 return Ret; 151 } 152 153 static Reloc::Model getEffectiveRelocModel(const Triple &TT, 154 bool JIT, 155 Optional<Reloc::Model> RM) { 156 bool is64Bit = TT.getArch() == Triple::x86_64; 157 if (!RM.hasValue()) { 158 // JIT codegen should use static relocations by default, since it's 159 // typically executed in process and not relocatable. 160 if (JIT) 161 return Reloc::Static; 162 163 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode. 164 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we 165 // use static relocation model by default. 166 if (TT.isOSDarwin()) { 167 if (is64Bit) 168 return Reloc::PIC_; 169 return Reloc::DynamicNoPIC; 170 } 171 if (TT.isOSWindows() && is64Bit) 172 return Reloc::PIC_; 173 return Reloc::Static; 174 } 175 176 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC 177 // is defined as a model for code which may be used in static or dynamic 178 // executables but not necessarily a shared library. On X86-32 we just 179 // compile in -static mode, in x86-64 we use PIC. 180 if (*RM == Reloc::DynamicNoPIC) { 181 if (is64Bit) 182 return Reloc::PIC_; 183 if (!TT.isOSDarwin()) 184 return Reloc::Static; 185 } 186 187 // If we are on Darwin, disallow static relocation model in X86-64 mode, since 188 // the Mach-O file format doesn't support it. 189 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit) 190 return Reloc::PIC_; 191 192 return *RM; 193 } 194 195 static CodeModel::Model getEffectiveX86CodeModel(Optional<CodeModel::Model> CM, 196 bool JIT, bool Is64Bit) { 197 if (CM) { 198 if (*CM == CodeModel::Tiny) 199 report_fatal_error("Target does not support the tiny CodeModel", false); 200 return *CM; 201 } 202 if (JIT) 203 return Is64Bit ? CodeModel::Large : CodeModel::Small; 204 return CodeModel::Small; 205 } 206 207 /// Create an X86 target. 208 /// 209 X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, 210 StringRef CPU, StringRef FS, 211 const TargetOptions &Options, 212 Optional<Reloc::Model> RM, 213 Optional<CodeModel::Model> CM, 214 CodeGenOpt::Level OL, bool JIT) 215 : LLVMTargetMachine( 216 T, computeDataLayout(TT), TT, CPU, FS, Options, 217 getEffectiveRelocModel(TT, JIT, RM), 218 getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64), 219 OL), 220 TLOF(createTLOF(getTargetTriple())) { 221 // Windows stack unwinder gets confused when execution flow "falls through" 222 // after a call to 'noreturn' function. 223 // To prevent that, we emit a trap for 'unreachable' IR instructions. 224 // (which on X86, happens to be the 'ud2' instruction) 225 // On PS4, the "return address" of a 'noreturn' call must still be within 226 // the calling function, and TrapUnreachable is an easy way to get that. 227 // The check here for 64-bit windows is a bit icky, but as we're unlikely 228 // to ever want to mix 32 and 64-bit windows code in a single module 229 // this should be fine. 230 if ((TT.isOSWindows() && TT.getArch() == Triple::x86_64) || TT.isPS4() || 231 TT.isOSBinFormatMachO()) { 232 this->Options.TrapUnreachable = true; 233 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO(); 234 } 235 236 // Outlining is available for x86-64. 237 if (TT.getArch() == Triple::x86_64) 238 setMachineOutliner(true); 239 240 initAsmInfo(); 241 } 242 243 X86TargetMachine::~X86TargetMachine() = default; 244 245 const X86Subtarget * 246 X86TargetMachine::getSubtargetImpl(const Function &F) const { 247 Attribute CPUAttr = F.getFnAttribute("target-cpu"); 248 Attribute FSAttr = F.getFnAttribute("target-features"); 249 250 StringRef CPU = !CPUAttr.hasAttribute(Attribute::None) 251 ? CPUAttr.getValueAsString() 252 : (StringRef)TargetCPU; 253 StringRef FS = !FSAttr.hasAttribute(Attribute::None) 254 ? FSAttr.getValueAsString() 255 : (StringRef)TargetFS; 256 257 SmallString<512> Key; 258 Key.reserve(CPU.size() + FS.size()); 259 Key += CPU; 260 Key += FS; 261 262 // FIXME: This is related to the code below to reset the target options, 263 // we need to know whether or not the soft float flag is set on the 264 // function before we can generate a subtarget. We also need to use 265 // it as a key for the subtarget since that can be the only difference 266 // between two functions. 267 bool SoftFloat = 268 F.getFnAttribute("use-soft-float").getValueAsString() == "true"; 269 // If the soft float attribute is set on the function turn on the soft float 270 // subtarget feature. 271 if (SoftFloat) 272 Key += FS.empty() ? "+soft-float" : ",+soft-float"; 273 274 // Keep track of the key width after all features are added so we can extract 275 // the feature string out later. 276 unsigned CPUFSWidth = Key.size(); 277 278 // Extract prefer-vector-width attribute. 279 unsigned PreferVectorWidthOverride = 0; 280 if (F.hasFnAttribute("prefer-vector-width")) { 281 StringRef Val = F.getFnAttribute("prefer-vector-width").getValueAsString(); 282 unsigned Width; 283 if (!Val.getAsInteger(0, Width)) { 284 Key += ",prefer-vector-width="; 285 Key += Val; 286 PreferVectorWidthOverride = Width; 287 } 288 } 289 290 // Extract min-legal-vector-width attribute. 291 unsigned RequiredVectorWidth = UINT32_MAX; 292 if (F.hasFnAttribute("min-legal-vector-width")) { 293 StringRef Val = 294 F.getFnAttribute("min-legal-vector-width").getValueAsString(); 295 unsigned Width; 296 if (!Val.getAsInteger(0, Width)) { 297 Key += ",min-legal-vector-width="; 298 Key += Val; 299 RequiredVectorWidth = Width; 300 } 301 } 302 303 // Extracted here so that we make sure there is backing for the StringRef. If 304 // we assigned earlier, its possible the SmallString reallocated leaving a 305 // dangling StringRef. 306 FS = Key.slice(CPU.size(), CPUFSWidth); 307 308 auto &I = SubtargetMap[Key]; 309 if (!I) { 310 // This needs to be done before we create a new subtarget since any 311 // creation will depend on the TM and the code generation flags on the 312 // function that reside in TargetOptions. 313 resetTargetOptions(F); 314 I = llvm::make_unique<X86Subtarget>(TargetTriple, CPU, FS, *this, 315 Options.StackAlignmentOverride, 316 PreferVectorWidthOverride, 317 RequiredVectorWidth); 318 } 319 return I.get(); 320 } 321 322 //===----------------------------------------------------------------------===// 323 // Command line options for x86 324 //===----------------------------------------------------------------------===// 325 static cl::opt<bool> 326 UseVZeroUpper("x86-use-vzeroupper", cl::Hidden, 327 cl::desc("Minimize AVX to SSE transition penalty"), 328 cl::init(true)); 329 330 //===----------------------------------------------------------------------===// 331 // X86 TTI query. 332 //===----------------------------------------------------------------------===// 333 334 TargetTransformInfo 335 X86TargetMachine::getTargetTransformInfo(const Function &F) { 336 return TargetTransformInfo(X86TTIImpl(this, F)); 337 } 338 339 //===----------------------------------------------------------------------===// 340 // Pass Pipeline Configuration 341 //===----------------------------------------------------------------------===// 342 343 namespace { 344 345 /// X86 Code Generator Pass Configuration Options. 346 class X86PassConfig : public TargetPassConfig { 347 public: 348 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM) 349 : TargetPassConfig(TM, PM) {} 350 351 X86TargetMachine &getX86TargetMachine() const { 352 return getTM<X86TargetMachine>(); 353 } 354 355 ScheduleDAGInstrs * 356 createMachineScheduler(MachineSchedContext *C) const override { 357 ScheduleDAGMILive *DAG = createGenericSchedLive(C); 358 DAG->addMutation(createX86MacroFusionDAGMutation()); 359 return DAG; 360 } 361 362 ScheduleDAGInstrs * 363 createPostMachineScheduler(MachineSchedContext *C) const override { 364 ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 365 DAG->addMutation(createX86MacroFusionDAGMutation()); 366 return DAG; 367 } 368 369 void addIRPasses() override; 370 bool addInstSelector() override; 371 bool addIRTranslator() override; 372 bool addLegalizeMachineIR() override; 373 bool addRegBankSelect() override; 374 bool addGlobalInstructionSelect() override; 375 bool addILPOpts() override; 376 bool addPreISel() override; 377 void addMachineSSAOptimization() override; 378 void addPreRegAlloc() override; 379 void addPostRegAlloc() override; 380 void addPreEmitPass() override; 381 void addPreEmitPass2() override; 382 void addPreSched2() override; 383 384 std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 385 }; 386 387 class X86ExecutionDomainFix : public ExecutionDomainFix { 388 public: 389 static char ID; 390 X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {} 391 StringRef getPassName() const override { 392 return "X86 Execution Dependency Fix"; 393 } 394 }; 395 char X86ExecutionDomainFix::ID; 396 397 } // end anonymous namespace 398 399 INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix", 400 "X86 Execution Domain Fix", false, false) 401 INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) 402 INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix", 403 "X86 Execution Domain Fix", false, false) 404 405 TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) { 406 return new X86PassConfig(*this, PM); 407 } 408 409 void X86PassConfig::addIRPasses() { 410 addPass(createAtomicExpandPass()); 411 412 TargetPassConfig::addIRPasses(); 413 414 if (TM->getOptLevel() != CodeGenOpt::None) 415 addPass(createInterleavedAccessPass()); 416 417 // Add passes that handle indirect branch removal and insertion of a retpoline 418 // thunk. These will be a no-op unless a function subtarget has the retpoline 419 // feature enabled. 420 addPass(createIndirectBrExpandPass()); 421 } 422 423 bool X86PassConfig::addInstSelector() { 424 // Install an instruction selector. 425 addPass(createX86ISelDag(getX86TargetMachine(), getOptLevel())); 426 427 // For ELF, cleanup any local-dynamic TLS accesses. 428 if (TM->getTargetTriple().isOSBinFormatELF() && 429 getOptLevel() != CodeGenOpt::None) 430 addPass(createCleanupLocalDynamicTLSPass()); 431 432 addPass(createX86GlobalBaseRegPass()); 433 return false; 434 } 435 436 bool X86PassConfig::addIRTranslator() { 437 addPass(new IRTranslator()); 438 return false; 439 } 440 441 bool X86PassConfig::addLegalizeMachineIR() { 442 addPass(new Legalizer()); 443 return false; 444 } 445 446 bool X86PassConfig::addRegBankSelect() { 447 addPass(new RegBankSelect()); 448 return false; 449 } 450 451 bool X86PassConfig::addGlobalInstructionSelect() { 452 addPass(new InstructionSelect()); 453 return false; 454 } 455 456 bool X86PassConfig::addILPOpts() { 457 if (EnableCondBrFoldingPass) 458 addPass(createX86CondBrFolding()); 459 addPass(&EarlyIfConverterID); 460 if (EnableMachineCombinerPass) 461 addPass(&MachineCombinerID); 462 addPass(createX86CmovConverterPass()); 463 return true; 464 } 465 466 bool X86PassConfig::addPreISel() { 467 // Only add this pass for 32-bit x86 Windows. 468 const Triple &TT = TM->getTargetTriple(); 469 if (TT.isOSWindows() && TT.getArch() == Triple::x86) 470 addPass(createX86WinEHStatePass()); 471 return true; 472 } 473 474 void X86PassConfig::addPreRegAlloc() { 475 if (getOptLevel() != CodeGenOpt::None) { 476 addPass(&LiveRangeShrinkID); 477 addPass(createX86FixupSetCC()); 478 addPass(createX86OptimizeLEAs()); 479 addPass(createX86CallFrameOptimization()); 480 addPass(createX86AvoidStoreForwardingBlocks()); 481 } 482 483 addPass(createX86SpeculativeLoadHardeningPass()); 484 addPass(createX86FlagsCopyLoweringPass()); 485 addPass(createX86WinAllocaExpander()); 486 } 487 void X86PassConfig::addMachineSSAOptimization() { 488 addPass(createX86DomainReassignmentPass()); 489 TargetPassConfig::addMachineSSAOptimization(); 490 } 491 492 void X86PassConfig::addPostRegAlloc() { 493 addPass(createX86FloatingPointStackifierPass()); 494 } 495 496 void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); } 497 498 void X86PassConfig::addPreEmitPass() { 499 if (getOptLevel() != CodeGenOpt::None) { 500 addPass(new X86ExecutionDomainFix()); 501 addPass(createBreakFalseDeps()); 502 } 503 504 addPass(createX86IndirectBranchTrackingPass()); 505 506 if (UseVZeroUpper) 507 addPass(createX86IssueVZeroUpperPass()); 508 509 if (getOptLevel() != CodeGenOpt::None) { 510 addPass(createX86FixupBWInsts()); 511 addPass(createX86PadShortFunctions()); 512 addPass(createX86FixupLEAs()); 513 addPass(createX86EvexToVexInsts()); 514 } 515 addPass(createX86DiscriminateMemOpsPass()); 516 addPass(createX86InsertPrefetchPass()); 517 } 518 519 void X86PassConfig::addPreEmitPass2() { 520 addPass(createX86RetpolineThunksPass()); 521 // Verify basic block incoming and outgoing cfa offset and register values and 522 // correct CFA calculation rule where needed by inserting appropriate CFI 523 // instructions. 524 const Triple &TT = TM->getTargetTriple(); 525 const MCAsmInfo *MAI = TM->getMCAsmInfo(); 526 if (!TT.isOSDarwin() && 527 (!TT.isOSWindows() || 528 MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) 529 addPass(createCFIInstrInserter()); 530 } 531 532 std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const { 533 return getStandardCSEConfigForOpt(TM->getOptLevel()); 534 } 535