1 //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the X86 specific subclass of TargetMachine. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "X86TargetMachine.h" 14 #include "MCTargetDesc/X86MCTargetDesc.h" 15 #include "TargetInfo/X86TargetInfo.h" 16 #include "X86.h" 17 #include "X86CallLowering.h" 18 #include "X86LegalizerInfo.h" 19 #include "X86MacroFusion.h" 20 #include "X86Subtarget.h" 21 #include "X86TargetObjectFile.h" 22 #include "X86TargetTransformInfo.h" 23 #include "llvm/ADT/Optional.h" 24 #include "llvm/ADT/STLExtras.h" 25 #include "llvm/ADT/SmallString.h" 26 #include "llvm/ADT/StringRef.h" 27 #include "llvm/ADT/Triple.h" 28 #include "llvm/Analysis/TargetTransformInfo.h" 29 #include "llvm/CodeGen/ExecutionDomainFix.h" 30 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 31 #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 32 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 33 #include "llvm/CodeGen/GlobalISel/Legalizer.h" 34 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 35 #include "llvm/CodeGen/MachineScheduler.h" 36 #include "llvm/CodeGen/Passes.h" 37 #include "llvm/CodeGen/TargetPassConfig.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/MC/MCAsmInfo.h" 42 #include "llvm/Pass.h" 43 #include "llvm/Support/CodeGen.h" 44 #include "llvm/Support/CommandLine.h" 45 #include "llvm/Support/ErrorHandling.h" 46 #include "llvm/Support/TargetRegistry.h" 47 #include "llvm/Target/TargetLoweringObjectFile.h" 48 #include "llvm/Target/TargetOptions.h" 49 #include "llvm/Transforms/CFGuard.h" 50 #include <memory> 51 #include <string> 52 53 using namespace llvm; 54 55 static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner", 56 cl::desc("Enable the machine combiner pass"), 57 cl::init(true), cl::Hidden); 58 59 static cl::opt<bool> EnableCondBrFoldingPass("x86-condbr-folding", 60 cl::desc("Enable the conditional branch " 61 "folding pass"), 62 cl::init(false), cl::Hidden); 63 64 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() { 65 // Register the target. 66 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target()); 67 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target()); 68 69 PassRegistry &PR = *PassRegistry::getPassRegistry(); 70 initializeGlobalISel(PR); 71 initializeWinEHStatePassPass(PR); 72 initializeFixupBWInstPassPass(PR); 73 initializeEvexToVexInstPassPass(PR); 74 initializeFixupLEAPassPass(PR); 75 initializeFPSPass(PR); 76 initializeX86CallFrameOptimizationPass(PR); 77 initializeX86CmovConverterPassPass(PR); 78 initializeX86ExpandPseudoPass(PR); 79 initializeX86ExecutionDomainFixPass(PR); 80 initializeX86DomainReassignmentPass(PR); 81 initializeX86AvoidSFBPassPass(PR); 82 initializeX86SpeculativeLoadHardeningPassPass(PR); 83 initializeX86FlagsCopyLoweringPassPass(PR); 84 initializeX86CondBrFoldingPassPass(PR); 85 initializeX86LoadValueInjectionLoadHardeningPassPass(PR); 86 initializeX86LoadValueInjectionRetHardeningPassPass(PR); 87 initializeX86OptimizeLEAPassPass(PR); 88 } 89 90 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 91 if (TT.isOSBinFormatMachO()) { 92 if (TT.getArch() == Triple::x86_64) 93 return std::make_unique<X86_64MachoTargetObjectFile>(); 94 return std::make_unique<TargetLoweringObjectFileMachO>(); 95 } 96 97 if (TT.isOSFreeBSD()) 98 return std::make_unique<X86FreeBSDTargetObjectFile>(); 99 if (TT.isOSLinux() || TT.isOSNaCl() || TT.isOSIAMCU()) 100 return std::make_unique<X86LinuxNaClTargetObjectFile>(); 101 if (TT.isOSSolaris()) 102 return std::make_unique<X86SolarisTargetObjectFile>(); 103 if (TT.isOSFuchsia()) 104 return std::make_unique<X86FuchsiaTargetObjectFile>(); 105 if (TT.isOSBinFormatELF()) 106 return std::make_unique<X86ELFTargetObjectFile>(); 107 if (TT.isOSBinFormatCOFF()) 108 return std::make_unique<TargetLoweringObjectFileCOFF>(); 109 llvm_unreachable("unknown subtarget type"); 110 } 111 112 static std::string computeDataLayout(const Triple &TT) { 113 // X86 is little endian 114 std::string Ret = "e"; 115 116 Ret += DataLayout::getManglingComponent(TT); 117 // X86 and x32 have 32 bit pointers. 118 if ((TT.isArch64Bit() && 119 (TT.getEnvironment() == Triple::GNUX32 || TT.isOSNaCl())) || 120 !TT.isArch64Bit()) 121 Ret += "-p:32:32"; 122 123 // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers. 124 Ret += "-p270:32:32-p271:32:32-p272:64:64"; 125 126 // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. 127 if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl()) 128 Ret += "-i64:64"; 129 else if (TT.isOSIAMCU()) 130 Ret += "-i64:32-f64:32"; 131 else 132 Ret += "-f64:32:64"; 133 134 // Some ABIs align long double to 128 bits, others to 32. 135 if (TT.isOSNaCl() || TT.isOSIAMCU()) 136 ; // No f80 137 else if (TT.isArch64Bit() || TT.isOSDarwin()) 138 Ret += "-f80:128"; 139 else 140 Ret += "-f80:32"; 141 142 if (TT.isOSIAMCU()) 143 Ret += "-f128:32"; 144 145 // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. 146 if (TT.isArch64Bit()) 147 Ret += "-n8:16:32:64"; 148 else 149 Ret += "-n8:16:32"; 150 151 // The stack is aligned to 32 bits on some ABIs and 128 bits on others. 152 if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU()) 153 Ret += "-a:0:32-S32"; 154 else 155 Ret += "-S128"; 156 157 return Ret; 158 } 159 160 static Reloc::Model getEffectiveRelocModel(const Triple &TT, 161 bool JIT, 162 Optional<Reloc::Model> RM) { 163 bool is64Bit = TT.getArch() == Triple::x86_64; 164 if (!RM.hasValue()) { 165 // JIT codegen should use static relocations by default, since it's 166 // typically executed in process and not relocatable. 167 if (JIT) 168 return Reloc::Static; 169 170 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode. 171 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we 172 // use static relocation model by default. 173 if (TT.isOSDarwin()) { 174 if (is64Bit) 175 return Reloc::PIC_; 176 return Reloc::DynamicNoPIC; 177 } 178 if (TT.isOSWindows() && is64Bit) 179 return Reloc::PIC_; 180 return Reloc::Static; 181 } 182 183 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC 184 // is defined as a model for code which may be used in static or dynamic 185 // executables but not necessarily a shared library. On X86-32 we just 186 // compile in -static mode, in x86-64 we use PIC. 187 if (*RM == Reloc::DynamicNoPIC) { 188 if (is64Bit) 189 return Reloc::PIC_; 190 if (!TT.isOSDarwin()) 191 return Reloc::Static; 192 } 193 194 // If we are on Darwin, disallow static relocation model in X86-64 mode, since 195 // the Mach-O file format doesn't support it. 196 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit) 197 return Reloc::PIC_; 198 199 return *RM; 200 } 201 202 static CodeModel::Model getEffectiveX86CodeModel(Optional<CodeModel::Model> CM, 203 bool JIT, bool Is64Bit) { 204 if (CM) { 205 if (*CM == CodeModel::Tiny) 206 report_fatal_error("Target does not support the tiny CodeModel", false); 207 return *CM; 208 } 209 if (JIT) 210 return Is64Bit ? CodeModel::Large : CodeModel::Small; 211 return CodeModel::Small; 212 } 213 214 /// Create an X86 target. 215 /// 216 X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, 217 StringRef CPU, StringRef FS, 218 const TargetOptions &Options, 219 Optional<Reloc::Model> RM, 220 Optional<CodeModel::Model> CM, 221 CodeGenOpt::Level OL, bool JIT) 222 : LLVMTargetMachine( 223 T, computeDataLayout(TT), TT, CPU, FS, Options, 224 getEffectiveRelocModel(TT, JIT, RM), 225 getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64), 226 OL), 227 TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) { 228 // On PS4, the "return address" of a 'noreturn' call must still be within 229 // the calling function, and TrapUnreachable is an easy way to get that. 230 if (TT.isPS4() || TT.isOSBinFormatMachO()) { 231 this->Options.TrapUnreachable = true; 232 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO(); 233 } 234 235 setMachineOutliner(true); 236 237 initAsmInfo(); 238 } 239 240 X86TargetMachine::~X86TargetMachine() = default; 241 242 const X86Subtarget * 243 X86TargetMachine::getSubtargetImpl(const Function &F) const { 244 Attribute CPUAttr = F.getFnAttribute("target-cpu"); 245 Attribute FSAttr = F.getFnAttribute("target-features"); 246 247 StringRef CPU = !CPUAttr.hasAttribute(Attribute::None) 248 ? CPUAttr.getValueAsString() 249 : (StringRef)TargetCPU; 250 StringRef FS = !FSAttr.hasAttribute(Attribute::None) 251 ? FSAttr.getValueAsString() 252 : (StringRef)TargetFS; 253 254 SmallString<512> Key; 255 Key.reserve(CPU.size() + FS.size()); 256 Key += CPU; 257 Key += FS; 258 259 // FIXME: This is related to the code below to reset the target options, 260 // we need to know whether or not the soft float flag is set on the 261 // function before we can generate a subtarget. We also need to use 262 // it as a key for the subtarget since that can be the only difference 263 // between two functions. 264 bool SoftFloat = 265 F.getFnAttribute("use-soft-float").getValueAsString() == "true"; 266 // If the soft float attribute is set on the function turn on the soft float 267 // subtarget feature. 268 if (SoftFloat) 269 Key += FS.empty() ? "+soft-float" : ",+soft-float"; 270 271 // Keep track of the key width after all features are added so we can extract 272 // the feature string out later. 273 unsigned CPUFSWidth = Key.size(); 274 275 // Extract prefer-vector-width attribute. 276 unsigned PreferVectorWidthOverride = 0; 277 if (F.hasFnAttribute("prefer-vector-width")) { 278 StringRef Val = F.getFnAttribute("prefer-vector-width").getValueAsString(); 279 unsigned Width; 280 if (!Val.getAsInteger(0, Width)) { 281 Key += ",prefer-vector-width="; 282 Key += Val; 283 PreferVectorWidthOverride = Width; 284 } 285 } 286 287 // Extract min-legal-vector-width attribute. 288 unsigned RequiredVectorWidth = UINT32_MAX; 289 if (F.hasFnAttribute("min-legal-vector-width")) { 290 StringRef Val = 291 F.getFnAttribute("min-legal-vector-width").getValueAsString(); 292 unsigned Width; 293 if (!Val.getAsInteger(0, Width)) { 294 Key += ",min-legal-vector-width="; 295 Key += Val; 296 RequiredVectorWidth = Width; 297 } 298 } 299 300 // Extracted here so that we make sure there is backing for the StringRef. If 301 // we assigned earlier, its possible the SmallString reallocated leaving a 302 // dangling StringRef. 303 FS = Key.slice(CPU.size(), CPUFSWidth); 304 305 auto &I = SubtargetMap[Key]; 306 if (!I) { 307 // This needs to be done before we create a new subtarget since any 308 // creation will depend on the TM and the code generation flags on the 309 // function that reside in TargetOptions. 310 resetTargetOptions(F); 311 I = std::make_unique<X86Subtarget>( 312 TargetTriple, CPU, FS, *this, 313 MaybeAlign(Options.StackAlignmentOverride), PreferVectorWidthOverride, 314 RequiredVectorWidth); 315 } 316 return I.get(); 317 } 318 319 //===----------------------------------------------------------------------===// 320 // Command line options for x86 321 //===----------------------------------------------------------------------===// 322 static cl::opt<bool> 323 UseVZeroUpper("x86-use-vzeroupper", cl::Hidden, 324 cl::desc("Minimize AVX to SSE transition penalty"), 325 cl::init(true)); 326 327 //===----------------------------------------------------------------------===// 328 // X86 TTI query. 329 //===----------------------------------------------------------------------===// 330 331 TargetTransformInfo 332 X86TargetMachine::getTargetTransformInfo(const Function &F) { 333 return TargetTransformInfo(X86TTIImpl(this, F)); 334 } 335 336 //===----------------------------------------------------------------------===// 337 // Pass Pipeline Configuration 338 //===----------------------------------------------------------------------===// 339 340 namespace { 341 342 /// X86 Code Generator Pass Configuration Options. 343 class X86PassConfig : public TargetPassConfig { 344 public: 345 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM) 346 : TargetPassConfig(TM, PM) {} 347 348 X86TargetMachine &getX86TargetMachine() const { 349 return getTM<X86TargetMachine>(); 350 } 351 352 ScheduleDAGInstrs * 353 createMachineScheduler(MachineSchedContext *C) const override { 354 ScheduleDAGMILive *DAG = createGenericSchedLive(C); 355 DAG->addMutation(createX86MacroFusionDAGMutation()); 356 return DAG; 357 } 358 359 ScheduleDAGInstrs * 360 createPostMachineScheduler(MachineSchedContext *C) const override { 361 ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 362 DAG->addMutation(createX86MacroFusionDAGMutation()); 363 return DAG; 364 } 365 366 void addIRPasses() override; 367 bool addInstSelector() override; 368 bool addIRTranslator() override; 369 bool addLegalizeMachineIR() override; 370 bool addRegBankSelect() override; 371 bool addGlobalInstructionSelect() override; 372 bool addILPOpts() override; 373 bool addPreISel() override; 374 void addMachineSSAOptimization() override; 375 void addPreRegAlloc() override; 376 void addPostRegAlloc() override; 377 void addPreEmitPass() override; 378 void addPreEmitPass2() override; 379 void addPreSched2() override; 380 381 std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 382 }; 383 384 class X86ExecutionDomainFix : public ExecutionDomainFix { 385 public: 386 static char ID; 387 X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {} 388 StringRef getPassName() const override { 389 return "X86 Execution Dependency Fix"; 390 } 391 }; 392 char X86ExecutionDomainFix::ID; 393 394 } // end anonymous namespace 395 396 INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix", 397 "X86 Execution Domain Fix", false, false) 398 INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) 399 INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix", 400 "X86 Execution Domain Fix", false, false) 401 402 TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) { 403 return new X86PassConfig(*this, PM); 404 } 405 406 void X86PassConfig::addIRPasses() { 407 addPass(createAtomicExpandPass()); 408 409 TargetPassConfig::addIRPasses(); 410 411 if (TM->getOptLevel() != CodeGenOpt::None) 412 addPass(createInterleavedAccessPass()); 413 414 // Add passes that handle indirect branch removal and insertion of a retpoline 415 // thunk. These will be a no-op unless a function subtarget has the retpoline 416 // feature enabled. 417 addPass(createIndirectBrExpandPass()); 418 419 // Add Control Flow Guard checks. 420 const Triple &TT = TM->getTargetTriple(); 421 if (TT.isOSWindows()) { 422 if (TT.getArch() == Triple::x86_64) { 423 addPass(createCFGuardDispatchPass()); 424 } else { 425 addPass(createCFGuardCheckPass()); 426 } 427 } 428 } 429 430 bool X86PassConfig::addInstSelector() { 431 // Install an instruction selector. 432 addPass(createX86ISelDag(getX86TargetMachine(), getOptLevel())); 433 434 // For ELF, cleanup any local-dynamic TLS accesses. 435 if (TM->getTargetTriple().isOSBinFormatELF() && 436 getOptLevel() != CodeGenOpt::None) 437 addPass(createCleanupLocalDynamicTLSPass()); 438 439 addPass(createX86GlobalBaseRegPass()); 440 return false; 441 } 442 443 bool X86PassConfig::addIRTranslator() { 444 addPass(new IRTranslator()); 445 return false; 446 } 447 448 bool X86PassConfig::addLegalizeMachineIR() { 449 addPass(new Legalizer()); 450 return false; 451 } 452 453 bool X86PassConfig::addRegBankSelect() { 454 addPass(new RegBankSelect()); 455 return false; 456 } 457 458 bool X86PassConfig::addGlobalInstructionSelect() { 459 addPass(new InstructionSelect()); 460 return false; 461 } 462 463 bool X86PassConfig::addILPOpts() { 464 if (EnableCondBrFoldingPass) 465 addPass(createX86CondBrFolding()); 466 addPass(&EarlyIfConverterID); 467 if (EnableMachineCombinerPass) 468 addPass(&MachineCombinerID); 469 addPass(createX86CmovConverterPass()); 470 return true; 471 } 472 473 bool X86PassConfig::addPreISel() { 474 // Only add this pass for 32-bit x86 Windows. 475 const Triple &TT = TM->getTargetTriple(); 476 if (TT.isOSWindows() && TT.getArch() == Triple::x86) 477 addPass(createX86WinEHStatePass()); 478 return true; 479 } 480 481 void X86PassConfig::addPreRegAlloc() { 482 if (getOptLevel() != CodeGenOpt::None) { 483 addPass(&LiveRangeShrinkID); 484 addPass(createX86FixupSetCC()); 485 addPass(createX86OptimizeLEAs()); 486 addPass(createX86CallFrameOptimization()); 487 addPass(createX86AvoidStoreForwardingBlocks()); 488 } 489 490 addPass(createX86SpeculativeLoadHardeningPass()); 491 addPass(createX86FlagsCopyLoweringPass()); 492 addPass(createX86WinAllocaExpander()); 493 } 494 void X86PassConfig::addMachineSSAOptimization() { 495 addPass(createX86DomainReassignmentPass()); 496 TargetPassConfig::addMachineSSAOptimization(); 497 } 498 499 void X86PassConfig::addPostRegAlloc() { 500 addPass(createX86FloatingPointStackifierPass()); 501 if (getOptLevel() != CodeGenOpt::None) 502 addPass(createX86LoadValueInjectionLoadHardeningPass()); 503 else 504 addPass(createX86LoadValueInjectionLoadHardeningUnoptimizedPass()); 505 } 506 507 void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); } 508 509 void X86PassConfig::addPreEmitPass() { 510 if (getOptLevel() != CodeGenOpt::None) { 511 addPass(new X86ExecutionDomainFix()); 512 addPass(createBreakFalseDeps()); 513 } 514 515 addPass(createX86IndirectBranchTrackingPass()); 516 517 if (UseVZeroUpper) 518 addPass(createX86IssueVZeroUpperPass()); 519 520 if (getOptLevel() != CodeGenOpt::None) { 521 addPass(createX86FixupBWInsts()); 522 addPass(createX86PadShortFunctions()); 523 addPass(createX86FixupLEAs()); 524 addPass(createX86EvexToVexInsts()); 525 } 526 addPass(createX86DiscriminateMemOpsPass()); 527 addPass(createX86InsertPrefetchPass()); 528 } 529 530 void X86PassConfig::addPreEmitPass2() { 531 const Triple &TT = TM->getTargetTriple(); 532 const MCAsmInfo *MAI = TM->getMCAsmInfo(); 533 534 addPass(createX86IndirectThunksPass()); 535 536 // Insert extra int3 instructions after trailing call instructions to avoid 537 // issues in the unwinder. 538 if (TT.isOSWindows() && TT.getArch() == Triple::x86_64) 539 addPass(createX86AvoidTrailingCallPass()); 540 541 // Verify basic block incoming and outgoing cfa offset and register values and 542 // correct CFA calculation rule where needed by inserting appropriate CFI 543 // instructions. 544 if (!TT.isOSDarwin() && 545 (!TT.isOSWindows() || 546 MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) 547 addPass(createCFIInstrInserter()); 548 // Identify valid longjmp targets for Windows Control Flow Guard. 549 if (TT.isOSWindows()) 550 addPass(createCFGuardLongjmpPass()); 551 addPass(createX86LoadValueInjectionRetHardeningPass()); 552 } 553 554 std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const { 555 return getStandardCSEConfigForOpt(TM->getOptLevel()); 556 } 557