10b57cec5SDimitry Andric //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file defines the X86 specific subclass of TargetMachine. 100b57cec5SDimitry Andric // 110b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 120b57cec5SDimitry Andric 130b57cec5SDimitry Andric #include "X86TargetMachine.h" 140b57cec5SDimitry Andric #include "MCTargetDesc/X86MCTargetDesc.h" 150b57cec5SDimitry Andric #include "TargetInfo/X86TargetInfo.h" 160b57cec5SDimitry Andric #include "X86.h" 170b57cec5SDimitry Andric #include "X86CallLowering.h" 180b57cec5SDimitry Andric #include "X86LegalizerInfo.h" 190b57cec5SDimitry Andric #include "X86MacroFusion.h" 200b57cec5SDimitry Andric #include "X86Subtarget.h" 210b57cec5SDimitry Andric #include "X86TargetObjectFile.h" 220b57cec5SDimitry Andric #include "X86TargetTransformInfo.h" 230b57cec5SDimitry Andric #include "llvm/ADT/Optional.h" 240b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h" 250b57cec5SDimitry Andric #include "llvm/ADT/SmallString.h" 260b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h" 270b57cec5SDimitry Andric #include "llvm/ADT/Triple.h" 280b57cec5SDimitry Andric #include "llvm/Analysis/TargetTransformInfo.h" 290b57cec5SDimitry Andric #include "llvm/CodeGen/ExecutionDomainFix.h" 30*81ad6265SDimitry Andric #include "llvm/CodeGen/GlobalISel/CSEInfo.h" 310b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/CallLowering.h" 320b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 330b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 34*81ad6265SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 350b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/Legalizer.h" 360b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 370b57cec5SDimitry Andric #include "llvm/CodeGen/MachineScheduler.h" 380b57cec5SDimitry Andric #include "llvm/CodeGen/Passes.h" 39*81ad6265SDimitry Andric #include "llvm/CodeGen/RegAllocRegistry.h" 400b57cec5SDimitry Andric #include "llvm/CodeGen/TargetPassConfig.h" 410b57cec5SDimitry Andric #include "llvm/IR/Attributes.h" 420b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h" 430b57cec5SDimitry Andric #include "llvm/IR/Function.h" 440b57cec5SDimitry Andric #include "llvm/MC/MCAsmInfo.h" 45349cc55cSDimitry Andric #include "llvm/MC/TargetRegistry.h" 460b57cec5SDimitry Andric #include "llvm/Pass.h" 470b57cec5SDimitry Andric #include "llvm/Support/CodeGen.h" 480b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h" 490b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h" 500b57cec5SDimitry Andric #include "llvm/Target/TargetLoweringObjectFile.h" 510b57cec5SDimitry Andric #include "llvm/Target/TargetOptions.h" 52480093f4SDimitry Andric #include "llvm/Transforms/CFGuard.h" 530b57cec5SDimitry Andric #include <memory> 540b57cec5SDimitry Andric #include <string> 550b57cec5SDimitry Andric 560b57cec5SDimitry Andric using namespace llvm; 570b57cec5SDimitry Andric 580b57cec5SDimitry Andric static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner", 590b57cec5SDimitry Andric cl::desc("Enable the machine combiner pass"), 600b57cec5SDimitry Andric cl::init(true), cl::Hidden); 610b57cec5SDimitry Andric 62*81ad6265SDimitry Andric static cl::opt<bool> 63*81ad6265SDimitry Andric EnableTileRAPass("x86-tile-ra", 64*81ad6265SDimitry Andric cl::desc("Enable the tile register allocation pass"), 65*81ad6265SDimitry Andric cl::init(true), cl::Hidden); 66*81ad6265SDimitry Andric 67480093f4SDimitry Andric extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() { 680b57cec5SDimitry Andric // Register the target. 690b57cec5SDimitry Andric RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target()); 700b57cec5SDimitry Andric RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target()); 710b57cec5SDimitry Andric 720b57cec5SDimitry Andric PassRegistry &PR = *PassRegistry::getPassRegistry(); 73fe6060f1SDimitry Andric initializeX86LowerAMXIntrinsicsLegacyPassPass(PR); 74e8d8bef9SDimitry Andric initializeX86LowerAMXTypeLegacyPassPass(PR); 75fe6060f1SDimitry Andric initializeX86PreAMXConfigPassPass(PR); 76*81ad6265SDimitry Andric initializeX86PreTileConfigPass(PR); 770b57cec5SDimitry Andric initializeGlobalISel(PR); 780b57cec5SDimitry Andric initializeWinEHStatePassPass(PR); 790b57cec5SDimitry Andric initializeFixupBWInstPassPass(PR); 800b57cec5SDimitry Andric initializeEvexToVexInstPassPass(PR); 810b57cec5SDimitry Andric initializeFixupLEAPassPass(PR); 820b57cec5SDimitry Andric initializeFPSPass(PR); 835ffd83dbSDimitry Andric initializeX86FixupSetCCPassPass(PR); 840b57cec5SDimitry Andric initializeX86CallFrameOptimizationPass(PR); 850b57cec5SDimitry Andric initializeX86CmovConverterPassPass(PR); 86e8d8bef9SDimitry Andric initializeX86TileConfigPass(PR); 87*81ad6265SDimitry Andric initializeX86FastPreTileConfigPass(PR); 88fe6060f1SDimitry Andric initializeX86FastTileConfigPass(PR); 89fe6060f1SDimitry Andric initializeX86LowerTileCopyPass(PR); 900b57cec5SDimitry Andric initializeX86ExpandPseudoPass(PR); 910b57cec5SDimitry Andric initializeX86ExecutionDomainFixPass(PR); 920b57cec5SDimitry Andric initializeX86DomainReassignmentPass(PR); 930b57cec5SDimitry Andric initializeX86AvoidSFBPassPass(PR); 945ffd83dbSDimitry Andric initializeX86AvoidTrailingCallPassPass(PR); 950b57cec5SDimitry Andric initializeX86SpeculativeLoadHardeningPassPass(PR); 965ffd83dbSDimitry Andric initializeX86SpeculativeExecutionSideEffectSuppressionPass(PR); 970b57cec5SDimitry Andric initializeX86FlagsCopyLoweringPassPass(PR); 980946e70aSDimitry Andric initializeX86LoadValueInjectionLoadHardeningPassPass(PR); 990946e70aSDimitry Andric initializeX86LoadValueInjectionRetHardeningPassPass(PR); 1008bcb0991SDimitry Andric initializeX86OptimizeLEAPassPass(PR); 1015ffd83dbSDimitry Andric initializeX86PartialReductionPass(PR); 102e8d8bef9SDimitry Andric initializePseudoProbeInserterPass(PR); 1030b57cec5SDimitry Andric } 1040b57cec5SDimitry Andric 1050b57cec5SDimitry Andric static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 1060b57cec5SDimitry Andric if (TT.isOSBinFormatMachO()) { 1070b57cec5SDimitry Andric if (TT.getArch() == Triple::x86_64) 1088bcb0991SDimitry Andric return std::make_unique<X86_64MachoTargetObjectFile>(); 1098bcb0991SDimitry Andric return std::make_unique<TargetLoweringObjectFileMachO>(); 1100b57cec5SDimitry Andric } 1110b57cec5SDimitry Andric 1120b57cec5SDimitry Andric if (TT.isOSBinFormatCOFF()) 1138bcb0991SDimitry Andric return std::make_unique<TargetLoweringObjectFileCOFF>(); 1145ffd83dbSDimitry Andric return std::make_unique<X86ELFTargetObjectFile>(); 1150b57cec5SDimitry Andric } 1160b57cec5SDimitry Andric 1170b57cec5SDimitry Andric static std::string computeDataLayout(const Triple &TT) { 1180b57cec5SDimitry Andric // X86 is little endian 1190b57cec5SDimitry Andric std::string Ret = "e"; 1200b57cec5SDimitry Andric 1210b57cec5SDimitry Andric Ret += DataLayout::getManglingComponent(TT); 1220b57cec5SDimitry Andric // X86 and x32 have 32 bit pointers. 123fe6060f1SDimitry Andric if (!TT.isArch64Bit() || TT.isX32() || TT.isOSNaCl()) 1240b57cec5SDimitry Andric Ret += "-p:32:32"; 1250b57cec5SDimitry Andric 1268bcb0991SDimitry Andric // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers. 1278bcb0991SDimitry Andric Ret += "-p270:32:32-p271:32:32-p272:64:64"; 1288bcb0991SDimitry Andric 1290b57cec5SDimitry Andric // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. 1300b57cec5SDimitry Andric if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl()) 1310b57cec5SDimitry Andric Ret += "-i64:64"; 1320b57cec5SDimitry Andric else if (TT.isOSIAMCU()) 1330b57cec5SDimitry Andric Ret += "-i64:32-f64:32"; 1340b57cec5SDimitry Andric else 1350b57cec5SDimitry Andric Ret += "-f64:32:64"; 1360b57cec5SDimitry Andric 1370b57cec5SDimitry Andric // Some ABIs align long double to 128 bits, others to 32. 1380b57cec5SDimitry Andric if (TT.isOSNaCl() || TT.isOSIAMCU()) 1390b57cec5SDimitry Andric ; // No f80 14004eeddc0SDimitry Andric else if (TT.isArch64Bit() || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment()) 1410b57cec5SDimitry Andric Ret += "-f80:128"; 1420b57cec5SDimitry Andric else 1430b57cec5SDimitry Andric Ret += "-f80:32"; 1440b57cec5SDimitry Andric 1450b57cec5SDimitry Andric if (TT.isOSIAMCU()) 1460b57cec5SDimitry Andric Ret += "-f128:32"; 1470b57cec5SDimitry Andric 1480b57cec5SDimitry Andric // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. 1490b57cec5SDimitry Andric if (TT.isArch64Bit()) 1500b57cec5SDimitry Andric Ret += "-n8:16:32:64"; 1510b57cec5SDimitry Andric else 1520b57cec5SDimitry Andric Ret += "-n8:16:32"; 1530b57cec5SDimitry Andric 1540b57cec5SDimitry Andric // The stack is aligned to 32 bits on some ABIs and 128 bits on others. 1550b57cec5SDimitry Andric if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU()) 1560b57cec5SDimitry Andric Ret += "-a:0:32-S32"; 1570b57cec5SDimitry Andric else 1580b57cec5SDimitry Andric Ret += "-S128"; 1590b57cec5SDimitry Andric 1600b57cec5SDimitry Andric return Ret; 1610b57cec5SDimitry Andric } 1620b57cec5SDimitry Andric 1630b57cec5SDimitry Andric static Reloc::Model getEffectiveRelocModel(const Triple &TT, 1640b57cec5SDimitry Andric bool JIT, 1650b57cec5SDimitry Andric Optional<Reloc::Model> RM) { 1660b57cec5SDimitry Andric bool is64Bit = TT.getArch() == Triple::x86_64; 167*81ad6265SDimitry Andric if (!RM) { 1680b57cec5SDimitry Andric // JIT codegen should use static relocations by default, since it's 1690b57cec5SDimitry Andric // typically executed in process and not relocatable. 1700b57cec5SDimitry Andric if (JIT) 1710b57cec5SDimitry Andric return Reloc::Static; 1720b57cec5SDimitry Andric 1730b57cec5SDimitry Andric // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode. 1740b57cec5SDimitry Andric // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we 1750b57cec5SDimitry Andric // use static relocation model by default. 1760b57cec5SDimitry Andric if (TT.isOSDarwin()) { 1770b57cec5SDimitry Andric if (is64Bit) 1780b57cec5SDimitry Andric return Reloc::PIC_; 1790b57cec5SDimitry Andric return Reloc::DynamicNoPIC; 1800b57cec5SDimitry Andric } 1810b57cec5SDimitry Andric if (TT.isOSWindows() && is64Bit) 1820b57cec5SDimitry Andric return Reloc::PIC_; 1830b57cec5SDimitry Andric return Reloc::Static; 1840b57cec5SDimitry Andric } 1850b57cec5SDimitry Andric 1860b57cec5SDimitry Andric // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC 1870b57cec5SDimitry Andric // is defined as a model for code which may be used in static or dynamic 1880b57cec5SDimitry Andric // executables but not necessarily a shared library. On X86-32 we just 1890b57cec5SDimitry Andric // compile in -static mode, in x86-64 we use PIC. 1900b57cec5SDimitry Andric if (*RM == Reloc::DynamicNoPIC) { 1910b57cec5SDimitry Andric if (is64Bit) 1920b57cec5SDimitry Andric return Reloc::PIC_; 1930b57cec5SDimitry Andric if (!TT.isOSDarwin()) 1940b57cec5SDimitry Andric return Reloc::Static; 1950b57cec5SDimitry Andric } 1960b57cec5SDimitry Andric 1970b57cec5SDimitry Andric // If we are on Darwin, disallow static relocation model in X86-64 mode, since 1980b57cec5SDimitry Andric // the Mach-O file format doesn't support it. 1990b57cec5SDimitry Andric if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit) 2000b57cec5SDimitry Andric return Reloc::PIC_; 2010b57cec5SDimitry Andric 2020b57cec5SDimitry Andric return *RM; 2030b57cec5SDimitry Andric } 2040b57cec5SDimitry Andric 2050b57cec5SDimitry Andric static CodeModel::Model getEffectiveX86CodeModel(Optional<CodeModel::Model> CM, 2060b57cec5SDimitry Andric bool JIT, bool Is64Bit) { 2070b57cec5SDimitry Andric if (CM) { 2080b57cec5SDimitry Andric if (*CM == CodeModel::Tiny) 2090b57cec5SDimitry Andric report_fatal_error("Target does not support the tiny CodeModel", false); 2100b57cec5SDimitry Andric return *CM; 2110b57cec5SDimitry Andric } 2120b57cec5SDimitry Andric if (JIT) 2130b57cec5SDimitry Andric return Is64Bit ? CodeModel::Large : CodeModel::Small; 2140b57cec5SDimitry Andric return CodeModel::Small; 2150b57cec5SDimitry Andric } 2160b57cec5SDimitry Andric 2170b57cec5SDimitry Andric /// Create an X86 target. 2180b57cec5SDimitry Andric /// 2190b57cec5SDimitry Andric X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, 2200b57cec5SDimitry Andric StringRef CPU, StringRef FS, 2210b57cec5SDimitry Andric const TargetOptions &Options, 2220b57cec5SDimitry Andric Optional<Reloc::Model> RM, 2230b57cec5SDimitry Andric Optional<CodeModel::Model> CM, 2240b57cec5SDimitry Andric CodeGenOpt::Level OL, bool JIT) 2250b57cec5SDimitry Andric : LLVMTargetMachine( 2260b57cec5SDimitry Andric T, computeDataLayout(TT), TT, CPU, FS, Options, 2270b57cec5SDimitry Andric getEffectiveRelocModel(TT, JIT, RM), 2280b57cec5SDimitry Andric getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64), 2290b57cec5SDimitry Andric OL), 230d65cd7a5SDimitry Andric TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) { 231*81ad6265SDimitry Andric // On PS4/PS5, the "return address" of a 'noreturn' call must still be within 2320b57cec5SDimitry Andric // the calling function, and TrapUnreachable is an easy way to get that. 233*81ad6265SDimitry Andric if (TT.isPS() || TT.isOSBinFormatMachO()) { 2340b57cec5SDimitry Andric this->Options.TrapUnreachable = true; 2350b57cec5SDimitry Andric this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO(); 2360b57cec5SDimitry Andric } 2370b57cec5SDimitry Andric 2380b57cec5SDimitry Andric setMachineOutliner(true); 2390b57cec5SDimitry Andric 2405ffd83dbSDimitry Andric // x86 supports the debug entry values. 2415ffd83dbSDimitry Andric setSupportsDebugEntryValues(true); 2425ffd83dbSDimitry Andric 2430b57cec5SDimitry Andric initAsmInfo(); 2440b57cec5SDimitry Andric } 2450b57cec5SDimitry Andric 2460b57cec5SDimitry Andric X86TargetMachine::~X86TargetMachine() = default; 2470b57cec5SDimitry Andric 2480b57cec5SDimitry Andric const X86Subtarget * 2490b57cec5SDimitry Andric X86TargetMachine::getSubtargetImpl(const Function &F) const { 2500b57cec5SDimitry Andric Attribute CPUAttr = F.getFnAttribute("target-cpu"); 251e8d8bef9SDimitry Andric Attribute TuneAttr = F.getFnAttribute("tune-cpu"); 2520b57cec5SDimitry Andric Attribute FSAttr = F.getFnAttribute("target-features"); 2530b57cec5SDimitry Andric 254e8d8bef9SDimitry Andric StringRef CPU = 255e8d8bef9SDimitry Andric CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU; 256e8d8bef9SDimitry Andric StringRef TuneCPU = 257e8d8bef9SDimitry Andric TuneAttr.isValid() ? TuneAttr.getValueAsString() : (StringRef)CPU; 258e8d8bef9SDimitry Andric StringRef FS = 259e8d8bef9SDimitry Andric FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS; 2600b57cec5SDimitry Andric 2610b57cec5SDimitry Andric SmallString<512> Key; 262e8d8bef9SDimitry Andric // The additions here are ordered so that the definitely short strings are 263e8d8bef9SDimitry Andric // added first so we won't exceed the small size. We append the 264e8d8bef9SDimitry Andric // much longer FS string at the end so that we only heap allocate at most 265e8d8bef9SDimitry Andric // one time. 266e8d8bef9SDimitry Andric 267e8d8bef9SDimitry Andric // Extract prefer-vector-width attribute. 268e8d8bef9SDimitry Andric unsigned PreferVectorWidthOverride = 0; 269e8d8bef9SDimitry Andric Attribute PreferVecWidthAttr = F.getFnAttribute("prefer-vector-width"); 270e8d8bef9SDimitry Andric if (PreferVecWidthAttr.isValid()) { 271e8d8bef9SDimitry Andric StringRef Val = PreferVecWidthAttr.getValueAsString(); 272e8d8bef9SDimitry Andric unsigned Width; 273e8d8bef9SDimitry Andric if (!Val.getAsInteger(0, Width)) { 274fe6060f1SDimitry Andric Key += 'p'; 275e8d8bef9SDimitry Andric Key += Val; 276e8d8bef9SDimitry Andric PreferVectorWidthOverride = Width; 277e8d8bef9SDimitry Andric } 278e8d8bef9SDimitry Andric } 279e8d8bef9SDimitry Andric 280e8d8bef9SDimitry Andric // Extract min-legal-vector-width attribute. 281e8d8bef9SDimitry Andric unsigned RequiredVectorWidth = UINT32_MAX; 282e8d8bef9SDimitry Andric Attribute MinLegalVecWidthAttr = F.getFnAttribute("min-legal-vector-width"); 283e8d8bef9SDimitry Andric if (MinLegalVecWidthAttr.isValid()) { 284e8d8bef9SDimitry Andric StringRef Val = MinLegalVecWidthAttr.getValueAsString(); 285e8d8bef9SDimitry Andric unsigned Width; 286e8d8bef9SDimitry Andric if (!Val.getAsInteger(0, Width)) { 287fe6060f1SDimitry Andric Key += 'm'; 288e8d8bef9SDimitry Andric Key += Val; 289e8d8bef9SDimitry Andric RequiredVectorWidth = Width; 290e8d8bef9SDimitry Andric } 291e8d8bef9SDimitry Andric } 292e8d8bef9SDimitry Andric 293e8d8bef9SDimitry Andric // Add CPU to the Key. 2940b57cec5SDimitry Andric Key += CPU; 295e8d8bef9SDimitry Andric 296e8d8bef9SDimitry Andric // Add tune CPU to the Key. 297e8d8bef9SDimitry Andric Key += TuneCPU; 298e8d8bef9SDimitry Andric 299e8d8bef9SDimitry Andric // Keep track of the start of the feature portion of the string. 300e8d8bef9SDimitry Andric unsigned FSStart = Key.size(); 3010b57cec5SDimitry Andric 3020b57cec5SDimitry Andric // FIXME: This is related to the code below to reset the target options, 3030b57cec5SDimitry Andric // we need to know whether or not the soft float flag is set on the 3040b57cec5SDimitry Andric // function before we can generate a subtarget. We also need to use 3050b57cec5SDimitry Andric // it as a key for the subtarget since that can be the only difference 3060b57cec5SDimitry Andric // between two functions. 307fe6060f1SDimitry Andric bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsBool(); 3080b57cec5SDimitry Andric // If the soft float attribute is set on the function turn on the soft float 3090b57cec5SDimitry Andric // subtarget feature. 3100b57cec5SDimitry Andric if (SoftFloat) 311e8d8bef9SDimitry Andric Key += FS.empty() ? "+soft-float" : "+soft-float,"; 3120b57cec5SDimitry Andric 313e8d8bef9SDimitry Andric Key += FS; 3140b57cec5SDimitry Andric 315e8d8bef9SDimitry Andric // We may have added +soft-float to the features so move the StringRef to 316e8d8bef9SDimitry Andric // point to the full string in the Key. 317e8d8bef9SDimitry Andric FS = Key.substr(FSStart); 3180b57cec5SDimitry Andric 3190b57cec5SDimitry Andric auto &I = SubtargetMap[Key]; 3200b57cec5SDimitry Andric if (!I) { 3210b57cec5SDimitry Andric // This needs to be done before we create a new subtarget since any 3220b57cec5SDimitry Andric // creation will depend on the TM and the code generation flags on the 3230b57cec5SDimitry Andric // function that reside in TargetOptions. 3240b57cec5SDimitry Andric resetTargetOptions(F); 3258bcb0991SDimitry Andric I = std::make_unique<X86Subtarget>( 326e8d8bef9SDimitry Andric TargetTriple, CPU, TuneCPU, FS, *this, 327fe6060f1SDimitry Andric MaybeAlign(F.getParent()->getOverrideStackAlignment()), 328fe6060f1SDimitry Andric PreferVectorWidthOverride, RequiredVectorWidth); 3290b57cec5SDimitry Andric } 3300b57cec5SDimitry Andric return I.get(); 3310b57cec5SDimitry Andric } 3320b57cec5SDimitry Andric 333e8d8bef9SDimitry Andric bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS, 334e8d8bef9SDimitry Andric unsigned DestAS) const { 335e8d8bef9SDimitry Andric assert(SrcAS != DestAS && "Expected different address spaces!"); 336e8d8bef9SDimitry Andric if (getPointerSize(SrcAS) != getPointerSize(DestAS)) 337e8d8bef9SDimitry Andric return false; 338e8d8bef9SDimitry Andric return SrcAS < 256 && DestAS < 256; 339e8d8bef9SDimitry Andric } 340e8d8bef9SDimitry Andric 3410b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 3420b57cec5SDimitry Andric // X86 TTI query. 3430b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 3440b57cec5SDimitry Andric 3450b57cec5SDimitry Andric TargetTransformInfo 346*81ad6265SDimitry Andric X86TargetMachine::getTargetTransformInfo(const Function &F) const { 3470b57cec5SDimitry Andric return TargetTransformInfo(X86TTIImpl(this, F)); 3480b57cec5SDimitry Andric } 3490b57cec5SDimitry Andric 3500b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 3510b57cec5SDimitry Andric // Pass Pipeline Configuration 3520b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 3530b57cec5SDimitry Andric 3540b57cec5SDimitry Andric namespace { 3550b57cec5SDimitry Andric 3560b57cec5SDimitry Andric /// X86 Code Generator Pass Configuration Options. 3570b57cec5SDimitry Andric class X86PassConfig : public TargetPassConfig { 3580b57cec5SDimitry Andric public: 3590b57cec5SDimitry Andric X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM) 3600b57cec5SDimitry Andric : TargetPassConfig(TM, PM) {} 3610b57cec5SDimitry Andric 3620b57cec5SDimitry Andric X86TargetMachine &getX86TargetMachine() const { 3630b57cec5SDimitry Andric return getTM<X86TargetMachine>(); 3640b57cec5SDimitry Andric } 3650b57cec5SDimitry Andric 3660b57cec5SDimitry Andric ScheduleDAGInstrs * 3670b57cec5SDimitry Andric createMachineScheduler(MachineSchedContext *C) const override { 3680b57cec5SDimitry Andric ScheduleDAGMILive *DAG = createGenericSchedLive(C); 3690b57cec5SDimitry Andric DAG->addMutation(createX86MacroFusionDAGMutation()); 3700b57cec5SDimitry Andric return DAG; 3710b57cec5SDimitry Andric } 3720b57cec5SDimitry Andric 3730b57cec5SDimitry Andric ScheduleDAGInstrs * 3740b57cec5SDimitry Andric createPostMachineScheduler(MachineSchedContext *C) const override { 3750b57cec5SDimitry Andric ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 3760b57cec5SDimitry Andric DAG->addMutation(createX86MacroFusionDAGMutation()); 3770b57cec5SDimitry Andric return DAG; 3780b57cec5SDimitry Andric } 3790b57cec5SDimitry Andric 3800b57cec5SDimitry Andric void addIRPasses() override; 3810b57cec5SDimitry Andric bool addInstSelector() override; 3820b57cec5SDimitry Andric bool addIRTranslator() override; 3830b57cec5SDimitry Andric bool addLegalizeMachineIR() override; 3840b57cec5SDimitry Andric bool addRegBankSelect() override; 3850b57cec5SDimitry Andric bool addGlobalInstructionSelect() override; 3860b57cec5SDimitry Andric bool addILPOpts() override; 3870b57cec5SDimitry Andric bool addPreISel() override; 3880b57cec5SDimitry Andric void addMachineSSAOptimization() override; 3890b57cec5SDimitry Andric void addPreRegAlloc() override; 390fe6060f1SDimitry Andric bool addPostFastRegAllocRewrite() override; 3910b57cec5SDimitry Andric void addPostRegAlloc() override; 3920b57cec5SDimitry Andric void addPreEmitPass() override; 3930b57cec5SDimitry Andric void addPreEmitPass2() override; 3940b57cec5SDimitry Andric void addPreSched2() override; 395*81ad6265SDimitry Andric bool addRegAssignAndRewriteOptimized() override; 3960b57cec5SDimitry Andric 3970b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 3980b57cec5SDimitry Andric }; 3990b57cec5SDimitry Andric 4000b57cec5SDimitry Andric class X86ExecutionDomainFix : public ExecutionDomainFix { 4010b57cec5SDimitry Andric public: 4020b57cec5SDimitry Andric static char ID; 4030b57cec5SDimitry Andric X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {} 4040b57cec5SDimitry Andric StringRef getPassName() const override { 4050b57cec5SDimitry Andric return "X86 Execution Dependency Fix"; 4060b57cec5SDimitry Andric } 4070b57cec5SDimitry Andric }; 4080b57cec5SDimitry Andric char X86ExecutionDomainFix::ID; 4090b57cec5SDimitry Andric 4100b57cec5SDimitry Andric } // end anonymous namespace 4110b57cec5SDimitry Andric 4120b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix", 4130b57cec5SDimitry Andric "X86 Execution Domain Fix", false, false) 4140b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) 4150b57cec5SDimitry Andric INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix", 4160b57cec5SDimitry Andric "X86 Execution Domain Fix", false, false) 4170b57cec5SDimitry Andric 4180b57cec5SDimitry Andric TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) { 4190b57cec5SDimitry Andric return new X86PassConfig(*this, PM); 4200b57cec5SDimitry Andric } 4210b57cec5SDimitry Andric 4220b57cec5SDimitry Andric void X86PassConfig::addIRPasses() { 4230b57cec5SDimitry Andric addPass(createAtomicExpandPass()); 424fe6060f1SDimitry Andric 425fe6060f1SDimitry Andric // We add both pass anyway and when these two passes run, we skip the pass 426fe6060f1SDimitry Andric // based on the option level and option attribute. 427fe6060f1SDimitry Andric addPass(createX86LowerAMXIntrinsicsPass()); 428e8d8bef9SDimitry Andric addPass(createX86LowerAMXTypePass()); 4290b57cec5SDimitry Andric 4300b57cec5SDimitry Andric TargetPassConfig::addIRPasses(); 4310b57cec5SDimitry Andric 4325ffd83dbSDimitry Andric if (TM->getOptLevel() != CodeGenOpt::None) { 4330b57cec5SDimitry Andric addPass(createInterleavedAccessPass()); 4345ffd83dbSDimitry Andric addPass(createX86PartialReductionPass()); 4355ffd83dbSDimitry Andric } 4360b57cec5SDimitry Andric 4370b57cec5SDimitry Andric // Add passes that handle indirect branch removal and insertion of a retpoline 4380b57cec5SDimitry Andric // thunk. These will be a no-op unless a function subtarget has the retpoline 4390b57cec5SDimitry Andric // feature enabled. 4400b57cec5SDimitry Andric addPass(createIndirectBrExpandPass()); 441480093f4SDimitry Andric 442480093f4SDimitry Andric // Add Control Flow Guard checks. 443480093f4SDimitry Andric const Triple &TT = TM->getTargetTriple(); 444480093f4SDimitry Andric if (TT.isOSWindows()) { 445480093f4SDimitry Andric if (TT.getArch() == Triple::x86_64) { 446480093f4SDimitry Andric addPass(createCFGuardDispatchPass()); 447480093f4SDimitry Andric } else { 448480093f4SDimitry Andric addPass(createCFGuardCheckPass()); 449480093f4SDimitry Andric } 450480093f4SDimitry Andric } 451*81ad6265SDimitry Andric 452*81ad6265SDimitry Andric if (TM->Options.JMCInstrument) 453*81ad6265SDimitry Andric addPass(createJMCInstrumenterPass()); 4540b57cec5SDimitry Andric } 4550b57cec5SDimitry Andric 4560b57cec5SDimitry Andric bool X86PassConfig::addInstSelector() { 4570b57cec5SDimitry Andric // Install an instruction selector. 4580b57cec5SDimitry Andric addPass(createX86ISelDag(getX86TargetMachine(), getOptLevel())); 4590b57cec5SDimitry Andric 4600b57cec5SDimitry Andric // For ELF, cleanup any local-dynamic TLS accesses. 4610b57cec5SDimitry Andric if (TM->getTargetTriple().isOSBinFormatELF() && 4620b57cec5SDimitry Andric getOptLevel() != CodeGenOpt::None) 4630b57cec5SDimitry Andric addPass(createCleanupLocalDynamicTLSPass()); 4640b57cec5SDimitry Andric 4650b57cec5SDimitry Andric addPass(createX86GlobalBaseRegPass()); 4660b57cec5SDimitry Andric return false; 4670b57cec5SDimitry Andric } 4680b57cec5SDimitry Andric 4690b57cec5SDimitry Andric bool X86PassConfig::addIRTranslator() { 470e8d8bef9SDimitry Andric addPass(new IRTranslator(getOptLevel())); 4710b57cec5SDimitry Andric return false; 4720b57cec5SDimitry Andric } 4730b57cec5SDimitry Andric 4740b57cec5SDimitry Andric bool X86PassConfig::addLegalizeMachineIR() { 4750b57cec5SDimitry Andric addPass(new Legalizer()); 4760b57cec5SDimitry Andric return false; 4770b57cec5SDimitry Andric } 4780b57cec5SDimitry Andric 4790b57cec5SDimitry Andric bool X86PassConfig::addRegBankSelect() { 4800b57cec5SDimitry Andric addPass(new RegBankSelect()); 4810b57cec5SDimitry Andric return false; 4820b57cec5SDimitry Andric } 4830b57cec5SDimitry Andric 4840b57cec5SDimitry Andric bool X86PassConfig::addGlobalInstructionSelect() { 485fe6060f1SDimitry Andric addPass(new InstructionSelect(getOptLevel())); 4860b57cec5SDimitry Andric return false; 4870b57cec5SDimitry Andric } 4880b57cec5SDimitry Andric 4890b57cec5SDimitry Andric bool X86PassConfig::addILPOpts() { 4900b57cec5SDimitry Andric addPass(&EarlyIfConverterID); 4910b57cec5SDimitry Andric if (EnableMachineCombinerPass) 4920b57cec5SDimitry Andric addPass(&MachineCombinerID); 4930b57cec5SDimitry Andric addPass(createX86CmovConverterPass()); 4940b57cec5SDimitry Andric return true; 4950b57cec5SDimitry Andric } 4960b57cec5SDimitry Andric 4970b57cec5SDimitry Andric bool X86PassConfig::addPreISel() { 4980b57cec5SDimitry Andric // Only add this pass for 32-bit x86 Windows. 4990b57cec5SDimitry Andric const Triple &TT = TM->getTargetTriple(); 5000b57cec5SDimitry Andric if (TT.isOSWindows() && TT.getArch() == Triple::x86) 5010b57cec5SDimitry Andric addPass(createX86WinEHStatePass()); 5020b57cec5SDimitry Andric return true; 5030b57cec5SDimitry Andric } 5040b57cec5SDimitry Andric 5050b57cec5SDimitry Andric void X86PassConfig::addPreRegAlloc() { 5060b57cec5SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 5070b57cec5SDimitry Andric addPass(&LiveRangeShrinkID); 5080b57cec5SDimitry Andric addPass(createX86FixupSetCC()); 5090b57cec5SDimitry Andric addPass(createX86OptimizeLEAs()); 5100b57cec5SDimitry Andric addPass(createX86CallFrameOptimization()); 5110b57cec5SDimitry Andric addPass(createX86AvoidStoreForwardingBlocks()); 5120b57cec5SDimitry Andric } 5130b57cec5SDimitry Andric 5140b57cec5SDimitry Andric addPass(createX86SpeculativeLoadHardeningPass()); 5150b57cec5SDimitry Andric addPass(createX86FlagsCopyLoweringPass()); 516349cc55cSDimitry Andric addPass(createX86DynAllocaExpander()); 517e8d8bef9SDimitry Andric 518*81ad6265SDimitry Andric if (getOptLevel() != CodeGenOpt::None) 519e8d8bef9SDimitry Andric addPass(createX86PreTileConfigPass()); 520*81ad6265SDimitry Andric else 521*81ad6265SDimitry Andric addPass(createX86FastPreTileConfigPass()); 522e8d8bef9SDimitry Andric } 523e8d8bef9SDimitry Andric 5240b57cec5SDimitry Andric void X86PassConfig::addMachineSSAOptimization() { 5250b57cec5SDimitry Andric addPass(createX86DomainReassignmentPass()); 5260b57cec5SDimitry Andric TargetPassConfig::addMachineSSAOptimization(); 5270b57cec5SDimitry Andric } 5280b57cec5SDimitry Andric 5290b57cec5SDimitry Andric void X86PassConfig::addPostRegAlloc() { 530fe6060f1SDimitry Andric addPass(createX86LowerTileCopyPass()); 5310b57cec5SDimitry Andric addPass(createX86FloatingPointStackifierPass()); 5325ffd83dbSDimitry Andric // When -O0 is enabled, the Load Value Injection Hardening pass will fall back 5335ffd83dbSDimitry Andric // to using the Speculative Execution Side Effect Suppression pass for 5345ffd83dbSDimitry Andric // mitigation. This is to prevent slow downs due to 5355ffd83dbSDimitry Andric // analyses needed by the LVIHardening pass when compiling at -O0. 5360946e70aSDimitry Andric if (getOptLevel() != CodeGenOpt::None) 5370946e70aSDimitry Andric addPass(createX86LoadValueInjectionLoadHardeningPass()); 5380b57cec5SDimitry Andric } 5390b57cec5SDimitry Andric 5400b57cec5SDimitry Andric void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); } 5410b57cec5SDimitry Andric 5420b57cec5SDimitry Andric void X86PassConfig::addPreEmitPass() { 5430b57cec5SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 5440b57cec5SDimitry Andric addPass(new X86ExecutionDomainFix()); 5450b57cec5SDimitry Andric addPass(createBreakFalseDeps()); 5460b57cec5SDimitry Andric } 5470b57cec5SDimitry Andric 5480b57cec5SDimitry Andric addPass(createX86IndirectBranchTrackingPass()); 5490b57cec5SDimitry Andric 5500b57cec5SDimitry Andric addPass(createX86IssueVZeroUpperPass()); 5510b57cec5SDimitry Andric 5520b57cec5SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 5530b57cec5SDimitry Andric addPass(createX86FixupBWInsts()); 5540b57cec5SDimitry Andric addPass(createX86PadShortFunctions()); 5550b57cec5SDimitry Andric addPass(createX86FixupLEAs()); 5560b57cec5SDimitry Andric } 5575ffd83dbSDimitry Andric addPass(createX86EvexToVexInsts()); 5580b57cec5SDimitry Andric addPass(createX86DiscriminateMemOpsPass()); 5590b57cec5SDimitry Andric addPass(createX86InsertPrefetchPass()); 5605ffd83dbSDimitry Andric addPass(createX86InsertX87waitPass()); 5610b57cec5SDimitry Andric } 5620b57cec5SDimitry Andric 5630b57cec5SDimitry Andric void X86PassConfig::addPreEmitPass2() { 5648bcb0991SDimitry Andric const Triple &TT = TM->getTargetTriple(); 5658bcb0991SDimitry Andric const MCAsmInfo *MAI = TM->getMCAsmInfo(); 5668bcb0991SDimitry Andric 5675ffd83dbSDimitry Andric // The X86 Speculative Execution Pass must run after all control 5685ffd83dbSDimitry Andric // flow graph modifying passes. As a result it was listed to run right before 5695ffd83dbSDimitry Andric // the X86 Retpoline Thunks pass. The reason it must run after control flow 5705ffd83dbSDimitry Andric // graph modifications is that the model of LFENCE in LLVM has to be updated 5715ffd83dbSDimitry Andric // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the 5725ffd83dbSDimitry Andric // placement of this pass was hand checked to ensure that the subsequent 5735ffd83dbSDimitry Andric // passes don't move the code around the LFENCEs in a way that will hurt the 5745ffd83dbSDimitry Andric // correctness of this pass. This placement has been shown to work based on 5755ffd83dbSDimitry Andric // hand inspection of the codegen output. 5765ffd83dbSDimitry Andric addPass(createX86SpeculativeExecutionSideEffectSuppression()); 5770946e70aSDimitry Andric addPass(createX86IndirectThunksPass()); 5788bcb0991SDimitry Andric 5798bcb0991SDimitry Andric // Insert extra int3 instructions after trailing call instructions to avoid 5808bcb0991SDimitry Andric // issues in the unwinder. 5818bcb0991SDimitry Andric if (TT.isOSWindows() && TT.getArch() == Triple::x86_64) 5828bcb0991SDimitry Andric addPass(createX86AvoidTrailingCallPass()); 5838bcb0991SDimitry Andric 5840b57cec5SDimitry Andric // Verify basic block incoming and outgoing cfa offset and register values and 5850b57cec5SDimitry Andric // correct CFA calculation rule where needed by inserting appropriate CFI 5860b57cec5SDimitry Andric // instructions. 5870b57cec5SDimitry Andric if (!TT.isOSDarwin() && 5880b57cec5SDimitry Andric (!TT.isOSWindows() || 5890b57cec5SDimitry Andric MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) 5900b57cec5SDimitry Andric addPass(createCFIInstrInserter()); 591fe6060f1SDimitry Andric 592fe6060f1SDimitry Andric if (TT.isOSWindows()) { 593480093f4SDimitry Andric // Identify valid longjmp targets for Windows Control Flow Guard. 594480093f4SDimitry Andric addPass(createCFGuardLongjmpPass()); 595fe6060f1SDimitry Andric // Identify valid eh continuation targets for Windows EHCont Guard. 596fe6060f1SDimitry Andric addPass(createEHContGuardCatchretPass()); 597fe6060f1SDimitry Andric } 5980946e70aSDimitry Andric addPass(createX86LoadValueInjectionRetHardeningPass()); 599349cc55cSDimitry Andric 600349cc55cSDimitry Andric // Insert pseudo probe annotation for callsite profiling 601349cc55cSDimitry Andric addPass(createPseudoProbeInserter()); 6020eae32dcSDimitry Andric 6030eae32dcSDimitry Andric // On Darwin platforms, BLR_RVMARKER pseudo instructions are lowered to 6040eae32dcSDimitry Andric // bundles. 6050eae32dcSDimitry Andric if (TT.isOSDarwin()) 6060eae32dcSDimitry Andric addPass(createUnpackMachineBundles([](const MachineFunction &MF) { 6070eae32dcSDimitry Andric // Only run bundle expansion if there are relevant ObjC runtime functions 6080eae32dcSDimitry Andric // present in the module. 6090eae32dcSDimitry Andric const Function &F = MF.getFunction(); 6100eae32dcSDimitry Andric const Module *M = F.getParent(); 6110eae32dcSDimitry Andric return M->getFunction("objc_retainAutoreleasedReturnValue") || 6120eae32dcSDimitry Andric M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"); 6130eae32dcSDimitry Andric })); 6140b57cec5SDimitry Andric } 6150b57cec5SDimitry Andric 616fe6060f1SDimitry Andric bool X86PassConfig::addPostFastRegAllocRewrite() { 617fe6060f1SDimitry Andric addPass(createX86FastTileConfigPass()); 618fe6060f1SDimitry Andric return true; 619fe6060f1SDimitry Andric } 620fe6060f1SDimitry Andric 6210b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const { 6220b57cec5SDimitry Andric return getStandardCSEConfigForOpt(TM->getOptLevel()); 6230b57cec5SDimitry Andric } 624*81ad6265SDimitry Andric 625*81ad6265SDimitry Andric static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, 626*81ad6265SDimitry Andric const TargetRegisterClass &RC) { 627*81ad6265SDimitry Andric return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(&RC); 628*81ad6265SDimitry Andric } 629*81ad6265SDimitry Andric 630*81ad6265SDimitry Andric bool X86PassConfig::addRegAssignAndRewriteOptimized() { 631*81ad6265SDimitry Andric // Don't support tile RA when RA is specified by command line "-regalloc". 632*81ad6265SDimitry Andric if (!isCustomizedRegAlloc() && EnableTileRAPass) { 633*81ad6265SDimitry Andric // Allocate tile register first. 634*81ad6265SDimitry Andric addPass(createGreedyRegisterAllocator(onlyAllocateTileRegisters)); 635*81ad6265SDimitry Andric addPass(createX86TileConfigPass()); 636*81ad6265SDimitry Andric } 637*81ad6265SDimitry Andric return TargetPassConfig::addRegAssignAndRewriteOptimized(); 638*81ad6265SDimitry Andric } 639