xref: /freebsd/contrib/llvm-project/llvm/lib/Target/X86/X86TargetMachine.cpp (revision 81ad626541db97eb356e2c1d4a20eb2a26a766ab)
10b57cec5SDimitry Andric //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file defines the X86 specific subclass of TargetMachine.
100b57cec5SDimitry Andric //
110b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
120b57cec5SDimitry Andric 
130b57cec5SDimitry Andric #include "X86TargetMachine.h"
140b57cec5SDimitry Andric #include "MCTargetDesc/X86MCTargetDesc.h"
150b57cec5SDimitry Andric #include "TargetInfo/X86TargetInfo.h"
160b57cec5SDimitry Andric #include "X86.h"
170b57cec5SDimitry Andric #include "X86CallLowering.h"
180b57cec5SDimitry Andric #include "X86LegalizerInfo.h"
190b57cec5SDimitry Andric #include "X86MacroFusion.h"
200b57cec5SDimitry Andric #include "X86Subtarget.h"
210b57cec5SDimitry Andric #include "X86TargetObjectFile.h"
220b57cec5SDimitry Andric #include "X86TargetTransformInfo.h"
230b57cec5SDimitry Andric #include "llvm/ADT/Optional.h"
240b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h"
250b57cec5SDimitry Andric #include "llvm/ADT/SmallString.h"
260b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
270b57cec5SDimitry Andric #include "llvm/ADT/Triple.h"
280b57cec5SDimitry Andric #include "llvm/Analysis/TargetTransformInfo.h"
290b57cec5SDimitry Andric #include "llvm/CodeGen/ExecutionDomainFix.h"
30*81ad6265SDimitry Andric #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
310b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/CallLowering.h"
320b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
330b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
34*81ad6265SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
350b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/Legalizer.h"
360b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
370b57cec5SDimitry Andric #include "llvm/CodeGen/MachineScheduler.h"
380b57cec5SDimitry Andric #include "llvm/CodeGen/Passes.h"
39*81ad6265SDimitry Andric #include "llvm/CodeGen/RegAllocRegistry.h"
400b57cec5SDimitry Andric #include "llvm/CodeGen/TargetPassConfig.h"
410b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
420b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
430b57cec5SDimitry Andric #include "llvm/IR/Function.h"
440b57cec5SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
45349cc55cSDimitry Andric #include "llvm/MC/TargetRegistry.h"
460b57cec5SDimitry Andric #include "llvm/Pass.h"
470b57cec5SDimitry Andric #include "llvm/Support/CodeGen.h"
480b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
490b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
500b57cec5SDimitry Andric #include "llvm/Target/TargetLoweringObjectFile.h"
510b57cec5SDimitry Andric #include "llvm/Target/TargetOptions.h"
52480093f4SDimitry Andric #include "llvm/Transforms/CFGuard.h"
530b57cec5SDimitry Andric #include <memory>
540b57cec5SDimitry Andric #include <string>
550b57cec5SDimitry Andric 
560b57cec5SDimitry Andric using namespace llvm;
570b57cec5SDimitry Andric 
580b57cec5SDimitry Andric static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner",
590b57cec5SDimitry Andric                                cl::desc("Enable the machine combiner pass"),
600b57cec5SDimitry Andric                                cl::init(true), cl::Hidden);
610b57cec5SDimitry Andric 
62*81ad6265SDimitry Andric static cl::opt<bool>
63*81ad6265SDimitry Andric     EnableTileRAPass("x86-tile-ra",
64*81ad6265SDimitry Andric                      cl::desc("Enable the tile register allocation pass"),
65*81ad6265SDimitry Andric                      cl::init(true), cl::Hidden);
66*81ad6265SDimitry Andric 
67480093f4SDimitry Andric extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
680b57cec5SDimitry Andric   // Register the target.
690b57cec5SDimitry Andric   RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target());
700b57cec5SDimitry Andric   RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target());
710b57cec5SDimitry Andric 
720b57cec5SDimitry Andric   PassRegistry &PR = *PassRegistry::getPassRegistry();
73fe6060f1SDimitry Andric   initializeX86LowerAMXIntrinsicsLegacyPassPass(PR);
74e8d8bef9SDimitry Andric   initializeX86LowerAMXTypeLegacyPassPass(PR);
75fe6060f1SDimitry Andric   initializeX86PreAMXConfigPassPass(PR);
76*81ad6265SDimitry Andric   initializeX86PreTileConfigPass(PR);
770b57cec5SDimitry Andric   initializeGlobalISel(PR);
780b57cec5SDimitry Andric   initializeWinEHStatePassPass(PR);
790b57cec5SDimitry Andric   initializeFixupBWInstPassPass(PR);
800b57cec5SDimitry Andric   initializeEvexToVexInstPassPass(PR);
810b57cec5SDimitry Andric   initializeFixupLEAPassPass(PR);
820b57cec5SDimitry Andric   initializeFPSPass(PR);
835ffd83dbSDimitry Andric   initializeX86FixupSetCCPassPass(PR);
840b57cec5SDimitry Andric   initializeX86CallFrameOptimizationPass(PR);
850b57cec5SDimitry Andric   initializeX86CmovConverterPassPass(PR);
86e8d8bef9SDimitry Andric   initializeX86TileConfigPass(PR);
87*81ad6265SDimitry Andric   initializeX86FastPreTileConfigPass(PR);
88fe6060f1SDimitry Andric   initializeX86FastTileConfigPass(PR);
89fe6060f1SDimitry Andric   initializeX86LowerTileCopyPass(PR);
900b57cec5SDimitry Andric   initializeX86ExpandPseudoPass(PR);
910b57cec5SDimitry Andric   initializeX86ExecutionDomainFixPass(PR);
920b57cec5SDimitry Andric   initializeX86DomainReassignmentPass(PR);
930b57cec5SDimitry Andric   initializeX86AvoidSFBPassPass(PR);
945ffd83dbSDimitry Andric   initializeX86AvoidTrailingCallPassPass(PR);
950b57cec5SDimitry Andric   initializeX86SpeculativeLoadHardeningPassPass(PR);
965ffd83dbSDimitry Andric   initializeX86SpeculativeExecutionSideEffectSuppressionPass(PR);
970b57cec5SDimitry Andric   initializeX86FlagsCopyLoweringPassPass(PR);
980946e70aSDimitry Andric   initializeX86LoadValueInjectionLoadHardeningPassPass(PR);
990946e70aSDimitry Andric   initializeX86LoadValueInjectionRetHardeningPassPass(PR);
1008bcb0991SDimitry Andric   initializeX86OptimizeLEAPassPass(PR);
1015ffd83dbSDimitry Andric   initializeX86PartialReductionPass(PR);
102e8d8bef9SDimitry Andric   initializePseudoProbeInserterPass(PR);
1030b57cec5SDimitry Andric }
1040b57cec5SDimitry Andric 
1050b57cec5SDimitry Andric static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
1060b57cec5SDimitry Andric   if (TT.isOSBinFormatMachO()) {
1070b57cec5SDimitry Andric     if (TT.getArch() == Triple::x86_64)
1088bcb0991SDimitry Andric       return std::make_unique<X86_64MachoTargetObjectFile>();
1098bcb0991SDimitry Andric     return std::make_unique<TargetLoweringObjectFileMachO>();
1100b57cec5SDimitry Andric   }
1110b57cec5SDimitry Andric 
1120b57cec5SDimitry Andric   if (TT.isOSBinFormatCOFF())
1138bcb0991SDimitry Andric     return std::make_unique<TargetLoweringObjectFileCOFF>();
1145ffd83dbSDimitry Andric   return std::make_unique<X86ELFTargetObjectFile>();
1150b57cec5SDimitry Andric }
1160b57cec5SDimitry Andric 
1170b57cec5SDimitry Andric static std::string computeDataLayout(const Triple &TT) {
1180b57cec5SDimitry Andric   // X86 is little endian
1190b57cec5SDimitry Andric   std::string Ret = "e";
1200b57cec5SDimitry Andric 
1210b57cec5SDimitry Andric   Ret += DataLayout::getManglingComponent(TT);
1220b57cec5SDimitry Andric   // X86 and x32 have 32 bit pointers.
123fe6060f1SDimitry Andric   if (!TT.isArch64Bit() || TT.isX32() || TT.isOSNaCl())
1240b57cec5SDimitry Andric     Ret += "-p:32:32";
1250b57cec5SDimitry Andric 
1268bcb0991SDimitry Andric   // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers.
1278bcb0991SDimitry Andric   Ret += "-p270:32:32-p271:32:32-p272:64:64";
1288bcb0991SDimitry Andric 
1290b57cec5SDimitry Andric   // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32.
1300b57cec5SDimitry Andric   if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl())
1310b57cec5SDimitry Andric     Ret += "-i64:64";
1320b57cec5SDimitry Andric   else if (TT.isOSIAMCU())
1330b57cec5SDimitry Andric     Ret += "-i64:32-f64:32";
1340b57cec5SDimitry Andric   else
1350b57cec5SDimitry Andric     Ret += "-f64:32:64";
1360b57cec5SDimitry Andric 
1370b57cec5SDimitry Andric   // Some ABIs align long double to 128 bits, others to 32.
1380b57cec5SDimitry Andric   if (TT.isOSNaCl() || TT.isOSIAMCU())
1390b57cec5SDimitry Andric     ; // No f80
14004eeddc0SDimitry Andric   else if (TT.isArch64Bit() || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment())
1410b57cec5SDimitry Andric     Ret += "-f80:128";
1420b57cec5SDimitry Andric   else
1430b57cec5SDimitry Andric     Ret += "-f80:32";
1440b57cec5SDimitry Andric 
1450b57cec5SDimitry Andric   if (TT.isOSIAMCU())
1460b57cec5SDimitry Andric     Ret += "-f128:32";
1470b57cec5SDimitry Andric 
1480b57cec5SDimitry Andric   // The registers can hold 8, 16, 32 or, in x86-64, 64 bits.
1490b57cec5SDimitry Andric   if (TT.isArch64Bit())
1500b57cec5SDimitry Andric     Ret += "-n8:16:32:64";
1510b57cec5SDimitry Andric   else
1520b57cec5SDimitry Andric     Ret += "-n8:16:32";
1530b57cec5SDimitry Andric 
1540b57cec5SDimitry Andric   // The stack is aligned to 32 bits on some ABIs and 128 bits on others.
1550b57cec5SDimitry Andric   if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU())
1560b57cec5SDimitry Andric     Ret += "-a:0:32-S32";
1570b57cec5SDimitry Andric   else
1580b57cec5SDimitry Andric     Ret += "-S128";
1590b57cec5SDimitry Andric 
1600b57cec5SDimitry Andric   return Ret;
1610b57cec5SDimitry Andric }
1620b57cec5SDimitry Andric 
1630b57cec5SDimitry Andric static Reloc::Model getEffectiveRelocModel(const Triple &TT,
1640b57cec5SDimitry Andric                                            bool JIT,
1650b57cec5SDimitry Andric                                            Optional<Reloc::Model> RM) {
1660b57cec5SDimitry Andric   bool is64Bit = TT.getArch() == Triple::x86_64;
167*81ad6265SDimitry Andric   if (!RM) {
1680b57cec5SDimitry Andric     // JIT codegen should use static relocations by default, since it's
1690b57cec5SDimitry Andric     // typically executed in process and not relocatable.
1700b57cec5SDimitry Andric     if (JIT)
1710b57cec5SDimitry Andric       return Reloc::Static;
1720b57cec5SDimitry Andric 
1730b57cec5SDimitry Andric     // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
1740b57cec5SDimitry Andric     // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
1750b57cec5SDimitry Andric     // use static relocation model by default.
1760b57cec5SDimitry Andric     if (TT.isOSDarwin()) {
1770b57cec5SDimitry Andric       if (is64Bit)
1780b57cec5SDimitry Andric         return Reloc::PIC_;
1790b57cec5SDimitry Andric       return Reloc::DynamicNoPIC;
1800b57cec5SDimitry Andric     }
1810b57cec5SDimitry Andric     if (TT.isOSWindows() && is64Bit)
1820b57cec5SDimitry Andric       return Reloc::PIC_;
1830b57cec5SDimitry Andric     return Reloc::Static;
1840b57cec5SDimitry Andric   }
1850b57cec5SDimitry Andric 
1860b57cec5SDimitry Andric   // ELF and X86-64 don't have a distinct DynamicNoPIC model.  DynamicNoPIC
1870b57cec5SDimitry Andric   // is defined as a model for code which may be used in static or dynamic
1880b57cec5SDimitry Andric   // executables but not necessarily a shared library. On X86-32 we just
1890b57cec5SDimitry Andric   // compile in -static mode, in x86-64 we use PIC.
1900b57cec5SDimitry Andric   if (*RM == Reloc::DynamicNoPIC) {
1910b57cec5SDimitry Andric     if (is64Bit)
1920b57cec5SDimitry Andric       return Reloc::PIC_;
1930b57cec5SDimitry Andric     if (!TT.isOSDarwin())
1940b57cec5SDimitry Andric       return Reloc::Static;
1950b57cec5SDimitry Andric   }
1960b57cec5SDimitry Andric 
1970b57cec5SDimitry Andric   // If we are on Darwin, disallow static relocation model in X86-64 mode, since
1980b57cec5SDimitry Andric   // the Mach-O file format doesn't support it.
1990b57cec5SDimitry Andric   if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit)
2000b57cec5SDimitry Andric     return Reloc::PIC_;
2010b57cec5SDimitry Andric 
2020b57cec5SDimitry Andric   return *RM;
2030b57cec5SDimitry Andric }
2040b57cec5SDimitry Andric 
2050b57cec5SDimitry Andric static CodeModel::Model getEffectiveX86CodeModel(Optional<CodeModel::Model> CM,
2060b57cec5SDimitry Andric                                                  bool JIT, bool Is64Bit) {
2070b57cec5SDimitry Andric   if (CM) {
2080b57cec5SDimitry Andric     if (*CM == CodeModel::Tiny)
2090b57cec5SDimitry Andric       report_fatal_error("Target does not support the tiny CodeModel", false);
2100b57cec5SDimitry Andric     return *CM;
2110b57cec5SDimitry Andric   }
2120b57cec5SDimitry Andric   if (JIT)
2130b57cec5SDimitry Andric     return Is64Bit ? CodeModel::Large : CodeModel::Small;
2140b57cec5SDimitry Andric   return CodeModel::Small;
2150b57cec5SDimitry Andric }
2160b57cec5SDimitry Andric 
2170b57cec5SDimitry Andric /// Create an X86 target.
2180b57cec5SDimitry Andric ///
2190b57cec5SDimitry Andric X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
2200b57cec5SDimitry Andric                                    StringRef CPU, StringRef FS,
2210b57cec5SDimitry Andric                                    const TargetOptions &Options,
2220b57cec5SDimitry Andric                                    Optional<Reloc::Model> RM,
2230b57cec5SDimitry Andric                                    Optional<CodeModel::Model> CM,
2240b57cec5SDimitry Andric                                    CodeGenOpt::Level OL, bool JIT)
2250b57cec5SDimitry Andric     : LLVMTargetMachine(
2260b57cec5SDimitry Andric           T, computeDataLayout(TT), TT, CPU, FS, Options,
2270b57cec5SDimitry Andric           getEffectiveRelocModel(TT, JIT, RM),
2280b57cec5SDimitry Andric           getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64),
2290b57cec5SDimitry Andric           OL),
230d65cd7a5SDimitry Andric       TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) {
231*81ad6265SDimitry Andric   // On PS4/PS5, the "return address" of a 'noreturn' call must still be within
2320b57cec5SDimitry Andric   // the calling function, and TrapUnreachable is an easy way to get that.
233*81ad6265SDimitry Andric   if (TT.isPS() || TT.isOSBinFormatMachO()) {
2340b57cec5SDimitry Andric     this->Options.TrapUnreachable = true;
2350b57cec5SDimitry Andric     this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
2360b57cec5SDimitry Andric   }
2370b57cec5SDimitry Andric 
2380b57cec5SDimitry Andric   setMachineOutliner(true);
2390b57cec5SDimitry Andric 
2405ffd83dbSDimitry Andric   // x86 supports the debug entry values.
2415ffd83dbSDimitry Andric   setSupportsDebugEntryValues(true);
2425ffd83dbSDimitry Andric 
2430b57cec5SDimitry Andric   initAsmInfo();
2440b57cec5SDimitry Andric }
2450b57cec5SDimitry Andric 
2460b57cec5SDimitry Andric X86TargetMachine::~X86TargetMachine() = default;
2470b57cec5SDimitry Andric 
2480b57cec5SDimitry Andric const X86Subtarget *
2490b57cec5SDimitry Andric X86TargetMachine::getSubtargetImpl(const Function &F) const {
2500b57cec5SDimitry Andric   Attribute CPUAttr = F.getFnAttribute("target-cpu");
251e8d8bef9SDimitry Andric   Attribute TuneAttr = F.getFnAttribute("tune-cpu");
2520b57cec5SDimitry Andric   Attribute FSAttr = F.getFnAttribute("target-features");
2530b57cec5SDimitry Andric 
254e8d8bef9SDimitry Andric   StringRef CPU =
255e8d8bef9SDimitry Andric       CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU;
256e8d8bef9SDimitry Andric   StringRef TuneCPU =
257e8d8bef9SDimitry Andric       TuneAttr.isValid() ? TuneAttr.getValueAsString() : (StringRef)CPU;
258e8d8bef9SDimitry Andric   StringRef FS =
259e8d8bef9SDimitry Andric       FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS;
2600b57cec5SDimitry Andric 
2610b57cec5SDimitry Andric   SmallString<512> Key;
262e8d8bef9SDimitry Andric   // The additions here are ordered so that the definitely short strings are
263e8d8bef9SDimitry Andric   // added first so we won't exceed the small size. We append the
264e8d8bef9SDimitry Andric   // much longer FS string at the end so that we only heap allocate at most
265e8d8bef9SDimitry Andric   // one time.
266e8d8bef9SDimitry Andric 
267e8d8bef9SDimitry Andric   // Extract prefer-vector-width attribute.
268e8d8bef9SDimitry Andric   unsigned PreferVectorWidthOverride = 0;
269e8d8bef9SDimitry Andric   Attribute PreferVecWidthAttr = F.getFnAttribute("prefer-vector-width");
270e8d8bef9SDimitry Andric   if (PreferVecWidthAttr.isValid()) {
271e8d8bef9SDimitry Andric     StringRef Val = PreferVecWidthAttr.getValueAsString();
272e8d8bef9SDimitry Andric     unsigned Width;
273e8d8bef9SDimitry Andric     if (!Val.getAsInteger(0, Width)) {
274fe6060f1SDimitry Andric       Key += 'p';
275e8d8bef9SDimitry Andric       Key += Val;
276e8d8bef9SDimitry Andric       PreferVectorWidthOverride = Width;
277e8d8bef9SDimitry Andric     }
278e8d8bef9SDimitry Andric   }
279e8d8bef9SDimitry Andric 
280e8d8bef9SDimitry Andric   // Extract min-legal-vector-width attribute.
281e8d8bef9SDimitry Andric   unsigned RequiredVectorWidth = UINT32_MAX;
282e8d8bef9SDimitry Andric   Attribute MinLegalVecWidthAttr = F.getFnAttribute("min-legal-vector-width");
283e8d8bef9SDimitry Andric   if (MinLegalVecWidthAttr.isValid()) {
284e8d8bef9SDimitry Andric     StringRef Val = MinLegalVecWidthAttr.getValueAsString();
285e8d8bef9SDimitry Andric     unsigned Width;
286e8d8bef9SDimitry Andric     if (!Val.getAsInteger(0, Width)) {
287fe6060f1SDimitry Andric       Key += 'm';
288e8d8bef9SDimitry Andric       Key += Val;
289e8d8bef9SDimitry Andric       RequiredVectorWidth = Width;
290e8d8bef9SDimitry Andric     }
291e8d8bef9SDimitry Andric   }
292e8d8bef9SDimitry Andric 
293e8d8bef9SDimitry Andric   // Add CPU to the Key.
2940b57cec5SDimitry Andric   Key += CPU;
295e8d8bef9SDimitry Andric 
296e8d8bef9SDimitry Andric   // Add tune CPU to the Key.
297e8d8bef9SDimitry Andric   Key += TuneCPU;
298e8d8bef9SDimitry Andric 
299e8d8bef9SDimitry Andric   // Keep track of the start of the feature portion of the string.
300e8d8bef9SDimitry Andric   unsigned FSStart = Key.size();
3010b57cec5SDimitry Andric 
3020b57cec5SDimitry Andric   // FIXME: This is related to the code below to reset the target options,
3030b57cec5SDimitry Andric   // we need to know whether or not the soft float flag is set on the
3040b57cec5SDimitry Andric   // function before we can generate a subtarget. We also need to use
3050b57cec5SDimitry Andric   // it as a key for the subtarget since that can be the only difference
3060b57cec5SDimitry Andric   // between two functions.
307fe6060f1SDimitry Andric   bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsBool();
3080b57cec5SDimitry Andric   // If the soft float attribute is set on the function turn on the soft float
3090b57cec5SDimitry Andric   // subtarget feature.
3100b57cec5SDimitry Andric   if (SoftFloat)
311e8d8bef9SDimitry Andric     Key += FS.empty() ? "+soft-float" : "+soft-float,";
3120b57cec5SDimitry Andric 
313e8d8bef9SDimitry Andric   Key += FS;
3140b57cec5SDimitry Andric 
315e8d8bef9SDimitry Andric   // We may have added +soft-float to the features so move the StringRef to
316e8d8bef9SDimitry Andric   // point to the full string in the Key.
317e8d8bef9SDimitry Andric   FS = Key.substr(FSStart);
3180b57cec5SDimitry Andric 
3190b57cec5SDimitry Andric   auto &I = SubtargetMap[Key];
3200b57cec5SDimitry Andric   if (!I) {
3210b57cec5SDimitry Andric     // This needs to be done before we create a new subtarget since any
3220b57cec5SDimitry Andric     // creation will depend on the TM and the code generation flags on the
3230b57cec5SDimitry Andric     // function that reside in TargetOptions.
3240b57cec5SDimitry Andric     resetTargetOptions(F);
3258bcb0991SDimitry Andric     I = std::make_unique<X86Subtarget>(
326e8d8bef9SDimitry Andric         TargetTriple, CPU, TuneCPU, FS, *this,
327fe6060f1SDimitry Andric         MaybeAlign(F.getParent()->getOverrideStackAlignment()),
328fe6060f1SDimitry Andric         PreferVectorWidthOverride, RequiredVectorWidth);
3290b57cec5SDimitry Andric   }
3300b57cec5SDimitry Andric   return I.get();
3310b57cec5SDimitry Andric }
3320b57cec5SDimitry Andric 
333e8d8bef9SDimitry Andric bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
334e8d8bef9SDimitry Andric                                            unsigned DestAS) const {
335e8d8bef9SDimitry Andric   assert(SrcAS != DestAS && "Expected different address spaces!");
336e8d8bef9SDimitry Andric   if (getPointerSize(SrcAS) != getPointerSize(DestAS))
337e8d8bef9SDimitry Andric     return false;
338e8d8bef9SDimitry Andric   return SrcAS < 256 && DestAS < 256;
339e8d8bef9SDimitry Andric }
340e8d8bef9SDimitry Andric 
3410b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
3420b57cec5SDimitry Andric // X86 TTI query.
3430b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
3440b57cec5SDimitry Andric 
3450b57cec5SDimitry Andric TargetTransformInfo
346*81ad6265SDimitry Andric X86TargetMachine::getTargetTransformInfo(const Function &F) const {
3470b57cec5SDimitry Andric   return TargetTransformInfo(X86TTIImpl(this, F));
3480b57cec5SDimitry Andric }
3490b57cec5SDimitry Andric 
3500b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
3510b57cec5SDimitry Andric // Pass Pipeline Configuration
3520b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
3530b57cec5SDimitry Andric 
3540b57cec5SDimitry Andric namespace {
3550b57cec5SDimitry Andric 
3560b57cec5SDimitry Andric /// X86 Code Generator Pass Configuration Options.
3570b57cec5SDimitry Andric class X86PassConfig : public TargetPassConfig {
3580b57cec5SDimitry Andric public:
3590b57cec5SDimitry Andric   X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM)
3600b57cec5SDimitry Andric     : TargetPassConfig(TM, PM) {}
3610b57cec5SDimitry Andric 
3620b57cec5SDimitry Andric   X86TargetMachine &getX86TargetMachine() const {
3630b57cec5SDimitry Andric     return getTM<X86TargetMachine>();
3640b57cec5SDimitry Andric   }
3650b57cec5SDimitry Andric 
3660b57cec5SDimitry Andric   ScheduleDAGInstrs *
3670b57cec5SDimitry Andric   createMachineScheduler(MachineSchedContext *C) const override {
3680b57cec5SDimitry Andric     ScheduleDAGMILive *DAG = createGenericSchedLive(C);
3690b57cec5SDimitry Andric     DAG->addMutation(createX86MacroFusionDAGMutation());
3700b57cec5SDimitry Andric     return DAG;
3710b57cec5SDimitry Andric   }
3720b57cec5SDimitry Andric 
3730b57cec5SDimitry Andric   ScheduleDAGInstrs *
3740b57cec5SDimitry Andric   createPostMachineScheduler(MachineSchedContext *C) const override {
3750b57cec5SDimitry Andric     ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
3760b57cec5SDimitry Andric     DAG->addMutation(createX86MacroFusionDAGMutation());
3770b57cec5SDimitry Andric     return DAG;
3780b57cec5SDimitry Andric   }
3790b57cec5SDimitry Andric 
3800b57cec5SDimitry Andric   void addIRPasses() override;
3810b57cec5SDimitry Andric   bool addInstSelector() override;
3820b57cec5SDimitry Andric   bool addIRTranslator() override;
3830b57cec5SDimitry Andric   bool addLegalizeMachineIR() override;
3840b57cec5SDimitry Andric   bool addRegBankSelect() override;
3850b57cec5SDimitry Andric   bool addGlobalInstructionSelect() override;
3860b57cec5SDimitry Andric   bool addILPOpts() override;
3870b57cec5SDimitry Andric   bool addPreISel() override;
3880b57cec5SDimitry Andric   void addMachineSSAOptimization() override;
3890b57cec5SDimitry Andric   void addPreRegAlloc() override;
390fe6060f1SDimitry Andric   bool addPostFastRegAllocRewrite() override;
3910b57cec5SDimitry Andric   void addPostRegAlloc() override;
3920b57cec5SDimitry Andric   void addPreEmitPass() override;
3930b57cec5SDimitry Andric   void addPreEmitPass2() override;
3940b57cec5SDimitry Andric   void addPreSched2() override;
395*81ad6265SDimitry Andric   bool addRegAssignAndRewriteOptimized() override;
3960b57cec5SDimitry Andric 
3970b57cec5SDimitry Andric   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
3980b57cec5SDimitry Andric };
3990b57cec5SDimitry Andric 
4000b57cec5SDimitry Andric class X86ExecutionDomainFix : public ExecutionDomainFix {
4010b57cec5SDimitry Andric public:
4020b57cec5SDimitry Andric   static char ID;
4030b57cec5SDimitry Andric   X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {}
4040b57cec5SDimitry Andric   StringRef getPassName() const override {
4050b57cec5SDimitry Andric     return "X86 Execution Dependency Fix";
4060b57cec5SDimitry Andric   }
4070b57cec5SDimitry Andric };
4080b57cec5SDimitry Andric char X86ExecutionDomainFix::ID;
4090b57cec5SDimitry Andric 
4100b57cec5SDimitry Andric } // end anonymous namespace
4110b57cec5SDimitry Andric 
4120b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix",
4130b57cec5SDimitry Andric   "X86 Execution Domain Fix", false, false)
4140b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis)
4150b57cec5SDimitry Andric INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix",
4160b57cec5SDimitry Andric   "X86 Execution Domain Fix", false, false)
4170b57cec5SDimitry Andric 
4180b57cec5SDimitry Andric TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
4190b57cec5SDimitry Andric   return new X86PassConfig(*this, PM);
4200b57cec5SDimitry Andric }
4210b57cec5SDimitry Andric 
4220b57cec5SDimitry Andric void X86PassConfig::addIRPasses() {
4230b57cec5SDimitry Andric   addPass(createAtomicExpandPass());
424fe6060f1SDimitry Andric 
425fe6060f1SDimitry Andric   // We add both pass anyway and when these two passes run, we skip the pass
426fe6060f1SDimitry Andric   // based on the option level and option attribute.
427fe6060f1SDimitry Andric   addPass(createX86LowerAMXIntrinsicsPass());
428e8d8bef9SDimitry Andric   addPass(createX86LowerAMXTypePass());
4290b57cec5SDimitry Andric 
4300b57cec5SDimitry Andric   TargetPassConfig::addIRPasses();
4310b57cec5SDimitry Andric 
4325ffd83dbSDimitry Andric   if (TM->getOptLevel() != CodeGenOpt::None) {
4330b57cec5SDimitry Andric     addPass(createInterleavedAccessPass());
4345ffd83dbSDimitry Andric     addPass(createX86PartialReductionPass());
4355ffd83dbSDimitry Andric   }
4360b57cec5SDimitry Andric 
4370b57cec5SDimitry Andric   // Add passes that handle indirect branch removal and insertion of a retpoline
4380b57cec5SDimitry Andric   // thunk. These will be a no-op unless a function subtarget has the retpoline
4390b57cec5SDimitry Andric   // feature enabled.
4400b57cec5SDimitry Andric   addPass(createIndirectBrExpandPass());
441480093f4SDimitry Andric 
442480093f4SDimitry Andric   // Add Control Flow Guard checks.
443480093f4SDimitry Andric   const Triple &TT = TM->getTargetTriple();
444480093f4SDimitry Andric   if (TT.isOSWindows()) {
445480093f4SDimitry Andric     if (TT.getArch() == Triple::x86_64) {
446480093f4SDimitry Andric       addPass(createCFGuardDispatchPass());
447480093f4SDimitry Andric     } else {
448480093f4SDimitry Andric       addPass(createCFGuardCheckPass());
449480093f4SDimitry Andric     }
450480093f4SDimitry Andric   }
451*81ad6265SDimitry Andric 
452*81ad6265SDimitry Andric   if (TM->Options.JMCInstrument)
453*81ad6265SDimitry Andric     addPass(createJMCInstrumenterPass());
4540b57cec5SDimitry Andric }
4550b57cec5SDimitry Andric 
4560b57cec5SDimitry Andric bool X86PassConfig::addInstSelector() {
4570b57cec5SDimitry Andric   // Install an instruction selector.
4580b57cec5SDimitry Andric   addPass(createX86ISelDag(getX86TargetMachine(), getOptLevel()));
4590b57cec5SDimitry Andric 
4600b57cec5SDimitry Andric   // For ELF, cleanup any local-dynamic TLS accesses.
4610b57cec5SDimitry Andric   if (TM->getTargetTriple().isOSBinFormatELF() &&
4620b57cec5SDimitry Andric       getOptLevel() != CodeGenOpt::None)
4630b57cec5SDimitry Andric     addPass(createCleanupLocalDynamicTLSPass());
4640b57cec5SDimitry Andric 
4650b57cec5SDimitry Andric   addPass(createX86GlobalBaseRegPass());
4660b57cec5SDimitry Andric   return false;
4670b57cec5SDimitry Andric }
4680b57cec5SDimitry Andric 
4690b57cec5SDimitry Andric bool X86PassConfig::addIRTranslator() {
470e8d8bef9SDimitry Andric   addPass(new IRTranslator(getOptLevel()));
4710b57cec5SDimitry Andric   return false;
4720b57cec5SDimitry Andric }
4730b57cec5SDimitry Andric 
4740b57cec5SDimitry Andric bool X86PassConfig::addLegalizeMachineIR() {
4750b57cec5SDimitry Andric   addPass(new Legalizer());
4760b57cec5SDimitry Andric   return false;
4770b57cec5SDimitry Andric }
4780b57cec5SDimitry Andric 
4790b57cec5SDimitry Andric bool X86PassConfig::addRegBankSelect() {
4800b57cec5SDimitry Andric   addPass(new RegBankSelect());
4810b57cec5SDimitry Andric   return false;
4820b57cec5SDimitry Andric }
4830b57cec5SDimitry Andric 
4840b57cec5SDimitry Andric bool X86PassConfig::addGlobalInstructionSelect() {
485fe6060f1SDimitry Andric   addPass(new InstructionSelect(getOptLevel()));
4860b57cec5SDimitry Andric   return false;
4870b57cec5SDimitry Andric }
4880b57cec5SDimitry Andric 
4890b57cec5SDimitry Andric bool X86PassConfig::addILPOpts() {
4900b57cec5SDimitry Andric   addPass(&EarlyIfConverterID);
4910b57cec5SDimitry Andric   if (EnableMachineCombinerPass)
4920b57cec5SDimitry Andric     addPass(&MachineCombinerID);
4930b57cec5SDimitry Andric   addPass(createX86CmovConverterPass());
4940b57cec5SDimitry Andric   return true;
4950b57cec5SDimitry Andric }
4960b57cec5SDimitry Andric 
4970b57cec5SDimitry Andric bool X86PassConfig::addPreISel() {
4980b57cec5SDimitry Andric   // Only add this pass for 32-bit x86 Windows.
4990b57cec5SDimitry Andric   const Triple &TT = TM->getTargetTriple();
5000b57cec5SDimitry Andric   if (TT.isOSWindows() && TT.getArch() == Triple::x86)
5010b57cec5SDimitry Andric     addPass(createX86WinEHStatePass());
5020b57cec5SDimitry Andric   return true;
5030b57cec5SDimitry Andric }
5040b57cec5SDimitry Andric 
5050b57cec5SDimitry Andric void X86PassConfig::addPreRegAlloc() {
5060b57cec5SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
5070b57cec5SDimitry Andric     addPass(&LiveRangeShrinkID);
5080b57cec5SDimitry Andric     addPass(createX86FixupSetCC());
5090b57cec5SDimitry Andric     addPass(createX86OptimizeLEAs());
5100b57cec5SDimitry Andric     addPass(createX86CallFrameOptimization());
5110b57cec5SDimitry Andric     addPass(createX86AvoidStoreForwardingBlocks());
5120b57cec5SDimitry Andric   }
5130b57cec5SDimitry Andric 
5140b57cec5SDimitry Andric   addPass(createX86SpeculativeLoadHardeningPass());
5150b57cec5SDimitry Andric   addPass(createX86FlagsCopyLoweringPass());
516349cc55cSDimitry Andric   addPass(createX86DynAllocaExpander());
517e8d8bef9SDimitry Andric 
518*81ad6265SDimitry Andric   if (getOptLevel() != CodeGenOpt::None)
519e8d8bef9SDimitry Andric     addPass(createX86PreTileConfigPass());
520*81ad6265SDimitry Andric   else
521*81ad6265SDimitry Andric     addPass(createX86FastPreTileConfigPass());
522e8d8bef9SDimitry Andric }
523e8d8bef9SDimitry Andric 
5240b57cec5SDimitry Andric void X86PassConfig::addMachineSSAOptimization() {
5250b57cec5SDimitry Andric   addPass(createX86DomainReassignmentPass());
5260b57cec5SDimitry Andric   TargetPassConfig::addMachineSSAOptimization();
5270b57cec5SDimitry Andric }
5280b57cec5SDimitry Andric 
5290b57cec5SDimitry Andric void X86PassConfig::addPostRegAlloc() {
530fe6060f1SDimitry Andric   addPass(createX86LowerTileCopyPass());
5310b57cec5SDimitry Andric   addPass(createX86FloatingPointStackifierPass());
5325ffd83dbSDimitry Andric   // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
5335ffd83dbSDimitry Andric   // to using the Speculative Execution Side Effect Suppression pass for
5345ffd83dbSDimitry Andric   // mitigation. This is to prevent slow downs due to
5355ffd83dbSDimitry Andric   // analyses needed by the LVIHardening pass when compiling at -O0.
5360946e70aSDimitry Andric   if (getOptLevel() != CodeGenOpt::None)
5370946e70aSDimitry Andric     addPass(createX86LoadValueInjectionLoadHardeningPass());
5380b57cec5SDimitry Andric }
5390b57cec5SDimitry Andric 
5400b57cec5SDimitry Andric void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); }
5410b57cec5SDimitry Andric 
5420b57cec5SDimitry Andric void X86PassConfig::addPreEmitPass() {
5430b57cec5SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
5440b57cec5SDimitry Andric     addPass(new X86ExecutionDomainFix());
5450b57cec5SDimitry Andric     addPass(createBreakFalseDeps());
5460b57cec5SDimitry Andric   }
5470b57cec5SDimitry Andric 
5480b57cec5SDimitry Andric   addPass(createX86IndirectBranchTrackingPass());
5490b57cec5SDimitry Andric 
5500b57cec5SDimitry Andric   addPass(createX86IssueVZeroUpperPass());
5510b57cec5SDimitry Andric 
5520b57cec5SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
5530b57cec5SDimitry Andric     addPass(createX86FixupBWInsts());
5540b57cec5SDimitry Andric     addPass(createX86PadShortFunctions());
5550b57cec5SDimitry Andric     addPass(createX86FixupLEAs());
5560b57cec5SDimitry Andric   }
5575ffd83dbSDimitry Andric   addPass(createX86EvexToVexInsts());
5580b57cec5SDimitry Andric   addPass(createX86DiscriminateMemOpsPass());
5590b57cec5SDimitry Andric   addPass(createX86InsertPrefetchPass());
5605ffd83dbSDimitry Andric   addPass(createX86InsertX87waitPass());
5610b57cec5SDimitry Andric }
5620b57cec5SDimitry Andric 
5630b57cec5SDimitry Andric void X86PassConfig::addPreEmitPass2() {
5648bcb0991SDimitry Andric   const Triple &TT = TM->getTargetTriple();
5658bcb0991SDimitry Andric   const MCAsmInfo *MAI = TM->getMCAsmInfo();
5668bcb0991SDimitry Andric 
5675ffd83dbSDimitry Andric   // The X86 Speculative Execution Pass must run after all control
5685ffd83dbSDimitry Andric   // flow graph modifying passes. As a result it was listed to run right before
5695ffd83dbSDimitry Andric   // the X86 Retpoline Thunks pass. The reason it must run after control flow
5705ffd83dbSDimitry Andric   // graph modifications is that the model of LFENCE in LLVM has to be updated
5715ffd83dbSDimitry Andric   // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
5725ffd83dbSDimitry Andric   // placement of this pass was hand checked to ensure that the subsequent
5735ffd83dbSDimitry Andric   // passes don't move the code around the LFENCEs in a way that will hurt the
5745ffd83dbSDimitry Andric   // correctness of this pass. This placement has been shown to work based on
5755ffd83dbSDimitry Andric   // hand inspection of the codegen output.
5765ffd83dbSDimitry Andric   addPass(createX86SpeculativeExecutionSideEffectSuppression());
5770946e70aSDimitry Andric   addPass(createX86IndirectThunksPass());
5788bcb0991SDimitry Andric 
5798bcb0991SDimitry Andric   // Insert extra int3 instructions after trailing call instructions to avoid
5808bcb0991SDimitry Andric   // issues in the unwinder.
5818bcb0991SDimitry Andric   if (TT.isOSWindows() && TT.getArch() == Triple::x86_64)
5828bcb0991SDimitry Andric     addPass(createX86AvoidTrailingCallPass());
5838bcb0991SDimitry Andric 
5840b57cec5SDimitry Andric   // Verify basic block incoming and outgoing cfa offset and register values and
5850b57cec5SDimitry Andric   // correct CFA calculation rule where needed by inserting appropriate CFI
5860b57cec5SDimitry Andric   // instructions.
5870b57cec5SDimitry Andric   if (!TT.isOSDarwin() &&
5880b57cec5SDimitry Andric       (!TT.isOSWindows() ||
5890b57cec5SDimitry Andric        MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI))
5900b57cec5SDimitry Andric     addPass(createCFIInstrInserter());
591fe6060f1SDimitry Andric 
592fe6060f1SDimitry Andric   if (TT.isOSWindows()) {
593480093f4SDimitry Andric     // Identify valid longjmp targets for Windows Control Flow Guard.
594480093f4SDimitry Andric     addPass(createCFGuardLongjmpPass());
595fe6060f1SDimitry Andric     // Identify valid eh continuation targets for Windows EHCont Guard.
596fe6060f1SDimitry Andric     addPass(createEHContGuardCatchretPass());
597fe6060f1SDimitry Andric   }
5980946e70aSDimitry Andric   addPass(createX86LoadValueInjectionRetHardeningPass());
599349cc55cSDimitry Andric 
600349cc55cSDimitry Andric   // Insert pseudo probe annotation for callsite profiling
601349cc55cSDimitry Andric   addPass(createPseudoProbeInserter());
6020eae32dcSDimitry Andric 
6030eae32dcSDimitry Andric   // On Darwin platforms, BLR_RVMARKER pseudo instructions are lowered to
6040eae32dcSDimitry Andric   // bundles.
6050eae32dcSDimitry Andric   if (TT.isOSDarwin())
6060eae32dcSDimitry Andric     addPass(createUnpackMachineBundles([](const MachineFunction &MF) {
6070eae32dcSDimitry Andric       // Only run bundle expansion if there are relevant ObjC runtime functions
6080eae32dcSDimitry Andric       // present in the module.
6090eae32dcSDimitry Andric       const Function &F = MF.getFunction();
6100eae32dcSDimitry Andric       const Module *M = F.getParent();
6110eae32dcSDimitry Andric       return M->getFunction("objc_retainAutoreleasedReturnValue") ||
6120eae32dcSDimitry Andric              M->getFunction("objc_unsafeClaimAutoreleasedReturnValue");
6130eae32dcSDimitry Andric     }));
6140b57cec5SDimitry Andric }
6150b57cec5SDimitry Andric 
616fe6060f1SDimitry Andric bool X86PassConfig::addPostFastRegAllocRewrite() {
617fe6060f1SDimitry Andric   addPass(createX86FastTileConfigPass());
618fe6060f1SDimitry Andric   return true;
619fe6060f1SDimitry Andric }
620fe6060f1SDimitry Andric 
6210b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
6220b57cec5SDimitry Andric   return getStandardCSEConfigForOpt(TM->getOptLevel());
6230b57cec5SDimitry Andric }
624*81ad6265SDimitry Andric 
625*81ad6265SDimitry Andric static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
626*81ad6265SDimitry Andric                                       const TargetRegisterClass &RC) {
627*81ad6265SDimitry Andric   return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(&RC);
628*81ad6265SDimitry Andric }
629*81ad6265SDimitry Andric 
630*81ad6265SDimitry Andric bool X86PassConfig::addRegAssignAndRewriteOptimized() {
631*81ad6265SDimitry Andric   // Don't support tile RA when RA is specified by command line "-regalloc".
632*81ad6265SDimitry Andric   if (!isCustomizedRegAlloc() && EnableTileRAPass) {
633*81ad6265SDimitry Andric     // Allocate tile register first.
634*81ad6265SDimitry Andric     addPass(createGreedyRegisterAllocator(onlyAllocateTileRegisters));
635*81ad6265SDimitry Andric     addPass(createX86TileConfigPass());
636*81ad6265SDimitry Andric   }
637*81ad6265SDimitry Andric   return TargetPassConfig::addRegAssignAndRewriteOptimized();
638*81ad6265SDimitry Andric }
639