xref: /freebsd/contrib/llvm-project/llvm/lib/Target/ARM/ARMTargetMachine.cpp (revision 81ad626541db97eb356e2c1d4a20eb2a26a766ab)
10b57cec5SDimitry Andric //===-- ARMTargetMachine.cpp - Define TargetMachine for ARM ---------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric //
100b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
110b57cec5SDimitry Andric 
120b57cec5SDimitry Andric #include "ARMTargetMachine.h"
130b57cec5SDimitry Andric #include "ARM.h"
140b57cec5SDimitry Andric #include "ARMMacroFusion.h"
150b57cec5SDimitry Andric #include "ARMSubtarget.h"
160b57cec5SDimitry Andric #include "ARMTargetObjectFile.h"
170b57cec5SDimitry Andric #include "ARMTargetTransformInfo.h"
180b57cec5SDimitry Andric #include "MCTargetDesc/ARMMCTargetDesc.h"
190b57cec5SDimitry Andric #include "TargetInfo/ARMTargetInfo.h"
200b57cec5SDimitry Andric #include "llvm/ADT/Optional.h"
210b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h"
220b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
230b57cec5SDimitry Andric #include "llvm/ADT/Triple.h"
240b57cec5SDimitry Andric #include "llvm/Analysis/TargetTransformInfo.h"
250b57cec5SDimitry Andric #include "llvm/CodeGen/ExecutionDomainFix.h"
26*81ad6265SDimitry Andric #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
270b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/CallLowering.h"
280b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
290b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
300b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
310b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/Legalizer.h"
320b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
330b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
340b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h"
350b57cec5SDimitry Andric #include "llvm/CodeGen/MachineScheduler.h"
360b57cec5SDimitry Andric #include "llvm/CodeGen/Passes.h"
37*81ad6265SDimitry Andric #include "llvm/CodeGen/RegisterBankInfo.h"
380b57cec5SDimitry Andric #include "llvm/CodeGen/TargetPassConfig.h"
390b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
400b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
410b57cec5SDimitry Andric #include "llvm/IR/Function.h"
42349cc55cSDimitry Andric #include "llvm/MC/TargetRegistry.h"
430b57cec5SDimitry Andric #include "llvm/Pass.h"
44*81ad6265SDimitry Andric #include "llvm/Support/ARMTargetParser.h"
450b57cec5SDimitry Andric #include "llvm/Support/CodeGen.h"
460b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
470b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
480b57cec5SDimitry Andric #include "llvm/Support/TargetParser.h"
490b57cec5SDimitry Andric #include "llvm/Target/TargetLoweringObjectFile.h"
500b57cec5SDimitry Andric #include "llvm/Target/TargetOptions.h"
51480093f4SDimitry Andric #include "llvm/Transforms/CFGuard.h"
52fe6060f1SDimitry Andric #include "llvm/Transforms/IPO.h"
530b57cec5SDimitry Andric #include "llvm/Transforms/Scalar.h"
540b57cec5SDimitry Andric #include <cassert>
550b57cec5SDimitry Andric #include <memory>
560b57cec5SDimitry Andric #include <string>
570b57cec5SDimitry Andric 
580b57cec5SDimitry Andric using namespace llvm;
590b57cec5SDimitry Andric 
600b57cec5SDimitry Andric static cl::opt<bool>
610b57cec5SDimitry Andric DisableA15SDOptimization("disable-a15-sd-optimization", cl::Hidden,
620b57cec5SDimitry Andric                    cl::desc("Inhibit optimization of S->D register accesses on A15"),
630b57cec5SDimitry Andric                    cl::init(false));
640b57cec5SDimitry Andric 
650b57cec5SDimitry Andric static cl::opt<bool>
660b57cec5SDimitry Andric EnableAtomicTidy("arm-atomic-cfg-tidy", cl::Hidden,
670b57cec5SDimitry Andric                  cl::desc("Run SimplifyCFG after expanding atomic operations"
680b57cec5SDimitry Andric                           " to make use of cmpxchg flow-based information"),
690b57cec5SDimitry Andric                  cl::init(true));
700b57cec5SDimitry Andric 
710b57cec5SDimitry Andric static cl::opt<bool>
720b57cec5SDimitry Andric EnableARMLoadStoreOpt("arm-load-store-opt", cl::Hidden,
730b57cec5SDimitry Andric                       cl::desc("Enable ARM load/store optimization pass"),
740b57cec5SDimitry Andric                       cl::init(true));
750b57cec5SDimitry Andric 
760b57cec5SDimitry Andric // FIXME: Unify control over GlobalMerge.
770b57cec5SDimitry Andric static cl::opt<cl::boolOrDefault>
780b57cec5SDimitry Andric EnableGlobalMerge("arm-global-merge", cl::Hidden,
790b57cec5SDimitry Andric                   cl::desc("Enable the global merge pass"));
800b57cec5SDimitry Andric 
810b57cec5SDimitry Andric namespace llvm {
820b57cec5SDimitry Andric   void initializeARMExecutionDomainFixPass(PassRegistry&);
830b57cec5SDimitry Andric }
840b57cec5SDimitry Andric 
85480093f4SDimitry Andric extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeARMTarget() {
860b57cec5SDimitry Andric   // Register the target.
870b57cec5SDimitry Andric   RegisterTargetMachine<ARMLETargetMachine> X(getTheARMLETarget());
880b57cec5SDimitry Andric   RegisterTargetMachine<ARMLETargetMachine> A(getTheThumbLETarget());
890b57cec5SDimitry Andric   RegisterTargetMachine<ARMBETargetMachine> Y(getTheARMBETarget());
900b57cec5SDimitry Andric   RegisterTargetMachine<ARMBETargetMachine> B(getTheThumbBETarget());
910b57cec5SDimitry Andric 
920b57cec5SDimitry Andric   PassRegistry &Registry = *PassRegistry::getPassRegistry();
930b57cec5SDimitry Andric   initializeGlobalISel(Registry);
940b57cec5SDimitry Andric   initializeARMLoadStoreOptPass(Registry);
950b57cec5SDimitry Andric   initializeARMPreAllocLoadStoreOptPass(Registry);
960b57cec5SDimitry Andric   initializeARMParallelDSPPass(Registry);
974824e7fdSDimitry Andric   initializeARMBranchTargetsPass(Registry);
980b57cec5SDimitry Andric   initializeARMConstantIslandsPass(Registry);
990b57cec5SDimitry Andric   initializeARMExecutionDomainFixPass(Registry);
1000b57cec5SDimitry Andric   initializeARMExpandPseudoPass(Registry);
1010b57cec5SDimitry Andric   initializeThumb2SizeReducePass(Registry);
1020b57cec5SDimitry Andric   initializeMVEVPTBlockPass(Registry);
103fe6060f1SDimitry Andric   initializeMVETPAndVPTOptimisationsPass(Registry);
1048bcb0991SDimitry Andric   initializeMVETailPredicationPass(Registry);
1050b57cec5SDimitry Andric   initializeARMLowOverheadLoopsPass(Registry);
106e8d8bef9SDimitry Andric   initializeARMBlockPlacementPass(Registry);
107480093f4SDimitry Andric   initializeMVEGatherScatterLoweringPass(Registry);
108e8d8bef9SDimitry Andric   initializeARMSLSHardeningPass(Registry);
109fe6060f1SDimitry Andric   initializeMVELaneInterleavingPass(Registry);
110*81ad6265SDimitry Andric   initializeARMFixCortexA57AES1742098Pass(Registry);
1110b57cec5SDimitry Andric }
1120b57cec5SDimitry Andric 
1130b57cec5SDimitry Andric static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
1140b57cec5SDimitry Andric   if (TT.isOSBinFormatMachO())
1158bcb0991SDimitry Andric     return std::make_unique<TargetLoweringObjectFileMachO>();
1160b57cec5SDimitry Andric   if (TT.isOSWindows())
1178bcb0991SDimitry Andric     return std::make_unique<TargetLoweringObjectFileCOFF>();
1188bcb0991SDimitry Andric   return std::make_unique<ARMElfTargetObjectFile>();
1190b57cec5SDimitry Andric }
1200b57cec5SDimitry Andric 
1210b57cec5SDimitry Andric static ARMBaseTargetMachine::ARMABI
1220b57cec5SDimitry Andric computeTargetABI(const Triple &TT, StringRef CPU,
1230b57cec5SDimitry Andric                  const TargetOptions &Options) {
1240b57cec5SDimitry Andric   StringRef ABIName = Options.MCOptions.getABIName();
1250b57cec5SDimitry Andric 
1260b57cec5SDimitry Andric   if (ABIName.empty())
1270b57cec5SDimitry Andric     ABIName = ARM::computeDefaultTargetABI(TT, CPU);
1280b57cec5SDimitry Andric 
1290b57cec5SDimitry Andric   if (ABIName == "aapcs16")
1300b57cec5SDimitry Andric     return ARMBaseTargetMachine::ARM_ABI_AAPCS16;
1310b57cec5SDimitry Andric   else if (ABIName.startswith("aapcs"))
1320b57cec5SDimitry Andric     return ARMBaseTargetMachine::ARM_ABI_AAPCS;
1330b57cec5SDimitry Andric   else if (ABIName.startswith("apcs"))
1340b57cec5SDimitry Andric     return ARMBaseTargetMachine::ARM_ABI_APCS;
1350b57cec5SDimitry Andric 
1360b57cec5SDimitry Andric   llvm_unreachable("Unhandled/unknown ABI Name!");
1370b57cec5SDimitry Andric   return ARMBaseTargetMachine::ARM_ABI_UNKNOWN;
1380b57cec5SDimitry Andric }
1390b57cec5SDimitry Andric 
1400b57cec5SDimitry Andric static std::string computeDataLayout(const Triple &TT, StringRef CPU,
1410b57cec5SDimitry Andric                                      const TargetOptions &Options,
1420b57cec5SDimitry Andric                                      bool isLittle) {
1430b57cec5SDimitry Andric   auto ABI = computeTargetABI(TT, CPU, Options);
1440b57cec5SDimitry Andric   std::string Ret;
1450b57cec5SDimitry Andric 
1460b57cec5SDimitry Andric   if (isLittle)
1470b57cec5SDimitry Andric     // Little endian.
1480b57cec5SDimitry Andric     Ret += "e";
1490b57cec5SDimitry Andric   else
1500b57cec5SDimitry Andric     // Big endian.
1510b57cec5SDimitry Andric     Ret += "E";
1520b57cec5SDimitry Andric 
1530b57cec5SDimitry Andric   Ret += DataLayout::getManglingComponent(TT);
1540b57cec5SDimitry Andric 
1550b57cec5SDimitry Andric   // Pointers are 32 bits and aligned to 32 bits.
1560b57cec5SDimitry Andric   Ret += "-p:32:32";
1570b57cec5SDimitry Andric 
1580b57cec5SDimitry Andric   // Function pointers are aligned to 8 bits (because the LSB stores the
1590b57cec5SDimitry Andric   // ARM/Thumb state).
1600b57cec5SDimitry Andric   Ret += "-Fi8";
1610b57cec5SDimitry Andric 
1620b57cec5SDimitry Andric   // ABIs other than APCS have 64 bit integers with natural alignment.
1630b57cec5SDimitry Andric   if (ABI != ARMBaseTargetMachine::ARM_ABI_APCS)
1640b57cec5SDimitry Andric     Ret += "-i64:64";
1650b57cec5SDimitry Andric 
1660b57cec5SDimitry Andric   // We have 64 bits floats. The APCS ABI requires them to be aligned to 32
1670b57cec5SDimitry Andric   // bits, others to 64 bits. We always try to align to 64 bits.
1680b57cec5SDimitry Andric   if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS)
1690b57cec5SDimitry Andric     Ret += "-f64:32:64";
1700b57cec5SDimitry Andric 
1710b57cec5SDimitry Andric   // We have 128 and 64 bit vectors. The APCS ABI aligns them to 32 bits, others
1720b57cec5SDimitry Andric   // to 64. We always ty to give them natural alignment.
1730b57cec5SDimitry Andric   if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS)
1740b57cec5SDimitry Andric     Ret += "-v64:32:64-v128:32:128";
1750b57cec5SDimitry Andric   else if (ABI != ARMBaseTargetMachine::ARM_ABI_AAPCS16)
1760b57cec5SDimitry Andric     Ret += "-v128:64:128";
1770b57cec5SDimitry Andric 
1780b57cec5SDimitry Andric   // Try to align aggregates to 32 bits (the default is 64 bits, which has no
1790b57cec5SDimitry Andric   // particular hardware support on 32-bit ARM).
1800b57cec5SDimitry Andric   Ret += "-a:0:32";
1810b57cec5SDimitry Andric 
1820b57cec5SDimitry Andric   // Integer registers are 32 bits.
1830b57cec5SDimitry Andric   Ret += "-n32";
1840b57cec5SDimitry Andric 
1850b57cec5SDimitry Andric   // The stack is 128 bit aligned on NaCl, 64 bit aligned on AAPCS and 32 bit
1860b57cec5SDimitry Andric   // aligned everywhere else.
1870b57cec5SDimitry Andric   if (TT.isOSNaCl() || ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16)
1880b57cec5SDimitry Andric     Ret += "-S128";
1890b57cec5SDimitry Andric   else if (ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS)
1900b57cec5SDimitry Andric     Ret += "-S64";
1910b57cec5SDimitry Andric   else
1920b57cec5SDimitry Andric     Ret += "-S32";
1930b57cec5SDimitry Andric 
1940b57cec5SDimitry Andric   return Ret;
1950b57cec5SDimitry Andric }
1960b57cec5SDimitry Andric 
1970b57cec5SDimitry Andric static Reloc::Model getEffectiveRelocModel(const Triple &TT,
1980b57cec5SDimitry Andric                                            Optional<Reloc::Model> RM) {
199*81ad6265SDimitry Andric   if (!RM)
2000b57cec5SDimitry Andric     // Default relocation model on Darwin is PIC.
2010b57cec5SDimitry Andric     return TT.isOSBinFormatMachO() ? Reloc::PIC_ : Reloc::Static;
2020b57cec5SDimitry Andric 
2030b57cec5SDimitry Andric   if (*RM == Reloc::ROPI || *RM == Reloc::RWPI || *RM == Reloc::ROPI_RWPI)
2040b57cec5SDimitry Andric     assert(TT.isOSBinFormatELF() &&
2050b57cec5SDimitry Andric            "ROPI/RWPI currently only supported for ELF");
2060b57cec5SDimitry Andric 
2070b57cec5SDimitry Andric   // DynamicNoPIC is only used on darwin.
2080b57cec5SDimitry Andric   if (*RM == Reloc::DynamicNoPIC && !TT.isOSDarwin())
2090b57cec5SDimitry Andric     return Reloc::Static;
2100b57cec5SDimitry Andric 
2110b57cec5SDimitry Andric   return *RM;
2120b57cec5SDimitry Andric }
2130b57cec5SDimitry Andric 
2140b57cec5SDimitry Andric /// Create an ARM architecture model.
2150b57cec5SDimitry Andric ///
2160b57cec5SDimitry Andric ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT,
2170b57cec5SDimitry Andric                                            StringRef CPU, StringRef FS,
2180b57cec5SDimitry Andric                                            const TargetOptions &Options,
2190b57cec5SDimitry Andric                                            Optional<Reloc::Model> RM,
2200b57cec5SDimitry Andric                                            Optional<CodeModel::Model> CM,
2210b57cec5SDimitry Andric                                            CodeGenOpt::Level OL, bool isLittle)
2220b57cec5SDimitry Andric     : LLVMTargetMachine(T, computeDataLayout(TT, CPU, Options, isLittle), TT,
2230b57cec5SDimitry Andric                         CPU, FS, Options, getEffectiveRelocModel(TT, RM),
2240b57cec5SDimitry Andric                         getEffectiveCodeModel(CM, CodeModel::Small), OL),
2250b57cec5SDimitry Andric       TargetABI(computeTargetABI(TT, CPU, Options)),
2260b57cec5SDimitry Andric       TLOF(createTLOF(getTargetTriple())), isLittle(isLittle) {
2270b57cec5SDimitry Andric 
2280b57cec5SDimitry Andric   // Default to triple-appropriate float ABI
2290b57cec5SDimitry Andric   if (Options.FloatABIType == FloatABI::Default) {
2300b57cec5SDimitry Andric     if (isTargetHardFloat())
2310b57cec5SDimitry Andric       this->Options.FloatABIType = FloatABI::Hard;
2320b57cec5SDimitry Andric     else
2330b57cec5SDimitry Andric       this->Options.FloatABIType = FloatABI::Soft;
2340b57cec5SDimitry Andric   }
2350b57cec5SDimitry Andric 
2360b57cec5SDimitry Andric   // Default to triple-appropriate EABI
2370b57cec5SDimitry Andric   if (Options.EABIVersion == EABI::Default ||
2380b57cec5SDimitry Andric       Options.EABIVersion == EABI::Unknown) {
2390b57cec5SDimitry Andric     // musl is compatible with glibc with regard to EABI version
2400b57cec5SDimitry Andric     if ((TargetTriple.getEnvironment() == Triple::GNUEABI ||
2410b57cec5SDimitry Andric          TargetTriple.getEnvironment() == Triple::GNUEABIHF ||
2420b57cec5SDimitry Andric          TargetTriple.getEnvironment() == Triple::MuslEABI ||
2430b57cec5SDimitry Andric          TargetTriple.getEnvironment() == Triple::MuslEABIHF) &&
2440b57cec5SDimitry Andric         !(TargetTriple.isOSWindows() || TargetTriple.isOSDarwin()))
2450b57cec5SDimitry Andric       this->Options.EABIVersion = EABI::GNU;
2460b57cec5SDimitry Andric     else
2470b57cec5SDimitry Andric       this->Options.EABIVersion = EABI::EABI5;
2480b57cec5SDimitry Andric   }
2490b57cec5SDimitry Andric 
2500b57cec5SDimitry Andric   if (TT.isOSBinFormatMachO()) {
2510b57cec5SDimitry Andric     this->Options.TrapUnreachable = true;
2520b57cec5SDimitry Andric     this->Options.NoTrapAfterNoreturn = true;
2530b57cec5SDimitry Andric   }
2540b57cec5SDimitry Andric 
2555ffd83dbSDimitry Andric   // ARM supports the debug entry values.
2565ffd83dbSDimitry Andric   setSupportsDebugEntryValues(true);
2575ffd83dbSDimitry Andric 
2580b57cec5SDimitry Andric   initAsmInfo();
2595ffd83dbSDimitry Andric 
2605ffd83dbSDimitry Andric   // ARM supports the MachineOutliner.
2615ffd83dbSDimitry Andric   setMachineOutliner(true);
262e8d8bef9SDimitry Andric   setSupportsDefaultOutlining(true);
2630b57cec5SDimitry Andric }
2640b57cec5SDimitry Andric 
2650b57cec5SDimitry Andric ARMBaseTargetMachine::~ARMBaseTargetMachine() = default;
2660b57cec5SDimitry Andric 
2670b57cec5SDimitry Andric const ARMSubtarget *
2680b57cec5SDimitry Andric ARMBaseTargetMachine::getSubtargetImpl(const Function &F) const {
2690b57cec5SDimitry Andric   Attribute CPUAttr = F.getFnAttribute("target-cpu");
2700b57cec5SDimitry Andric   Attribute FSAttr = F.getFnAttribute("target-features");
2710b57cec5SDimitry Andric 
272e8d8bef9SDimitry Andric   std::string CPU =
273e8d8bef9SDimitry Andric       CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
274e8d8bef9SDimitry Andric   std::string FS =
275e8d8bef9SDimitry Andric       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
2760b57cec5SDimitry Andric 
2770b57cec5SDimitry Andric   // FIXME: This is related to the code below to reset the target options,
2780b57cec5SDimitry Andric   // we need to know whether or not the soft float flag is set on the
2790b57cec5SDimitry Andric   // function before we can generate a subtarget. We also need to use
2800b57cec5SDimitry Andric   // it as a key for the subtarget since that can be the only difference
2810b57cec5SDimitry Andric   // between two functions.
282fe6060f1SDimitry Andric   bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsBool();
2830b57cec5SDimitry Andric   // If the soft float attribute is set on the function turn on the soft float
2840b57cec5SDimitry Andric   // subtarget feature.
2850b57cec5SDimitry Andric   if (SoftFloat)
2860b57cec5SDimitry Andric     FS += FS.empty() ? "+soft-float" : ",+soft-float";
2870b57cec5SDimitry Andric 
2880b57cec5SDimitry Andric   // Use the optminsize to identify the subtarget, but don't use it in the
2890b57cec5SDimitry Andric   // feature string.
2900b57cec5SDimitry Andric   std::string Key = CPU + FS;
2910b57cec5SDimitry Andric   if (F.hasMinSize())
2920b57cec5SDimitry Andric     Key += "+minsize";
2930b57cec5SDimitry Andric 
2940b57cec5SDimitry Andric   auto &I = SubtargetMap[Key];
2950b57cec5SDimitry Andric   if (!I) {
2960b57cec5SDimitry Andric     // This needs to be done before we create a new subtarget since any
2970b57cec5SDimitry Andric     // creation will depend on the TM and the code generation flags on the
2980b57cec5SDimitry Andric     // function that reside in TargetOptions.
2990b57cec5SDimitry Andric     resetTargetOptions(F);
3008bcb0991SDimitry Andric     I = std::make_unique<ARMSubtarget>(TargetTriple, CPU, FS, *this, isLittle,
3010b57cec5SDimitry Andric                                         F.hasMinSize());
3020b57cec5SDimitry Andric 
3030b57cec5SDimitry Andric     if (!I->isThumb() && !I->hasARMOps())
3040b57cec5SDimitry Andric       F.getContext().emitError("Function '" + F.getName() + "' uses ARM "
3050b57cec5SDimitry Andric           "instructions, but the target does not support ARM mode execution.");
3060b57cec5SDimitry Andric   }
3070b57cec5SDimitry Andric 
3080b57cec5SDimitry Andric   return I.get();
3090b57cec5SDimitry Andric }
3100b57cec5SDimitry Andric 
3110b57cec5SDimitry Andric TargetTransformInfo
312*81ad6265SDimitry Andric ARMBaseTargetMachine::getTargetTransformInfo(const Function &F) const {
3130b57cec5SDimitry Andric   return TargetTransformInfo(ARMTTIImpl(this, F));
3140b57cec5SDimitry Andric }
3150b57cec5SDimitry Andric 
3160b57cec5SDimitry Andric ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT,
3170b57cec5SDimitry Andric                                        StringRef CPU, StringRef FS,
3180b57cec5SDimitry Andric                                        const TargetOptions &Options,
3190b57cec5SDimitry Andric                                        Optional<Reloc::Model> RM,
3200b57cec5SDimitry Andric                                        Optional<CodeModel::Model> CM,
3210b57cec5SDimitry Andric                                        CodeGenOpt::Level OL, bool JIT)
3220b57cec5SDimitry Andric     : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
3230b57cec5SDimitry Andric 
3240b57cec5SDimitry Andric ARMBETargetMachine::ARMBETargetMachine(const Target &T, const Triple &TT,
3250b57cec5SDimitry Andric                                        StringRef CPU, StringRef FS,
3260b57cec5SDimitry Andric                                        const TargetOptions &Options,
3270b57cec5SDimitry Andric                                        Optional<Reloc::Model> RM,
3280b57cec5SDimitry Andric                                        Optional<CodeModel::Model> CM,
3290b57cec5SDimitry Andric                                        CodeGenOpt::Level OL, bool JIT)
3300b57cec5SDimitry Andric     : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
3310b57cec5SDimitry Andric 
3320b57cec5SDimitry Andric namespace {
3330b57cec5SDimitry Andric 
3340b57cec5SDimitry Andric /// ARM Code Generator Pass Configuration Options.
3350b57cec5SDimitry Andric class ARMPassConfig : public TargetPassConfig {
3360b57cec5SDimitry Andric public:
3370b57cec5SDimitry Andric   ARMPassConfig(ARMBaseTargetMachine &TM, PassManagerBase &PM)
338480093f4SDimitry Andric       : TargetPassConfig(TM, PM) {}
3390b57cec5SDimitry Andric 
3400b57cec5SDimitry Andric   ARMBaseTargetMachine &getARMTargetMachine() const {
3410b57cec5SDimitry Andric     return getTM<ARMBaseTargetMachine>();
3420b57cec5SDimitry Andric   }
3430b57cec5SDimitry Andric 
3440b57cec5SDimitry Andric   ScheduleDAGInstrs *
3450b57cec5SDimitry Andric   createMachineScheduler(MachineSchedContext *C) const override {
3460b57cec5SDimitry Andric     ScheduleDAGMILive *DAG = createGenericSchedLive(C);
3470b57cec5SDimitry Andric     // add DAG Mutations here.
3480b57cec5SDimitry Andric     const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
3490b57cec5SDimitry Andric     if (ST.hasFusion())
3500b57cec5SDimitry Andric       DAG->addMutation(createARMMacroFusionDAGMutation());
3510b57cec5SDimitry Andric     return DAG;
3520b57cec5SDimitry Andric   }
3530b57cec5SDimitry Andric 
3540b57cec5SDimitry Andric   ScheduleDAGInstrs *
3550b57cec5SDimitry Andric   createPostMachineScheduler(MachineSchedContext *C) const override {
3560b57cec5SDimitry Andric     ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
3570b57cec5SDimitry Andric     // add DAG Mutations here.
3580b57cec5SDimitry Andric     const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
3590b57cec5SDimitry Andric     if (ST.hasFusion())
3600b57cec5SDimitry Andric       DAG->addMutation(createARMMacroFusionDAGMutation());
3610b57cec5SDimitry Andric     return DAG;
3620b57cec5SDimitry Andric   }
3630b57cec5SDimitry Andric 
3640b57cec5SDimitry Andric   void addIRPasses() override;
3650b57cec5SDimitry Andric   void addCodeGenPrepare() override;
3660b57cec5SDimitry Andric   bool addPreISel() override;
3670b57cec5SDimitry Andric   bool addInstSelector() override;
3680b57cec5SDimitry Andric   bool addIRTranslator() override;
3690b57cec5SDimitry Andric   bool addLegalizeMachineIR() override;
3700b57cec5SDimitry Andric   bool addRegBankSelect() override;
3710b57cec5SDimitry Andric   bool addGlobalInstructionSelect() override;
3720b57cec5SDimitry Andric   void addPreRegAlloc() override;
3730b57cec5SDimitry Andric   void addPreSched2() override;
3740b57cec5SDimitry Andric   void addPreEmitPass() override;
3755ffd83dbSDimitry Andric   void addPreEmitPass2() override;
3760b57cec5SDimitry Andric 
3770b57cec5SDimitry Andric   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
3780b57cec5SDimitry Andric };
3790b57cec5SDimitry Andric 
3800b57cec5SDimitry Andric class ARMExecutionDomainFix : public ExecutionDomainFix {
3810b57cec5SDimitry Andric public:
3820b57cec5SDimitry Andric   static char ID;
3830b57cec5SDimitry Andric   ARMExecutionDomainFix() : ExecutionDomainFix(ID, ARM::DPRRegClass) {}
3840b57cec5SDimitry Andric   StringRef getPassName() const override {
3850b57cec5SDimitry Andric     return "ARM Execution Domain Fix";
3860b57cec5SDimitry Andric   }
3870b57cec5SDimitry Andric };
3880b57cec5SDimitry Andric char ARMExecutionDomainFix::ID;
3890b57cec5SDimitry Andric 
3900b57cec5SDimitry Andric } // end anonymous namespace
3910b57cec5SDimitry Andric 
3920b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(ARMExecutionDomainFix, "arm-execution-domain-fix",
3930b57cec5SDimitry Andric   "ARM Execution Domain Fix", false, false)
3940b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis)
3950b57cec5SDimitry Andric INITIALIZE_PASS_END(ARMExecutionDomainFix, "arm-execution-domain-fix",
3960b57cec5SDimitry Andric   "ARM Execution Domain Fix", false, false)
3970b57cec5SDimitry Andric 
3980b57cec5SDimitry Andric TargetPassConfig *ARMBaseTargetMachine::createPassConfig(PassManagerBase &PM) {
3990b57cec5SDimitry Andric   return new ARMPassConfig(*this, PM);
4000b57cec5SDimitry Andric }
4010b57cec5SDimitry Andric 
4020b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> ARMPassConfig::getCSEConfig() const {
4030b57cec5SDimitry Andric   return getStandardCSEConfigForOpt(TM->getOptLevel());
4040b57cec5SDimitry Andric }
4050b57cec5SDimitry Andric 
4060b57cec5SDimitry Andric void ARMPassConfig::addIRPasses() {
4070b57cec5SDimitry Andric   if (TM->Options.ThreadModel == ThreadModel::Single)
4080b57cec5SDimitry Andric     addPass(createLowerAtomicPass());
4090b57cec5SDimitry Andric   else
4100b57cec5SDimitry Andric     addPass(createAtomicExpandPass());
4110b57cec5SDimitry Andric 
4120b57cec5SDimitry Andric   // Cmpxchg instructions are often used with a subsequent comparison to
4130b57cec5SDimitry Andric   // determine whether it succeeded. We can exploit existing control-flow in
4140b57cec5SDimitry Andric   // ldrex/strex loops to simplify this, but it needs tidying up.
4150b57cec5SDimitry Andric   if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy)
4160b57cec5SDimitry Andric     addPass(createCFGSimplificationPass(
417e8d8bef9SDimitry Andric         SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true),
418e8d8bef9SDimitry Andric         [this](const Function &F) {
4190b57cec5SDimitry Andric           const auto &ST = this->TM->getSubtarget<ARMSubtarget>(F);
4200b57cec5SDimitry Andric           return ST.hasAnyDataBarrier() && !ST.isThumb1Only();
4210b57cec5SDimitry Andric         }));
4220b57cec5SDimitry Andric 
423480093f4SDimitry Andric   addPass(createMVEGatherScatterLoweringPass());
424fe6060f1SDimitry Andric   addPass(createMVELaneInterleavingPass());
425480093f4SDimitry Andric 
4260b57cec5SDimitry Andric   TargetPassConfig::addIRPasses();
4270b57cec5SDimitry Andric 
4280b57cec5SDimitry Andric   // Run the parallel DSP pass.
4290b57cec5SDimitry Andric   if (getOptLevel() == CodeGenOpt::Aggressive)
4300b57cec5SDimitry Andric     addPass(createARMParallelDSPPass());
4310b57cec5SDimitry Andric 
4320b57cec5SDimitry Andric   // Match interleaved memory accesses to ldN/stN intrinsics.
4330b57cec5SDimitry Andric   if (TM->getOptLevel() != CodeGenOpt::None)
4340b57cec5SDimitry Andric     addPass(createInterleavedAccessPass());
435480093f4SDimitry Andric 
436480093f4SDimitry Andric   // Add Control Flow Guard checks.
437480093f4SDimitry Andric   if (TM->getTargetTriple().isOSWindows())
438480093f4SDimitry Andric     addPass(createCFGuardCheckPass());
439*81ad6265SDimitry Andric 
440*81ad6265SDimitry Andric   if (TM->Options.JMCInstrument)
441*81ad6265SDimitry Andric     addPass(createJMCInstrumenterPass());
4420b57cec5SDimitry Andric }
4430b57cec5SDimitry Andric 
4440b57cec5SDimitry Andric void ARMPassConfig::addCodeGenPrepare() {
4450b57cec5SDimitry Andric   if (getOptLevel() != CodeGenOpt::None)
446480093f4SDimitry Andric     addPass(createTypePromotionPass());
4470b57cec5SDimitry Andric   TargetPassConfig::addCodeGenPrepare();
4480b57cec5SDimitry Andric }
4490b57cec5SDimitry Andric 
4500b57cec5SDimitry Andric bool ARMPassConfig::addPreISel() {
4510b57cec5SDimitry Andric   if ((TM->getOptLevel() != CodeGenOpt::None &&
4520b57cec5SDimitry Andric        EnableGlobalMerge == cl::BOU_UNSET) ||
4530b57cec5SDimitry Andric       EnableGlobalMerge == cl::BOU_TRUE) {
4540b57cec5SDimitry Andric     // FIXME: This is using the thumb1 only constant value for
4550b57cec5SDimitry Andric     // maximal global offset for merging globals. We may want
4560b57cec5SDimitry Andric     // to look into using the old value for non-thumb1 code of
4570b57cec5SDimitry Andric     // 4095 based on the TargetMachine, but this starts to become
4580b57cec5SDimitry Andric     // tricky when doing code gen per function.
4590b57cec5SDimitry Andric     bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) &&
4600b57cec5SDimitry Andric                                (EnableGlobalMerge == cl::BOU_UNSET);
4610b57cec5SDimitry Andric     // Merging of extern globals is enabled by default on non-Mach-O as we
4620b57cec5SDimitry Andric     // expect it to be generally either beneficial or harmless. On Mach-O it
4630b57cec5SDimitry Andric     // is disabled as we emit the .subsections_via_symbols directive which
4640b57cec5SDimitry Andric     // means that merging extern globals is not safe.
4650b57cec5SDimitry Andric     bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO();
4660b57cec5SDimitry Andric     addPass(createGlobalMergePass(TM, 127, OnlyOptimizeForSize,
4670b57cec5SDimitry Andric                                   MergeExternalByDefault));
4680b57cec5SDimitry Andric   }
4690b57cec5SDimitry Andric 
4708bcb0991SDimitry Andric   if (TM->getOptLevel() != CodeGenOpt::None) {
4710b57cec5SDimitry Andric     addPass(createHardwareLoopsPass());
4728bcb0991SDimitry Andric     addPass(createMVETailPredicationPass());
473fe6060f1SDimitry Andric     // FIXME: IR passes can delete address-taken basic blocks, deleting
474fe6060f1SDimitry Andric     // corresponding blockaddresses. ARMConstantPoolConstant holds references to
475fe6060f1SDimitry Andric     // address-taken basic blocks which can be invalidated if the function
476fe6060f1SDimitry Andric     // containing the blockaddress has already been codegen'd and the basic
477fe6060f1SDimitry Andric     // block is removed. Work around this by forcing all IR passes to run before
478fe6060f1SDimitry Andric     // any ISel takes place. We should have a more principled way of handling
479fe6060f1SDimitry Andric     // this. See D99707 for more details.
480fe6060f1SDimitry Andric     addPass(createBarrierNoopPass());
4818bcb0991SDimitry Andric   }
4820b57cec5SDimitry Andric 
4830b57cec5SDimitry Andric   return false;
4840b57cec5SDimitry Andric }
4850b57cec5SDimitry Andric 
4860b57cec5SDimitry Andric bool ARMPassConfig::addInstSelector() {
4870b57cec5SDimitry Andric   addPass(createARMISelDag(getARMTargetMachine(), getOptLevel()));
4880b57cec5SDimitry Andric   return false;
4890b57cec5SDimitry Andric }
4900b57cec5SDimitry Andric 
4910b57cec5SDimitry Andric bool ARMPassConfig::addIRTranslator() {
492e8d8bef9SDimitry Andric   addPass(new IRTranslator(getOptLevel()));
4930b57cec5SDimitry Andric   return false;
4940b57cec5SDimitry Andric }
4950b57cec5SDimitry Andric 
4960b57cec5SDimitry Andric bool ARMPassConfig::addLegalizeMachineIR() {
4970b57cec5SDimitry Andric   addPass(new Legalizer());
4980b57cec5SDimitry Andric   return false;
4990b57cec5SDimitry Andric }
5000b57cec5SDimitry Andric 
5010b57cec5SDimitry Andric bool ARMPassConfig::addRegBankSelect() {
5020b57cec5SDimitry Andric   addPass(new RegBankSelect());
5030b57cec5SDimitry Andric   return false;
5040b57cec5SDimitry Andric }
5050b57cec5SDimitry Andric 
5060b57cec5SDimitry Andric bool ARMPassConfig::addGlobalInstructionSelect() {
507fe6060f1SDimitry Andric   addPass(new InstructionSelect(getOptLevel()));
5080b57cec5SDimitry Andric   return false;
5090b57cec5SDimitry Andric }
5100b57cec5SDimitry Andric 
5110b57cec5SDimitry Andric void ARMPassConfig::addPreRegAlloc() {
5120b57cec5SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
513*81ad6265SDimitry Andric     if (getOptLevel() == CodeGenOpt::Aggressive)
514*81ad6265SDimitry Andric       addPass(&MachinePipelinerID);
515*81ad6265SDimitry Andric 
516fe6060f1SDimitry Andric     addPass(createMVETPAndVPTOptimisationsPass());
5175ffd83dbSDimitry Andric 
5180b57cec5SDimitry Andric     addPass(createMLxExpansionPass());
5190b57cec5SDimitry Andric 
5200b57cec5SDimitry Andric     if (EnableARMLoadStoreOpt)
5210b57cec5SDimitry Andric       addPass(createARMLoadStoreOptimizationPass(/* pre-register alloc */ true));
5220b57cec5SDimitry Andric 
5230b57cec5SDimitry Andric     if (!DisableA15SDOptimization)
5240b57cec5SDimitry Andric       addPass(createA15SDOptimizerPass());
5250b57cec5SDimitry Andric   }
5260b57cec5SDimitry Andric }
5270b57cec5SDimitry Andric 
5280b57cec5SDimitry Andric void ARMPassConfig::addPreSched2() {
5290b57cec5SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
5300b57cec5SDimitry Andric     if (EnableARMLoadStoreOpt)
5310b57cec5SDimitry Andric       addPass(createARMLoadStoreOptimizationPass());
5320b57cec5SDimitry Andric 
5330b57cec5SDimitry Andric     addPass(new ARMExecutionDomainFix());
5340b57cec5SDimitry Andric     addPass(createBreakFalseDeps());
5350b57cec5SDimitry Andric   }
5360b57cec5SDimitry Andric 
5370b57cec5SDimitry Andric   // Expand some pseudo instructions into multiple instructions to allow
5380b57cec5SDimitry Andric   // proper scheduling.
5390b57cec5SDimitry Andric   addPass(createARMExpandPseudoPass());
5400b57cec5SDimitry Andric 
5410b57cec5SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
5425ffd83dbSDimitry Andric     // When optimising for size, always run the Thumb2SizeReduction pass before
5435ffd83dbSDimitry Andric     // IfConversion. Otherwise, check whether IT blocks are restricted
5445ffd83dbSDimitry Andric     // (e.g. in v8, IfConversion depends on Thumb instruction widths)
5450b57cec5SDimitry Andric     addPass(createThumb2SizeReductionPass([this](const Function &F) {
5465ffd83dbSDimitry Andric       return this->TM->getSubtarget<ARMSubtarget>(F).hasMinSize() ||
5475ffd83dbSDimitry Andric              this->TM->getSubtarget<ARMSubtarget>(F).restrictIT();
5480b57cec5SDimitry Andric     }));
5490b57cec5SDimitry Andric 
5500b57cec5SDimitry Andric     addPass(createIfConverter([](const MachineFunction &MF) {
5510b57cec5SDimitry Andric       return !MF.getSubtarget<ARMSubtarget>().isThumb1Only();
5520b57cec5SDimitry Andric     }));
5530b57cec5SDimitry Andric   }
5540b57cec5SDimitry Andric   addPass(createThumb2ITBlockPass());
555480093f4SDimitry Andric 
556480093f4SDimitry Andric   // Add both scheduling passes to give the subtarget an opportunity to pick
557480093f4SDimitry Andric   // between them.
558480093f4SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
559480093f4SDimitry Andric     addPass(&PostMachineSchedulerID);
560480093f4SDimitry Andric     addPass(&PostRASchedulerID);
561480093f4SDimitry Andric   }
562e8d8bef9SDimitry Andric 
563349cc55cSDimitry Andric   addPass(createMVEVPTBlockPass());
564e8d8bef9SDimitry Andric   addPass(createARMIndirectThunks());
565e8d8bef9SDimitry Andric   addPass(createARMSLSHardeningPass());
5660b57cec5SDimitry Andric }
5670b57cec5SDimitry Andric 
5680b57cec5SDimitry Andric void ARMPassConfig::addPreEmitPass() {
5690b57cec5SDimitry Andric   addPass(createThumb2SizeReductionPass());
5700b57cec5SDimitry Andric 
5710b57cec5SDimitry Andric   // Constant island pass work on unbundled instructions.
5720b57cec5SDimitry Andric   addPass(createUnpackMachineBundles([](const MachineFunction &MF) {
5730b57cec5SDimitry Andric     return MF.getSubtarget<ARMSubtarget>().isThumb2();
5740b57cec5SDimitry Andric   }));
5750b57cec5SDimitry Andric 
576e8d8bef9SDimitry Andric   // Don't optimize barriers or block placement at -O0.
577e8d8bef9SDimitry Andric   if (getOptLevel() != CodeGenOpt::None) {
578e8d8bef9SDimitry Andric     addPass(createARMBlockPlacementPass());
5790b57cec5SDimitry Andric     addPass(createARMOptimizeBarriersPass());
5805ffd83dbSDimitry Andric   }
581e8d8bef9SDimitry Andric }
5820b57cec5SDimitry Andric 
5835ffd83dbSDimitry Andric void ARMPassConfig::addPreEmitPass2() {
584*81ad6265SDimitry Andric   // Inserts fixup instructions before unsafe AES operations. Instructions may
585*81ad6265SDimitry Andric   // be inserted at the start of blocks and at within blocks so this pass has to
586*81ad6265SDimitry Andric   // come before those below.
587*81ad6265SDimitry Andric   addPass(createARMFixCortexA57AES1742098Pass());
588*81ad6265SDimitry Andric   // Inserts BTIs at the start of functions and indirectly-called basic blocks,
589*81ad6265SDimitry Andric   // so passes cannot add to the start of basic blocks once this has run.
5904824e7fdSDimitry Andric   addPass(createARMBranchTargetsPass());
591*81ad6265SDimitry Andric   // Inserts Constant Islands. Block sizes cannot be increased after this point,
592*81ad6265SDimitry Andric   // as this may push the branch ranges and load offsets of accessing constant
593*81ad6265SDimitry Andric   // pools out of range..
5940b57cec5SDimitry Andric   addPass(createARMConstantIslandPass());
595*81ad6265SDimitry Andric   // Finalises Low-Overhead Loops. This replaces pseudo instructions with real
596*81ad6265SDimitry Andric   // instructions, but the pseudos all have conservative sizes so that block
597*81ad6265SDimitry Andric   // sizes will only be decreased by this pass.
5980b57cec5SDimitry Andric   addPass(createARMLowOverheadLoopsPass());
599480093f4SDimitry Andric 
600fe6060f1SDimitry Andric   if (TM->getTargetTriple().isOSWindows()) {
601480093f4SDimitry Andric     // Identify valid longjmp targets for Windows Control Flow Guard.
602480093f4SDimitry Andric     addPass(createCFGuardLongjmpPass());
603fe6060f1SDimitry Andric     // Identify valid eh continuation targets for Windows EHCont Guard.
604fe6060f1SDimitry Andric     addPass(createEHContGuardCatchretPass());
605fe6060f1SDimitry Andric   }
6060b57cec5SDimitry Andric }
607