10b57cec5SDimitry Andric //===-- ARMTargetMachine.cpp - Define TargetMachine for ARM ---------------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // 100b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 110b57cec5SDimitry Andric 120b57cec5SDimitry Andric #include "ARMTargetMachine.h" 130b57cec5SDimitry Andric #include "ARM.h" 140b57cec5SDimitry Andric #include "ARMMacroFusion.h" 150b57cec5SDimitry Andric #include "ARMSubtarget.h" 160b57cec5SDimitry Andric #include "ARMTargetObjectFile.h" 170b57cec5SDimitry Andric #include "ARMTargetTransformInfo.h" 180b57cec5SDimitry Andric #include "MCTargetDesc/ARMMCTargetDesc.h" 190b57cec5SDimitry Andric #include "TargetInfo/ARMTargetInfo.h" 200b57cec5SDimitry Andric #include "llvm/ADT/Optional.h" 210b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h" 220b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h" 230b57cec5SDimitry Andric #include "llvm/ADT/Triple.h" 240b57cec5SDimitry Andric #include "llvm/Analysis/TargetTransformInfo.h" 250b57cec5SDimitry Andric #include "llvm/CodeGen/ExecutionDomainFix.h" 26*81ad6265SDimitry Andric #include "llvm/CodeGen/GlobalISel/CSEInfo.h" 270b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/CallLowering.h" 280b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 290b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 300b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 310b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/Legalizer.h" 320b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 330b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 340b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h" 350b57cec5SDimitry Andric #include "llvm/CodeGen/MachineScheduler.h" 360b57cec5SDimitry Andric #include "llvm/CodeGen/Passes.h" 37*81ad6265SDimitry Andric #include "llvm/CodeGen/RegisterBankInfo.h" 380b57cec5SDimitry Andric #include "llvm/CodeGen/TargetPassConfig.h" 390b57cec5SDimitry Andric #include "llvm/IR/Attributes.h" 400b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h" 410b57cec5SDimitry Andric #include "llvm/IR/Function.h" 42349cc55cSDimitry Andric #include "llvm/MC/TargetRegistry.h" 430b57cec5SDimitry Andric #include "llvm/Pass.h" 44*81ad6265SDimitry Andric #include "llvm/Support/ARMTargetParser.h" 450b57cec5SDimitry Andric #include "llvm/Support/CodeGen.h" 460b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h" 470b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h" 480b57cec5SDimitry Andric #include "llvm/Support/TargetParser.h" 490b57cec5SDimitry Andric #include "llvm/Target/TargetLoweringObjectFile.h" 500b57cec5SDimitry Andric #include "llvm/Target/TargetOptions.h" 51480093f4SDimitry Andric #include "llvm/Transforms/CFGuard.h" 52fe6060f1SDimitry Andric #include "llvm/Transforms/IPO.h" 530b57cec5SDimitry Andric #include "llvm/Transforms/Scalar.h" 540b57cec5SDimitry Andric #include <cassert> 550b57cec5SDimitry Andric #include <memory> 560b57cec5SDimitry Andric #include <string> 570b57cec5SDimitry Andric 580b57cec5SDimitry Andric using namespace llvm; 590b57cec5SDimitry Andric 600b57cec5SDimitry Andric static cl::opt<bool> 610b57cec5SDimitry Andric DisableA15SDOptimization("disable-a15-sd-optimization", cl::Hidden, 620b57cec5SDimitry Andric cl::desc("Inhibit optimization of S->D register accesses on A15"), 630b57cec5SDimitry Andric cl::init(false)); 640b57cec5SDimitry Andric 650b57cec5SDimitry Andric static cl::opt<bool> 660b57cec5SDimitry Andric EnableAtomicTidy("arm-atomic-cfg-tidy", cl::Hidden, 670b57cec5SDimitry Andric cl::desc("Run SimplifyCFG after expanding atomic operations" 680b57cec5SDimitry Andric " to make use of cmpxchg flow-based information"), 690b57cec5SDimitry Andric cl::init(true)); 700b57cec5SDimitry Andric 710b57cec5SDimitry Andric static cl::opt<bool> 720b57cec5SDimitry Andric EnableARMLoadStoreOpt("arm-load-store-opt", cl::Hidden, 730b57cec5SDimitry Andric cl::desc("Enable ARM load/store optimization pass"), 740b57cec5SDimitry Andric cl::init(true)); 750b57cec5SDimitry Andric 760b57cec5SDimitry Andric // FIXME: Unify control over GlobalMerge. 770b57cec5SDimitry Andric static cl::opt<cl::boolOrDefault> 780b57cec5SDimitry Andric EnableGlobalMerge("arm-global-merge", cl::Hidden, 790b57cec5SDimitry Andric cl::desc("Enable the global merge pass")); 800b57cec5SDimitry Andric 810b57cec5SDimitry Andric namespace llvm { 820b57cec5SDimitry Andric void initializeARMExecutionDomainFixPass(PassRegistry&); 830b57cec5SDimitry Andric } 840b57cec5SDimitry Andric 85480093f4SDimitry Andric extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeARMTarget() { 860b57cec5SDimitry Andric // Register the target. 870b57cec5SDimitry Andric RegisterTargetMachine<ARMLETargetMachine> X(getTheARMLETarget()); 880b57cec5SDimitry Andric RegisterTargetMachine<ARMLETargetMachine> A(getTheThumbLETarget()); 890b57cec5SDimitry Andric RegisterTargetMachine<ARMBETargetMachine> Y(getTheARMBETarget()); 900b57cec5SDimitry Andric RegisterTargetMachine<ARMBETargetMachine> B(getTheThumbBETarget()); 910b57cec5SDimitry Andric 920b57cec5SDimitry Andric PassRegistry &Registry = *PassRegistry::getPassRegistry(); 930b57cec5SDimitry Andric initializeGlobalISel(Registry); 940b57cec5SDimitry Andric initializeARMLoadStoreOptPass(Registry); 950b57cec5SDimitry Andric initializeARMPreAllocLoadStoreOptPass(Registry); 960b57cec5SDimitry Andric initializeARMParallelDSPPass(Registry); 974824e7fdSDimitry Andric initializeARMBranchTargetsPass(Registry); 980b57cec5SDimitry Andric initializeARMConstantIslandsPass(Registry); 990b57cec5SDimitry Andric initializeARMExecutionDomainFixPass(Registry); 1000b57cec5SDimitry Andric initializeARMExpandPseudoPass(Registry); 1010b57cec5SDimitry Andric initializeThumb2SizeReducePass(Registry); 1020b57cec5SDimitry Andric initializeMVEVPTBlockPass(Registry); 103fe6060f1SDimitry Andric initializeMVETPAndVPTOptimisationsPass(Registry); 1048bcb0991SDimitry Andric initializeMVETailPredicationPass(Registry); 1050b57cec5SDimitry Andric initializeARMLowOverheadLoopsPass(Registry); 106e8d8bef9SDimitry Andric initializeARMBlockPlacementPass(Registry); 107480093f4SDimitry Andric initializeMVEGatherScatterLoweringPass(Registry); 108e8d8bef9SDimitry Andric initializeARMSLSHardeningPass(Registry); 109fe6060f1SDimitry Andric initializeMVELaneInterleavingPass(Registry); 110*81ad6265SDimitry Andric initializeARMFixCortexA57AES1742098Pass(Registry); 1110b57cec5SDimitry Andric } 1120b57cec5SDimitry Andric 1130b57cec5SDimitry Andric static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 1140b57cec5SDimitry Andric if (TT.isOSBinFormatMachO()) 1158bcb0991SDimitry Andric return std::make_unique<TargetLoweringObjectFileMachO>(); 1160b57cec5SDimitry Andric if (TT.isOSWindows()) 1178bcb0991SDimitry Andric return std::make_unique<TargetLoweringObjectFileCOFF>(); 1188bcb0991SDimitry Andric return std::make_unique<ARMElfTargetObjectFile>(); 1190b57cec5SDimitry Andric } 1200b57cec5SDimitry Andric 1210b57cec5SDimitry Andric static ARMBaseTargetMachine::ARMABI 1220b57cec5SDimitry Andric computeTargetABI(const Triple &TT, StringRef CPU, 1230b57cec5SDimitry Andric const TargetOptions &Options) { 1240b57cec5SDimitry Andric StringRef ABIName = Options.MCOptions.getABIName(); 1250b57cec5SDimitry Andric 1260b57cec5SDimitry Andric if (ABIName.empty()) 1270b57cec5SDimitry Andric ABIName = ARM::computeDefaultTargetABI(TT, CPU); 1280b57cec5SDimitry Andric 1290b57cec5SDimitry Andric if (ABIName == "aapcs16") 1300b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_AAPCS16; 1310b57cec5SDimitry Andric else if (ABIName.startswith("aapcs")) 1320b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_AAPCS; 1330b57cec5SDimitry Andric else if (ABIName.startswith("apcs")) 1340b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_APCS; 1350b57cec5SDimitry Andric 1360b57cec5SDimitry Andric llvm_unreachable("Unhandled/unknown ABI Name!"); 1370b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_UNKNOWN; 1380b57cec5SDimitry Andric } 1390b57cec5SDimitry Andric 1400b57cec5SDimitry Andric static std::string computeDataLayout(const Triple &TT, StringRef CPU, 1410b57cec5SDimitry Andric const TargetOptions &Options, 1420b57cec5SDimitry Andric bool isLittle) { 1430b57cec5SDimitry Andric auto ABI = computeTargetABI(TT, CPU, Options); 1440b57cec5SDimitry Andric std::string Ret; 1450b57cec5SDimitry Andric 1460b57cec5SDimitry Andric if (isLittle) 1470b57cec5SDimitry Andric // Little endian. 1480b57cec5SDimitry Andric Ret += "e"; 1490b57cec5SDimitry Andric else 1500b57cec5SDimitry Andric // Big endian. 1510b57cec5SDimitry Andric Ret += "E"; 1520b57cec5SDimitry Andric 1530b57cec5SDimitry Andric Ret += DataLayout::getManglingComponent(TT); 1540b57cec5SDimitry Andric 1550b57cec5SDimitry Andric // Pointers are 32 bits and aligned to 32 bits. 1560b57cec5SDimitry Andric Ret += "-p:32:32"; 1570b57cec5SDimitry Andric 1580b57cec5SDimitry Andric // Function pointers are aligned to 8 bits (because the LSB stores the 1590b57cec5SDimitry Andric // ARM/Thumb state). 1600b57cec5SDimitry Andric Ret += "-Fi8"; 1610b57cec5SDimitry Andric 1620b57cec5SDimitry Andric // ABIs other than APCS have 64 bit integers with natural alignment. 1630b57cec5SDimitry Andric if (ABI != ARMBaseTargetMachine::ARM_ABI_APCS) 1640b57cec5SDimitry Andric Ret += "-i64:64"; 1650b57cec5SDimitry Andric 1660b57cec5SDimitry Andric // We have 64 bits floats. The APCS ABI requires them to be aligned to 32 1670b57cec5SDimitry Andric // bits, others to 64 bits. We always try to align to 64 bits. 1680b57cec5SDimitry Andric if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS) 1690b57cec5SDimitry Andric Ret += "-f64:32:64"; 1700b57cec5SDimitry Andric 1710b57cec5SDimitry Andric // We have 128 and 64 bit vectors. The APCS ABI aligns them to 32 bits, others 1720b57cec5SDimitry Andric // to 64. We always ty to give them natural alignment. 1730b57cec5SDimitry Andric if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS) 1740b57cec5SDimitry Andric Ret += "-v64:32:64-v128:32:128"; 1750b57cec5SDimitry Andric else if (ABI != ARMBaseTargetMachine::ARM_ABI_AAPCS16) 1760b57cec5SDimitry Andric Ret += "-v128:64:128"; 1770b57cec5SDimitry Andric 1780b57cec5SDimitry Andric // Try to align aggregates to 32 bits (the default is 64 bits, which has no 1790b57cec5SDimitry Andric // particular hardware support on 32-bit ARM). 1800b57cec5SDimitry Andric Ret += "-a:0:32"; 1810b57cec5SDimitry Andric 1820b57cec5SDimitry Andric // Integer registers are 32 bits. 1830b57cec5SDimitry Andric Ret += "-n32"; 1840b57cec5SDimitry Andric 1850b57cec5SDimitry Andric // The stack is 128 bit aligned on NaCl, 64 bit aligned on AAPCS and 32 bit 1860b57cec5SDimitry Andric // aligned everywhere else. 1870b57cec5SDimitry Andric if (TT.isOSNaCl() || ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16) 1880b57cec5SDimitry Andric Ret += "-S128"; 1890b57cec5SDimitry Andric else if (ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS) 1900b57cec5SDimitry Andric Ret += "-S64"; 1910b57cec5SDimitry Andric else 1920b57cec5SDimitry Andric Ret += "-S32"; 1930b57cec5SDimitry Andric 1940b57cec5SDimitry Andric return Ret; 1950b57cec5SDimitry Andric } 1960b57cec5SDimitry Andric 1970b57cec5SDimitry Andric static Reloc::Model getEffectiveRelocModel(const Triple &TT, 1980b57cec5SDimitry Andric Optional<Reloc::Model> RM) { 199*81ad6265SDimitry Andric if (!RM) 2000b57cec5SDimitry Andric // Default relocation model on Darwin is PIC. 2010b57cec5SDimitry Andric return TT.isOSBinFormatMachO() ? Reloc::PIC_ : Reloc::Static; 2020b57cec5SDimitry Andric 2030b57cec5SDimitry Andric if (*RM == Reloc::ROPI || *RM == Reloc::RWPI || *RM == Reloc::ROPI_RWPI) 2040b57cec5SDimitry Andric assert(TT.isOSBinFormatELF() && 2050b57cec5SDimitry Andric "ROPI/RWPI currently only supported for ELF"); 2060b57cec5SDimitry Andric 2070b57cec5SDimitry Andric // DynamicNoPIC is only used on darwin. 2080b57cec5SDimitry Andric if (*RM == Reloc::DynamicNoPIC && !TT.isOSDarwin()) 2090b57cec5SDimitry Andric return Reloc::Static; 2100b57cec5SDimitry Andric 2110b57cec5SDimitry Andric return *RM; 2120b57cec5SDimitry Andric } 2130b57cec5SDimitry Andric 2140b57cec5SDimitry Andric /// Create an ARM architecture model. 2150b57cec5SDimitry Andric /// 2160b57cec5SDimitry Andric ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT, 2170b57cec5SDimitry Andric StringRef CPU, StringRef FS, 2180b57cec5SDimitry Andric const TargetOptions &Options, 2190b57cec5SDimitry Andric Optional<Reloc::Model> RM, 2200b57cec5SDimitry Andric Optional<CodeModel::Model> CM, 2210b57cec5SDimitry Andric CodeGenOpt::Level OL, bool isLittle) 2220b57cec5SDimitry Andric : LLVMTargetMachine(T, computeDataLayout(TT, CPU, Options, isLittle), TT, 2230b57cec5SDimitry Andric CPU, FS, Options, getEffectiveRelocModel(TT, RM), 2240b57cec5SDimitry Andric getEffectiveCodeModel(CM, CodeModel::Small), OL), 2250b57cec5SDimitry Andric TargetABI(computeTargetABI(TT, CPU, Options)), 2260b57cec5SDimitry Andric TLOF(createTLOF(getTargetTriple())), isLittle(isLittle) { 2270b57cec5SDimitry Andric 2280b57cec5SDimitry Andric // Default to triple-appropriate float ABI 2290b57cec5SDimitry Andric if (Options.FloatABIType == FloatABI::Default) { 2300b57cec5SDimitry Andric if (isTargetHardFloat()) 2310b57cec5SDimitry Andric this->Options.FloatABIType = FloatABI::Hard; 2320b57cec5SDimitry Andric else 2330b57cec5SDimitry Andric this->Options.FloatABIType = FloatABI::Soft; 2340b57cec5SDimitry Andric } 2350b57cec5SDimitry Andric 2360b57cec5SDimitry Andric // Default to triple-appropriate EABI 2370b57cec5SDimitry Andric if (Options.EABIVersion == EABI::Default || 2380b57cec5SDimitry Andric Options.EABIVersion == EABI::Unknown) { 2390b57cec5SDimitry Andric // musl is compatible with glibc with regard to EABI version 2400b57cec5SDimitry Andric if ((TargetTriple.getEnvironment() == Triple::GNUEABI || 2410b57cec5SDimitry Andric TargetTriple.getEnvironment() == Triple::GNUEABIHF || 2420b57cec5SDimitry Andric TargetTriple.getEnvironment() == Triple::MuslEABI || 2430b57cec5SDimitry Andric TargetTriple.getEnvironment() == Triple::MuslEABIHF) && 2440b57cec5SDimitry Andric !(TargetTriple.isOSWindows() || TargetTriple.isOSDarwin())) 2450b57cec5SDimitry Andric this->Options.EABIVersion = EABI::GNU; 2460b57cec5SDimitry Andric else 2470b57cec5SDimitry Andric this->Options.EABIVersion = EABI::EABI5; 2480b57cec5SDimitry Andric } 2490b57cec5SDimitry Andric 2500b57cec5SDimitry Andric if (TT.isOSBinFormatMachO()) { 2510b57cec5SDimitry Andric this->Options.TrapUnreachable = true; 2520b57cec5SDimitry Andric this->Options.NoTrapAfterNoreturn = true; 2530b57cec5SDimitry Andric } 2540b57cec5SDimitry Andric 2555ffd83dbSDimitry Andric // ARM supports the debug entry values. 2565ffd83dbSDimitry Andric setSupportsDebugEntryValues(true); 2575ffd83dbSDimitry Andric 2580b57cec5SDimitry Andric initAsmInfo(); 2595ffd83dbSDimitry Andric 2605ffd83dbSDimitry Andric // ARM supports the MachineOutliner. 2615ffd83dbSDimitry Andric setMachineOutliner(true); 262e8d8bef9SDimitry Andric setSupportsDefaultOutlining(true); 2630b57cec5SDimitry Andric } 2640b57cec5SDimitry Andric 2650b57cec5SDimitry Andric ARMBaseTargetMachine::~ARMBaseTargetMachine() = default; 2660b57cec5SDimitry Andric 2670b57cec5SDimitry Andric const ARMSubtarget * 2680b57cec5SDimitry Andric ARMBaseTargetMachine::getSubtargetImpl(const Function &F) const { 2690b57cec5SDimitry Andric Attribute CPUAttr = F.getFnAttribute("target-cpu"); 2700b57cec5SDimitry Andric Attribute FSAttr = F.getFnAttribute("target-features"); 2710b57cec5SDimitry Andric 272e8d8bef9SDimitry Andric std::string CPU = 273e8d8bef9SDimitry Andric CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU; 274e8d8bef9SDimitry Andric std::string FS = 275e8d8bef9SDimitry Andric FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS; 2760b57cec5SDimitry Andric 2770b57cec5SDimitry Andric // FIXME: This is related to the code below to reset the target options, 2780b57cec5SDimitry Andric // we need to know whether or not the soft float flag is set on the 2790b57cec5SDimitry Andric // function before we can generate a subtarget. We also need to use 2800b57cec5SDimitry Andric // it as a key for the subtarget since that can be the only difference 2810b57cec5SDimitry Andric // between two functions. 282fe6060f1SDimitry Andric bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsBool(); 2830b57cec5SDimitry Andric // If the soft float attribute is set on the function turn on the soft float 2840b57cec5SDimitry Andric // subtarget feature. 2850b57cec5SDimitry Andric if (SoftFloat) 2860b57cec5SDimitry Andric FS += FS.empty() ? "+soft-float" : ",+soft-float"; 2870b57cec5SDimitry Andric 2880b57cec5SDimitry Andric // Use the optminsize to identify the subtarget, but don't use it in the 2890b57cec5SDimitry Andric // feature string. 2900b57cec5SDimitry Andric std::string Key = CPU + FS; 2910b57cec5SDimitry Andric if (F.hasMinSize()) 2920b57cec5SDimitry Andric Key += "+minsize"; 2930b57cec5SDimitry Andric 2940b57cec5SDimitry Andric auto &I = SubtargetMap[Key]; 2950b57cec5SDimitry Andric if (!I) { 2960b57cec5SDimitry Andric // This needs to be done before we create a new subtarget since any 2970b57cec5SDimitry Andric // creation will depend on the TM and the code generation flags on the 2980b57cec5SDimitry Andric // function that reside in TargetOptions. 2990b57cec5SDimitry Andric resetTargetOptions(F); 3008bcb0991SDimitry Andric I = std::make_unique<ARMSubtarget>(TargetTriple, CPU, FS, *this, isLittle, 3010b57cec5SDimitry Andric F.hasMinSize()); 3020b57cec5SDimitry Andric 3030b57cec5SDimitry Andric if (!I->isThumb() && !I->hasARMOps()) 3040b57cec5SDimitry Andric F.getContext().emitError("Function '" + F.getName() + "' uses ARM " 3050b57cec5SDimitry Andric "instructions, but the target does not support ARM mode execution."); 3060b57cec5SDimitry Andric } 3070b57cec5SDimitry Andric 3080b57cec5SDimitry Andric return I.get(); 3090b57cec5SDimitry Andric } 3100b57cec5SDimitry Andric 3110b57cec5SDimitry Andric TargetTransformInfo 312*81ad6265SDimitry Andric ARMBaseTargetMachine::getTargetTransformInfo(const Function &F) const { 3130b57cec5SDimitry Andric return TargetTransformInfo(ARMTTIImpl(this, F)); 3140b57cec5SDimitry Andric } 3150b57cec5SDimitry Andric 3160b57cec5SDimitry Andric ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT, 3170b57cec5SDimitry Andric StringRef CPU, StringRef FS, 3180b57cec5SDimitry Andric const TargetOptions &Options, 3190b57cec5SDimitry Andric Optional<Reloc::Model> RM, 3200b57cec5SDimitry Andric Optional<CodeModel::Model> CM, 3210b57cec5SDimitry Andric CodeGenOpt::Level OL, bool JIT) 3220b57cec5SDimitry Andric : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {} 3230b57cec5SDimitry Andric 3240b57cec5SDimitry Andric ARMBETargetMachine::ARMBETargetMachine(const Target &T, const Triple &TT, 3250b57cec5SDimitry Andric StringRef CPU, StringRef FS, 3260b57cec5SDimitry Andric const TargetOptions &Options, 3270b57cec5SDimitry Andric Optional<Reloc::Model> RM, 3280b57cec5SDimitry Andric Optional<CodeModel::Model> CM, 3290b57cec5SDimitry Andric CodeGenOpt::Level OL, bool JIT) 3300b57cec5SDimitry Andric : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {} 3310b57cec5SDimitry Andric 3320b57cec5SDimitry Andric namespace { 3330b57cec5SDimitry Andric 3340b57cec5SDimitry Andric /// ARM Code Generator Pass Configuration Options. 3350b57cec5SDimitry Andric class ARMPassConfig : public TargetPassConfig { 3360b57cec5SDimitry Andric public: 3370b57cec5SDimitry Andric ARMPassConfig(ARMBaseTargetMachine &TM, PassManagerBase &PM) 338480093f4SDimitry Andric : TargetPassConfig(TM, PM) {} 3390b57cec5SDimitry Andric 3400b57cec5SDimitry Andric ARMBaseTargetMachine &getARMTargetMachine() const { 3410b57cec5SDimitry Andric return getTM<ARMBaseTargetMachine>(); 3420b57cec5SDimitry Andric } 3430b57cec5SDimitry Andric 3440b57cec5SDimitry Andric ScheduleDAGInstrs * 3450b57cec5SDimitry Andric createMachineScheduler(MachineSchedContext *C) const override { 3460b57cec5SDimitry Andric ScheduleDAGMILive *DAG = createGenericSchedLive(C); 3470b57cec5SDimitry Andric // add DAG Mutations here. 3480b57cec5SDimitry Andric const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>(); 3490b57cec5SDimitry Andric if (ST.hasFusion()) 3500b57cec5SDimitry Andric DAG->addMutation(createARMMacroFusionDAGMutation()); 3510b57cec5SDimitry Andric return DAG; 3520b57cec5SDimitry Andric } 3530b57cec5SDimitry Andric 3540b57cec5SDimitry Andric ScheduleDAGInstrs * 3550b57cec5SDimitry Andric createPostMachineScheduler(MachineSchedContext *C) const override { 3560b57cec5SDimitry Andric ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 3570b57cec5SDimitry Andric // add DAG Mutations here. 3580b57cec5SDimitry Andric const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>(); 3590b57cec5SDimitry Andric if (ST.hasFusion()) 3600b57cec5SDimitry Andric DAG->addMutation(createARMMacroFusionDAGMutation()); 3610b57cec5SDimitry Andric return DAG; 3620b57cec5SDimitry Andric } 3630b57cec5SDimitry Andric 3640b57cec5SDimitry Andric void addIRPasses() override; 3650b57cec5SDimitry Andric void addCodeGenPrepare() override; 3660b57cec5SDimitry Andric bool addPreISel() override; 3670b57cec5SDimitry Andric bool addInstSelector() override; 3680b57cec5SDimitry Andric bool addIRTranslator() override; 3690b57cec5SDimitry Andric bool addLegalizeMachineIR() override; 3700b57cec5SDimitry Andric bool addRegBankSelect() override; 3710b57cec5SDimitry Andric bool addGlobalInstructionSelect() override; 3720b57cec5SDimitry Andric void addPreRegAlloc() override; 3730b57cec5SDimitry Andric void addPreSched2() override; 3740b57cec5SDimitry Andric void addPreEmitPass() override; 3755ffd83dbSDimitry Andric void addPreEmitPass2() override; 3760b57cec5SDimitry Andric 3770b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 3780b57cec5SDimitry Andric }; 3790b57cec5SDimitry Andric 3800b57cec5SDimitry Andric class ARMExecutionDomainFix : public ExecutionDomainFix { 3810b57cec5SDimitry Andric public: 3820b57cec5SDimitry Andric static char ID; 3830b57cec5SDimitry Andric ARMExecutionDomainFix() : ExecutionDomainFix(ID, ARM::DPRRegClass) {} 3840b57cec5SDimitry Andric StringRef getPassName() const override { 3850b57cec5SDimitry Andric return "ARM Execution Domain Fix"; 3860b57cec5SDimitry Andric } 3870b57cec5SDimitry Andric }; 3880b57cec5SDimitry Andric char ARMExecutionDomainFix::ID; 3890b57cec5SDimitry Andric 3900b57cec5SDimitry Andric } // end anonymous namespace 3910b57cec5SDimitry Andric 3920b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(ARMExecutionDomainFix, "arm-execution-domain-fix", 3930b57cec5SDimitry Andric "ARM Execution Domain Fix", false, false) 3940b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) 3950b57cec5SDimitry Andric INITIALIZE_PASS_END(ARMExecutionDomainFix, "arm-execution-domain-fix", 3960b57cec5SDimitry Andric "ARM Execution Domain Fix", false, false) 3970b57cec5SDimitry Andric 3980b57cec5SDimitry Andric TargetPassConfig *ARMBaseTargetMachine::createPassConfig(PassManagerBase &PM) { 3990b57cec5SDimitry Andric return new ARMPassConfig(*this, PM); 4000b57cec5SDimitry Andric } 4010b57cec5SDimitry Andric 4020b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> ARMPassConfig::getCSEConfig() const { 4030b57cec5SDimitry Andric return getStandardCSEConfigForOpt(TM->getOptLevel()); 4040b57cec5SDimitry Andric } 4050b57cec5SDimitry Andric 4060b57cec5SDimitry Andric void ARMPassConfig::addIRPasses() { 4070b57cec5SDimitry Andric if (TM->Options.ThreadModel == ThreadModel::Single) 4080b57cec5SDimitry Andric addPass(createLowerAtomicPass()); 4090b57cec5SDimitry Andric else 4100b57cec5SDimitry Andric addPass(createAtomicExpandPass()); 4110b57cec5SDimitry Andric 4120b57cec5SDimitry Andric // Cmpxchg instructions are often used with a subsequent comparison to 4130b57cec5SDimitry Andric // determine whether it succeeded. We can exploit existing control-flow in 4140b57cec5SDimitry Andric // ldrex/strex loops to simplify this, but it needs tidying up. 4150b57cec5SDimitry Andric if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy) 4160b57cec5SDimitry Andric addPass(createCFGSimplificationPass( 417e8d8bef9SDimitry Andric SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true), 418e8d8bef9SDimitry Andric [this](const Function &F) { 4190b57cec5SDimitry Andric const auto &ST = this->TM->getSubtarget<ARMSubtarget>(F); 4200b57cec5SDimitry Andric return ST.hasAnyDataBarrier() && !ST.isThumb1Only(); 4210b57cec5SDimitry Andric })); 4220b57cec5SDimitry Andric 423480093f4SDimitry Andric addPass(createMVEGatherScatterLoweringPass()); 424fe6060f1SDimitry Andric addPass(createMVELaneInterleavingPass()); 425480093f4SDimitry Andric 4260b57cec5SDimitry Andric TargetPassConfig::addIRPasses(); 4270b57cec5SDimitry Andric 4280b57cec5SDimitry Andric // Run the parallel DSP pass. 4290b57cec5SDimitry Andric if (getOptLevel() == CodeGenOpt::Aggressive) 4300b57cec5SDimitry Andric addPass(createARMParallelDSPPass()); 4310b57cec5SDimitry Andric 4320b57cec5SDimitry Andric // Match interleaved memory accesses to ldN/stN intrinsics. 4330b57cec5SDimitry Andric if (TM->getOptLevel() != CodeGenOpt::None) 4340b57cec5SDimitry Andric addPass(createInterleavedAccessPass()); 435480093f4SDimitry Andric 436480093f4SDimitry Andric // Add Control Flow Guard checks. 437480093f4SDimitry Andric if (TM->getTargetTriple().isOSWindows()) 438480093f4SDimitry Andric addPass(createCFGuardCheckPass()); 439*81ad6265SDimitry Andric 440*81ad6265SDimitry Andric if (TM->Options.JMCInstrument) 441*81ad6265SDimitry Andric addPass(createJMCInstrumenterPass()); 4420b57cec5SDimitry Andric } 4430b57cec5SDimitry Andric 4440b57cec5SDimitry Andric void ARMPassConfig::addCodeGenPrepare() { 4450b57cec5SDimitry Andric if (getOptLevel() != CodeGenOpt::None) 446480093f4SDimitry Andric addPass(createTypePromotionPass()); 4470b57cec5SDimitry Andric TargetPassConfig::addCodeGenPrepare(); 4480b57cec5SDimitry Andric } 4490b57cec5SDimitry Andric 4500b57cec5SDimitry Andric bool ARMPassConfig::addPreISel() { 4510b57cec5SDimitry Andric if ((TM->getOptLevel() != CodeGenOpt::None && 4520b57cec5SDimitry Andric EnableGlobalMerge == cl::BOU_UNSET) || 4530b57cec5SDimitry Andric EnableGlobalMerge == cl::BOU_TRUE) { 4540b57cec5SDimitry Andric // FIXME: This is using the thumb1 only constant value for 4550b57cec5SDimitry Andric // maximal global offset for merging globals. We may want 4560b57cec5SDimitry Andric // to look into using the old value for non-thumb1 code of 4570b57cec5SDimitry Andric // 4095 based on the TargetMachine, but this starts to become 4580b57cec5SDimitry Andric // tricky when doing code gen per function. 4590b57cec5SDimitry Andric bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) && 4600b57cec5SDimitry Andric (EnableGlobalMerge == cl::BOU_UNSET); 4610b57cec5SDimitry Andric // Merging of extern globals is enabled by default on non-Mach-O as we 4620b57cec5SDimitry Andric // expect it to be generally either beneficial or harmless. On Mach-O it 4630b57cec5SDimitry Andric // is disabled as we emit the .subsections_via_symbols directive which 4640b57cec5SDimitry Andric // means that merging extern globals is not safe. 4650b57cec5SDimitry Andric bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO(); 4660b57cec5SDimitry Andric addPass(createGlobalMergePass(TM, 127, OnlyOptimizeForSize, 4670b57cec5SDimitry Andric MergeExternalByDefault)); 4680b57cec5SDimitry Andric } 4690b57cec5SDimitry Andric 4708bcb0991SDimitry Andric if (TM->getOptLevel() != CodeGenOpt::None) { 4710b57cec5SDimitry Andric addPass(createHardwareLoopsPass()); 4728bcb0991SDimitry Andric addPass(createMVETailPredicationPass()); 473fe6060f1SDimitry Andric // FIXME: IR passes can delete address-taken basic blocks, deleting 474fe6060f1SDimitry Andric // corresponding blockaddresses. ARMConstantPoolConstant holds references to 475fe6060f1SDimitry Andric // address-taken basic blocks which can be invalidated if the function 476fe6060f1SDimitry Andric // containing the blockaddress has already been codegen'd and the basic 477fe6060f1SDimitry Andric // block is removed. Work around this by forcing all IR passes to run before 478fe6060f1SDimitry Andric // any ISel takes place. We should have a more principled way of handling 479fe6060f1SDimitry Andric // this. See D99707 for more details. 480fe6060f1SDimitry Andric addPass(createBarrierNoopPass()); 4818bcb0991SDimitry Andric } 4820b57cec5SDimitry Andric 4830b57cec5SDimitry Andric return false; 4840b57cec5SDimitry Andric } 4850b57cec5SDimitry Andric 4860b57cec5SDimitry Andric bool ARMPassConfig::addInstSelector() { 4870b57cec5SDimitry Andric addPass(createARMISelDag(getARMTargetMachine(), getOptLevel())); 4880b57cec5SDimitry Andric return false; 4890b57cec5SDimitry Andric } 4900b57cec5SDimitry Andric 4910b57cec5SDimitry Andric bool ARMPassConfig::addIRTranslator() { 492e8d8bef9SDimitry Andric addPass(new IRTranslator(getOptLevel())); 4930b57cec5SDimitry Andric return false; 4940b57cec5SDimitry Andric } 4950b57cec5SDimitry Andric 4960b57cec5SDimitry Andric bool ARMPassConfig::addLegalizeMachineIR() { 4970b57cec5SDimitry Andric addPass(new Legalizer()); 4980b57cec5SDimitry Andric return false; 4990b57cec5SDimitry Andric } 5000b57cec5SDimitry Andric 5010b57cec5SDimitry Andric bool ARMPassConfig::addRegBankSelect() { 5020b57cec5SDimitry Andric addPass(new RegBankSelect()); 5030b57cec5SDimitry Andric return false; 5040b57cec5SDimitry Andric } 5050b57cec5SDimitry Andric 5060b57cec5SDimitry Andric bool ARMPassConfig::addGlobalInstructionSelect() { 507fe6060f1SDimitry Andric addPass(new InstructionSelect(getOptLevel())); 5080b57cec5SDimitry Andric return false; 5090b57cec5SDimitry Andric } 5100b57cec5SDimitry Andric 5110b57cec5SDimitry Andric void ARMPassConfig::addPreRegAlloc() { 5120b57cec5SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 513*81ad6265SDimitry Andric if (getOptLevel() == CodeGenOpt::Aggressive) 514*81ad6265SDimitry Andric addPass(&MachinePipelinerID); 515*81ad6265SDimitry Andric 516fe6060f1SDimitry Andric addPass(createMVETPAndVPTOptimisationsPass()); 5175ffd83dbSDimitry Andric 5180b57cec5SDimitry Andric addPass(createMLxExpansionPass()); 5190b57cec5SDimitry Andric 5200b57cec5SDimitry Andric if (EnableARMLoadStoreOpt) 5210b57cec5SDimitry Andric addPass(createARMLoadStoreOptimizationPass(/* pre-register alloc */ true)); 5220b57cec5SDimitry Andric 5230b57cec5SDimitry Andric if (!DisableA15SDOptimization) 5240b57cec5SDimitry Andric addPass(createA15SDOptimizerPass()); 5250b57cec5SDimitry Andric } 5260b57cec5SDimitry Andric } 5270b57cec5SDimitry Andric 5280b57cec5SDimitry Andric void ARMPassConfig::addPreSched2() { 5290b57cec5SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 5300b57cec5SDimitry Andric if (EnableARMLoadStoreOpt) 5310b57cec5SDimitry Andric addPass(createARMLoadStoreOptimizationPass()); 5320b57cec5SDimitry Andric 5330b57cec5SDimitry Andric addPass(new ARMExecutionDomainFix()); 5340b57cec5SDimitry Andric addPass(createBreakFalseDeps()); 5350b57cec5SDimitry Andric } 5360b57cec5SDimitry Andric 5370b57cec5SDimitry Andric // Expand some pseudo instructions into multiple instructions to allow 5380b57cec5SDimitry Andric // proper scheduling. 5390b57cec5SDimitry Andric addPass(createARMExpandPseudoPass()); 5400b57cec5SDimitry Andric 5410b57cec5SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 5425ffd83dbSDimitry Andric // When optimising for size, always run the Thumb2SizeReduction pass before 5435ffd83dbSDimitry Andric // IfConversion. Otherwise, check whether IT blocks are restricted 5445ffd83dbSDimitry Andric // (e.g. in v8, IfConversion depends on Thumb instruction widths) 5450b57cec5SDimitry Andric addPass(createThumb2SizeReductionPass([this](const Function &F) { 5465ffd83dbSDimitry Andric return this->TM->getSubtarget<ARMSubtarget>(F).hasMinSize() || 5475ffd83dbSDimitry Andric this->TM->getSubtarget<ARMSubtarget>(F).restrictIT(); 5480b57cec5SDimitry Andric })); 5490b57cec5SDimitry Andric 5500b57cec5SDimitry Andric addPass(createIfConverter([](const MachineFunction &MF) { 5510b57cec5SDimitry Andric return !MF.getSubtarget<ARMSubtarget>().isThumb1Only(); 5520b57cec5SDimitry Andric })); 5530b57cec5SDimitry Andric } 5540b57cec5SDimitry Andric addPass(createThumb2ITBlockPass()); 555480093f4SDimitry Andric 556480093f4SDimitry Andric // Add both scheduling passes to give the subtarget an opportunity to pick 557480093f4SDimitry Andric // between them. 558480093f4SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 559480093f4SDimitry Andric addPass(&PostMachineSchedulerID); 560480093f4SDimitry Andric addPass(&PostRASchedulerID); 561480093f4SDimitry Andric } 562e8d8bef9SDimitry Andric 563349cc55cSDimitry Andric addPass(createMVEVPTBlockPass()); 564e8d8bef9SDimitry Andric addPass(createARMIndirectThunks()); 565e8d8bef9SDimitry Andric addPass(createARMSLSHardeningPass()); 5660b57cec5SDimitry Andric } 5670b57cec5SDimitry Andric 5680b57cec5SDimitry Andric void ARMPassConfig::addPreEmitPass() { 5690b57cec5SDimitry Andric addPass(createThumb2SizeReductionPass()); 5700b57cec5SDimitry Andric 5710b57cec5SDimitry Andric // Constant island pass work on unbundled instructions. 5720b57cec5SDimitry Andric addPass(createUnpackMachineBundles([](const MachineFunction &MF) { 5730b57cec5SDimitry Andric return MF.getSubtarget<ARMSubtarget>().isThumb2(); 5740b57cec5SDimitry Andric })); 5750b57cec5SDimitry Andric 576e8d8bef9SDimitry Andric // Don't optimize barriers or block placement at -O0. 577e8d8bef9SDimitry Andric if (getOptLevel() != CodeGenOpt::None) { 578e8d8bef9SDimitry Andric addPass(createARMBlockPlacementPass()); 5790b57cec5SDimitry Andric addPass(createARMOptimizeBarriersPass()); 5805ffd83dbSDimitry Andric } 581e8d8bef9SDimitry Andric } 5820b57cec5SDimitry Andric 5835ffd83dbSDimitry Andric void ARMPassConfig::addPreEmitPass2() { 584*81ad6265SDimitry Andric // Inserts fixup instructions before unsafe AES operations. Instructions may 585*81ad6265SDimitry Andric // be inserted at the start of blocks and at within blocks so this pass has to 586*81ad6265SDimitry Andric // come before those below. 587*81ad6265SDimitry Andric addPass(createARMFixCortexA57AES1742098Pass()); 588*81ad6265SDimitry Andric // Inserts BTIs at the start of functions and indirectly-called basic blocks, 589*81ad6265SDimitry Andric // so passes cannot add to the start of basic blocks once this has run. 5904824e7fdSDimitry Andric addPass(createARMBranchTargetsPass()); 591*81ad6265SDimitry Andric // Inserts Constant Islands. Block sizes cannot be increased after this point, 592*81ad6265SDimitry Andric // as this may push the branch ranges and load offsets of accessing constant 593*81ad6265SDimitry Andric // pools out of range.. 5940b57cec5SDimitry Andric addPass(createARMConstantIslandPass()); 595*81ad6265SDimitry Andric // Finalises Low-Overhead Loops. This replaces pseudo instructions with real 596*81ad6265SDimitry Andric // instructions, but the pseudos all have conservative sizes so that block 597*81ad6265SDimitry Andric // sizes will only be decreased by this pass. 5980b57cec5SDimitry Andric addPass(createARMLowOverheadLoopsPass()); 599480093f4SDimitry Andric 600fe6060f1SDimitry Andric if (TM->getTargetTriple().isOSWindows()) { 601480093f4SDimitry Andric // Identify valid longjmp targets for Windows Control Flow Guard. 602480093f4SDimitry Andric addPass(createCFGuardLongjmpPass()); 603fe6060f1SDimitry Andric // Identify valid eh continuation targets for Windows EHCont Guard. 604fe6060f1SDimitry Andric addPass(createEHContGuardCatchretPass()); 605fe6060f1SDimitry Andric } 6060b57cec5SDimitry Andric } 607