1 //===-- ARMTargetMachine.cpp - Define TargetMachine for ARM ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // 10 //===----------------------------------------------------------------------===// 11 12 #include "ARMTargetMachine.h" 13 #include "ARM.h" 14 #include "ARMMacroFusion.h" 15 #include "ARMSubtarget.h" 16 #include "ARMTargetObjectFile.h" 17 #include "ARMTargetTransformInfo.h" 18 #include "MCTargetDesc/ARMMCTargetDesc.h" 19 #include "TargetInfo/ARMTargetInfo.h" 20 #include "llvm/ADT/Optional.h" 21 #include "llvm/ADT/STLExtras.h" 22 #include "llvm/ADT/StringRef.h" 23 #include "llvm/ADT/Triple.h" 24 #include "llvm/Analysis/TargetTransformInfo.h" 25 #include "llvm/CodeGen/ExecutionDomainFix.h" 26 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 27 #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 29 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 30 #include "llvm/CodeGen/GlobalISel/Legalizer.h" 31 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 32 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 33 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" 34 #include "llvm/CodeGen/MachineFunction.h" 35 #include "llvm/CodeGen/MachineScheduler.h" 36 #include "llvm/CodeGen/Passes.h" 37 #include "llvm/CodeGen/TargetPassConfig.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/Pass.h" 42 #include "llvm/Support/CodeGen.h" 43 #include "llvm/Support/CommandLine.h" 44 #include "llvm/Support/ErrorHandling.h" 45 #include "llvm/Support/TargetParser.h" 46 #include "llvm/Support/TargetRegistry.h" 47 #include "llvm/Target/TargetLoweringObjectFile.h" 48 #include "llvm/Target/TargetOptions.h" 49 #include "llvm/Transforms/CFGuard.h" 50 #include "llvm/Transforms/Scalar.h" 51 #include <cassert> 52 #include <memory> 53 #include <string> 54 55 using namespace llvm; 56 57 static cl::opt<bool> 58 DisableA15SDOptimization("disable-a15-sd-optimization", cl::Hidden, 59 cl::desc("Inhibit optimization of S->D register accesses on A15"), 60 cl::init(false)); 61 62 static cl::opt<bool> 63 EnableAtomicTidy("arm-atomic-cfg-tidy", cl::Hidden, 64 cl::desc("Run SimplifyCFG after expanding atomic operations" 65 " to make use of cmpxchg flow-based information"), 66 cl::init(true)); 67 68 static cl::opt<bool> 69 EnableARMLoadStoreOpt("arm-load-store-opt", cl::Hidden, 70 cl::desc("Enable ARM load/store optimization pass"), 71 cl::init(true)); 72 73 // FIXME: Unify control over GlobalMerge. 74 static cl::opt<cl::boolOrDefault> 75 EnableGlobalMerge("arm-global-merge", cl::Hidden, 76 cl::desc("Enable the global merge pass")); 77 78 namespace llvm { 79 void initializeARMExecutionDomainFixPass(PassRegistry&); 80 } 81 82 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeARMTarget() { 83 // Register the target. 84 RegisterTargetMachine<ARMLETargetMachine> X(getTheARMLETarget()); 85 RegisterTargetMachine<ARMLETargetMachine> A(getTheThumbLETarget()); 86 RegisterTargetMachine<ARMBETargetMachine> Y(getTheARMBETarget()); 87 RegisterTargetMachine<ARMBETargetMachine> B(getTheThumbBETarget()); 88 89 PassRegistry &Registry = *PassRegistry::getPassRegistry(); 90 initializeGlobalISel(Registry); 91 initializeARMLoadStoreOptPass(Registry); 92 initializeARMPreAllocLoadStoreOptPass(Registry); 93 initializeARMParallelDSPPass(Registry); 94 initializeARMConstantIslandsPass(Registry); 95 initializeARMExecutionDomainFixPass(Registry); 96 initializeARMExpandPseudoPass(Registry); 97 initializeThumb2SizeReducePass(Registry); 98 initializeMVEVPTBlockPass(Registry); 99 initializeMVEVPTOptimisationsPass(Registry); 100 initializeMVETailPredicationPass(Registry); 101 initializeARMLowOverheadLoopsPass(Registry); 102 initializeMVEGatherScatterLoweringPass(Registry); 103 } 104 105 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 106 if (TT.isOSBinFormatMachO()) 107 return std::make_unique<TargetLoweringObjectFileMachO>(); 108 if (TT.isOSWindows()) 109 return std::make_unique<TargetLoweringObjectFileCOFF>(); 110 return std::make_unique<ARMElfTargetObjectFile>(); 111 } 112 113 static ARMBaseTargetMachine::ARMABI 114 computeTargetABI(const Triple &TT, StringRef CPU, 115 const TargetOptions &Options) { 116 StringRef ABIName = Options.MCOptions.getABIName(); 117 118 if (ABIName.empty()) 119 ABIName = ARM::computeDefaultTargetABI(TT, CPU); 120 121 if (ABIName == "aapcs16") 122 return ARMBaseTargetMachine::ARM_ABI_AAPCS16; 123 else if (ABIName.startswith("aapcs")) 124 return ARMBaseTargetMachine::ARM_ABI_AAPCS; 125 else if (ABIName.startswith("apcs")) 126 return ARMBaseTargetMachine::ARM_ABI_APCS; 127 128 llvm_unreachable("Unhandled/unknown ABI Name!"); 129 return ARMBaseTargetMachine::ARM_ABI_UNKNOWN; 130 } 131 132 static std::string computeDataLayout(const Triple &TT, StringRef CPU, 133 const TargetOptions &Options, 134 bool isLittle) { 135 auto ABI = computeTargetABI(TT, CPU, Options); 136 std::string Ret; 137 138 if (isLittle) 139 // Little endian. 140 Ret += "e"; 141 else 142 // Big endian. 143 Ret += "E"; 144 145 Ret += DataLayout::getManglingComponent(TT); 146 147 // Pointers are 32 bits and aligned to 32 bits. 148 Ret += "-p:32:32"; 149 150 // Function pointers are aligned to 8 bits (because the LSB stores the 151 // ARM/Thumb state). 152 Ret += "-Fi8"; 153 154 // ABIs other than APCS have 64 bit integers with natural alignment. 155 if (ABI != ARMBaseTargetMachine::ARM_ABI_APCS) 156 Ret += "-i64:64"; 157 158 // We have 64 bits floats. The APCS ABI requires them to be aligned to 32 159 // bits, others to 64 bits. We always try to align to 64 bits. 160 if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS) 161 Ret += "-f64:32:64"; 162 163 // We have 128 and 64 bit vectors. The APCS ABI aligns them to 32 bits, others 164 // to 64. We always ty to give them natural alignment. 165 if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS) 166 Ret += "-v64:32:64-v128:32:128"; 167 else if (ABI != ARMBaseTargetMachine::ARM_ABI_AAPCS16) 168 Ret += "-v128:64:128"; 169 170 // Try to align aggregates to 32 bits (the default is 64 bits, which has no 171 // particular hardware support on 32-bit ARM). 172 Ret += "-a:0:32"; 173 174 // Integer registers are 32 bits. 175 Ret += "-n32"; 176 177 // The stack is 128 bit aligned on NaCl, 64 bit aligned on AAPCS and 32 bit 178 // aligned everywhere else. 179 if (TT.isOSNaCl() || ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16) 180 Ret += "-S128"; 181 else if (ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS) 182 Ret += "-S64"; 183 else 184 Ret += "-S32"; 185 186 return Ret; 187 } 188 189 static Reloc::Model getEffectiveRelocModel(const Triple &TT, 190 Optional<Reloc::Model> RM) { 191 if (!RM.hasValue()) 192 // Default relocation model on Darwin is PIC. 193 return TT.isOSBinFormatMachO() ? Reloc::PIC_ : Reloc::Static; 194 195 if (*RM == Reloc::ROPI || *RM == Reloc::RWPI || *RM == Reloc::ROPI_RWPI) 196 assert(TT.isOSBinFormatELF() && 197 "ROPI/RWPI currently only supported for ELF"); 198 199 // DynamicNoPIC is only used on darwin. 200 if (*RM == Reloc::DynamicNoPIC && !TT.isOSDarwin()) 201 return Reloc::Static; 202 203 return *RM; 204 } 205 206 /// Create an ARM architecture model. 207 /// 208 ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT, 209 StringRef CPU, StringRef FS, 210 const TargetOptions &Options, 211 Optional<Reloc::Model> RM, 212 Optional<CodeModel::Model> CM, 213 CodeGenOpt::Level OL, bool isLittle) 214 : LLVMTargetMachine(T, computeDataLayout(TT, CPU, Options, isLittle), TT, 215 CPU, FS, Options, getEffectiveRelocModel(TT, RM), 216 getEffectiveCodeModel(CM, CodeModel::Small), OL), 217 TargetABI(computeTargetABI(TT, CPU, Options)), 218 TLOF(createTLOF(getTargetTriple())), isLittle(isLittle) { 219 220 // Default to triple-appropriate float ABI 221 if (Options.FloatABIType == FloatABI::Default) { 222 if (isTargetHardFloat()) 223 this->Options.FloatABIType = FloatABI::Hard; 224 else 225 this->Options.FloatABIType = FloatABI::Soft; 226 } 227 228 // Default to triple-appropriate EABI 229 if (Options.EABIVersion == EABI::Default || 230 Options.EABIVersion == EABI::Unknown) { 231 // musl is compatible with glibc with regard to EABI version 232 if ((TargetTriple.getEnvironment() == Triple::GNUEABI || 233 TargetTriple.getEnvironment() == Triple::GNUEABIHF || 234 TargetTriple.getEnvironment() == Triple::MuslEABI || 235 TargetTriple.getEnvironment() == Triple::MuslEABIHF) && 236 !(TargetTriple.isOSWindows() || TargetTriple.isOSDarwin())) 237 this->Options.EABIVersion = EABI::GNU; 238 else 239 this->Options.EABIVersion = EABI::EABI5; 240 } 241 242 if (TT.isOSBinFormatMachO()) { 243 this->Options.TrapUnreachable = true; 244 this->Options.NoTrapAfterNoreturn = true; 245 } 246 247 // ARM supports the debug entry values. 248 setSupportsDebugEntryValues(true); 249 250 initAsmInfo(); 251 252 // ARM supports the MachineOutliner. 253 setMachineOutliner(true); 254 setSupportsDefaultOutlining(false); 255 } 256 257 ARMBaseTargetMachine::~ARMBaseTargetMachine() = default; 258 259 const ARMSubtarget * 260 ARMBaseTargetMachine::getSubtargetImpl(const Function &F) const { 261 Attribute CPUAttr = F.getFnAttribute("target-cpu"); 262 Attribute FSAttr = F.getFnAttribute("target-features"); 263 264 std::string CPU = !CPUAttr.hasAttribute(Attribute::None) 265 ? CPUAttr.getValueAsString().str() 266 : TargetCPU; 267 std::string FS = !FSAttr.hasAttribute(Attribute::None) 268 ? FSAttr.getValueAsString().str() 269 : TargetFS; 270 271 // FIXME: This is related to the code below to reset the target options, 272 // we need to know whether or not the soft float flag is set on the 273 // function before we can generate a subtarget. We also need to use 274 // it as a key for the subtarget since that can be the only difference 275 // between two functions. 276 bool SoftFloat = 277 F.getFnAttribute("use-soft-float").getValueAsString() == "true"; 278 // If the soft float attribute is set on the function turn on the soft float 279 // subtarget feature. 280 if (SoftFloat) 281 FS += FS.empty() ? "+soft-float" : ",+soft-float"; 282 283 // Use the optminsize to identify the subtarget, but don't use it in the 284 // feature string. 285 std::string Key = CPU + FS; 286 if (F.hasMinSize()) 287 Key += "+minsize"; 288 289 auto &I = SubtargetMap[Key]; 290 if (!I) { 291 // This needs to be done before we create a new subtarget since any 292 // creation will depend on the TM and the code generation flags on the 293 // function that reside in TargetOptions. 294 resetTargetOptions(F); 295 I = std::make_unique<ARMSubtarget>(TargetTriple, CPU, FS, *this, isLittle, 296 F.hasMinSize()); 297 298 if (!I->isThumb() && !I->hasARMOps()) 299 F.getContext().emitError("Function '" + F.getName() + "' uses ARM " 300 "instructions, but the target does not support ARM mode execution."); 301 } 302 303 return I.get(); 304 } 305 306 TargetTransformInfo 307 ARMBaseTargetMachine::getTargetTransformInfo(const Function &F) { 308 return TargetTransformInfo(ARMTTIImpl(this, F)); 309 } 310 311 ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT, 312 StringRef CPU, StringRef FS, 313 const TargetOptions &Options, 314 Optional<Reloc::Model> RM, 315 Optional<CodeModel::Model> CM, 316 CodeGenOpt::Level OL, bool JIT) 317 : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {} 318 319 ARMBETargetMachine::ARMBETargetMachine(const Target &T, const Triple &TT, 320 StringRef CPU, StringRef FS, 321 const TargetOptions &Options, 322 Optional<Reloc::Model> RM, 323 Optional<CodeModel::Model> CM, 324 CodeGenOpt::Level OL, bool JIT) 325 : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {} 326 327 namespace { 328 329 /// ARM Code Generator Pass Configuration Options. 330 class ARMPassConfig : public TargetPassConfig { 331 public: 332 ARMPassConfig(ARMBaseTargetMachine &TM, PassManagerBase &PM) 333 : TargetPassConfig(TM, PM) {} 334 335 ARMBaseTargetMachine &getARMTargetMachine() const { 336 return getTM<ARMBaseTargetMachine>(); 337 } 338 339 ScheduleDAGInstrs * 340 createMachineScheduler(MachineSchedContext *C) const override { 341 ScheduleDAGMILive *DAG = createGenericSchedLive(C); 342 // add DAG Mutations here. 343 const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>(); 344 if (ST.hasFusion()) 345 DAG->addMutation(createARMMacroFusionDAGMutation()); 346 return DAG; 347 } 348 349 ScheduleDAGInstrs * 350 createPostMachineScheduler(MachineSchedContext *C) const override { 351 ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 352 // add DAG Mutations here. 353 const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>(); 354 if (ST.hasFusion()) 355 DAG->addMutation(createARMMacroFusionDAGMutation()); 356 return DAG; 357 } 358 359 void addIRPasses() override; 360 void addCodeGenPrepare() override; 361 bool addPreISel() override; 362 bool addInstSelector() override; 363 bool addIRTranslator() override; 364 bool addLegalizeMachineIR() override; 365 bool addRegBankSelect() override; 366 bool addGlobalInstructionSelect() override; 367 void addPreRegAlloc() override; 368 void addPreSched2() override; 369 void addPreEmitPass() override; 370 void addPreEmitPass2() override; 371 372 std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 373 }; 374 375 class ARMExecutionDomainFix : public ExecutionDomainFix { 376 public: 377 static char ID; 378 ARMExecutionDomainFix() : ExecutionDomainFix(ID, ARM::DPRRegClass) {} 379 StringRef getPassName() const override { 380 return "ARM Execution Domain Fix"; 381 } 382 }; 383 char ARMExecutionDomainFix::ID; 384 385 } // end anonymous namespace 386 387 INITIALIZE_PASS_BEGIN(ARMExecutionDomainFix, "arm-execution-domain-fix", 388 "ARM Execution Domain Fix", false, false) 389 INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) 390 INITIALIZE_PASS_END(ARMExecutionDomainFix, "arm-execution-domain-fix", 391 "ARM Execution Domain Fix", false, false) 392 393 TargetPassConfig *ARMBaseTargetMachine::createPassConfig(PassManagerBase &PM) { 394 return new ARMPassConfig(*this, PM); 395 } 396 397 std::unique_ptr<CSEConfigBase> ARMPassConfig::getCSEConfig() const { 398 return getStandardCSEConfigForOpt(TM->getOptLevel()); 399 } 400 401 void ARMPassConfig::addIRPasses() { 402 if (TM->Options.ThreadModel == ThreadModel::Single) 403 addPass(createLowerAtomicPass()); 404 else 405 addPass(createAtomicExpandPass()); 406 407 // Cmpxchg instructions are often used with a subsequent comparison to 408 // determine whether it succeeded. We can exploit existing control-flow in 409 // ldrex/strex loops to simplify this, but it needs tidying up. 410 if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy) 411 addPass(createCFGSimplificationPass( 412 1, false, false, true, true, [this](const Function &F) { 413 const auto &ST = this->TM->getSubtarget<ARMSubtarget>(F); 414 return ST.hasAnyDataBarrier() && !ST.isThumb1Only(); 415 })); 416 417 addPass(createMVEGatherScatterLoweringPass()); 418 419 TargetPassConfig::addIRPasses(); 420 421 // Run the parallel DSP pass. 422 if (getOptLevel() == CodeGenOpt::Aggressive) 423 addPass(createARMParallelDSPPass()); 424 425 // Match interleaved memory accesses to ldN/stN intrinsics. 426 if (TM->getOptLevel() != CodeGenOpt::None) 427 addPass(createInterleavedAccessPass()); 428 429 // Add Control Flow Guard checks. 430 if (TM->getTargetTriple().isOSWindows()) 431 addPass(createCFGuardCheckPass()); 432 } 433 434 void ARMPassConfig::addCodeGenPrepare() { 435 if (getOptLevel() != CodeGenOpt::None) 436 addPass(createTypePromotionPass()); 437 TargetPassConfig::addCodeGenPrepare(); 438 } 439 440 bool ARMPassConfig::addPreISel() { 441 if ((TM->getOptLevel() != CodeGenOpt::None && 442 EnableGlobalMerge == cl::BOU_UNSET) || 443 EnableGlobalMerge == cl::BOU_TRUE) { 444 // FIXME: This is using the thumb1 only constant value for 445 // maximal global offset for merging globals. We may want 446 // to look into using the old value for non-thumb1 code of 447 // 4095 based on the TargetMachine, but this starts to become 448 // tricky when doing code gen per function. 449 bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) && 450 (EnableGlobalMerge == cl::BOU_UNSET); 451 // Merging of extern globals is enabled by default on non-Mach-O as we 452 // expect it to be generally either beneficial or harmless. On Mach-O it 453 // is disabled as we emit the .subsections_via_symbols directive which 454 // means that merging extern globals is not safe. 455 bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO(); 456 addPass(createGlobalMergePass(TM, 127, OnlyOptimizeForSize, 457 MergeExternalByDefault)); 458 } 459 460 if (TM->getOptLevel() != CodeGenOpt::None) { 461 addPass(createHardwareLoopsPass()); 462 addPass(createMVETailPredicationPass()); 463 } 464 465 return false; 466 } 467 468 bool ARMPassConfig::addInstSelector() { 469 addPass(createARMISelDag(getARMTargetMachine(), getOptLevel())); 470 return false; 471 } 472 473 bool ARMPassConfig::addIRTranslator() { 474 addPass(new IRTranslator()); 475 return false; 476 } 477 478 bool ARMPassConfig::addLegalizeMachineIR() { 479 addPass(new Legalizer()); 480 return false; 481 } 482 483 bool ARMPassConfig::addRegBankSelect() { 484 addPass(new RegBankSelect()); 485 return false; 486 } 487 488 bool ARMPassConfig::addGlobalInstructionSelect() { 489 addPass(new InstructionSelect()); 490 return false; 491 } 492 493 void ARMPassConfig::addPreRegAlloc() { 494 if (getOptLevel() != CodeGenOpt::None) { 495 addPass(createMVEVPTOptimisationsPass()); 496 497 addPass(createMLxExpansionPass()); 498 499 if (EnableARMLoadStoreOpt) 500 addPass(createARMLoadStoreOptimizationPass(/* pre-register alloc */ true)); 501 502 if (!DisableA15SDOptimization) 503 addPass(createA15SDOptimizerPass()); 504 } 505 } 506 507 void ARMPassConfig::addPreSched2() { 508 if (getOptLevel() != CodeGenOpt::None) { 509 if (EnableARMLoadStoreOpt) 510 addPass(createARMLoadStoreOptimizationPass()); 511 512 addPass(new ARMExecutionDomainFix()); 513 addPass(createBreakFalseDeps()); 514 } 515 516 // Expand some pseudo instructions into multiple instructions to allow 517 // proper scheduling. 518 addPass(createARMExpandPseudoPass()); 519 520 if (getOptLevel() != CodeGenOpt::None) { 521 // When optimising for size, always run the Thumb2SizeReduction pass before 522 // IfConversion. Otherwise, check whether IT blocks are restricted 523 // (e.g. in v8, IfConversion depends on Thumb instruction widths) 524 addPass(createThumb2SizeReductionPass([this](const Function &F) { 525 return this->TM->getSubtarget<ARMSubtarget>(F).hasMinSize() || 526 this->TM->getSubtarget<ARMSubtarget>(F).restrictIT(); 527 })); 528 529 addPass(createIfConverter([](const MachineFunction &MF) { 530 return !MF.getSubtarget<ARMSubtarget>().isThumb1Only(); 531 })); 532 } 533 addPass(createMVEVPTBlockPass()); 534 addPass(createThumb2ITBlockPass()); 535 536 // Add both scheduling passes to give the subtarget an opportunity to pick 537 // between them. 538 if (getOptLevel() != CodeGenOpt::None) { 539 addPass(&PostMachineSchedulerID); 540 addPass(&PostRASchedulerID); 541 } 542 } 543 544 void ARMPassConfig::addPreEmitPass() { 545 addPass(createThumb2SizeReductionPass()); 546 547 // Constant island pass work on unbundled instructions. 548 addPass(createUnpackMachineBundles([](const MachineFunction &MF) { 549 return MF.getSubtarget<ARMSubtarget>().isThumb2(); 550 })); 551 552 // Don't optimize barriers at -O0. 553 if (getOptLevel() != CodeGenOpt::None) 554 addPass(createARMOptimizeBarriersPass()); 555 } 556 557 void ARMPassConfig::addPreEmitPass2() { 558 addPass(createARMConstantIslandPass()); 559 addPass(createARMLowOverheadLoopsPass()); 560 561 // Identify valid longjmp targets for Windows Control Flow Guard. 562 if (TM->getTargetTriple().isOSWindows()) 563 addPass(createCFGuardLongjmpPass()); 564 } 565