1 //===-- AArch64TargetMachine.cpp - Define TargetMachine for AArch64 -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // 10 //===----------------------------------------------------------------------===// 11 12 #include "AArch64TargetMachine.h" 13 #include "AArch64.h" 14 #include "AArch64MachineFunctionInfo.h" 15 #include "AArch64MacroFusion.h" 16 #include "AArch64Subtarget.h" 17 #include "AArch64TargetObjectFile.h" 18 #include "AArch64TargetTransformInfo.h" 19 #include "MCTargetDesc/AArch64MCTargetDesc.h" 20 #include "TargetInfo/AArch64TargetInfo.h" 21 #include "llvm/ADT/STLExtras.h" 22 #include "llvm/ADT/Triple.h" 23 #include "llvm/Analysis/TargetTransformInfo.h" 24 #include "llvm/CodeGen/CSEConfigBase.h" 25 #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 26 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 27 #include "llvm/CodeGen/GlobalISel/Legalizer.h" 28 #include "llvm/CodeGen/GlobalISel/LoadStoreOpt.h" 29 #include "llvm/CodeGen/GlobalISel/Localizer.h" 30 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 31 #include "llvm/CodeGen/MIRParser/MIParser.h" 32 #include "llvm/CodeGen/MachineScheduler.h" 33 #include "llvm/CodeGen/Passes.h" 34 #include "llvm/CodeGen/TargetPassConfig.h" 35 #include "llvm/IR/Attributes.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/InitializePasses.h" 38 #include "llvm/MC/MCAsmInfo.h" 39 #include "llvm/MC/MCTargetOptions.h" 40 #include "llvm/MC/TargetRegistry.h" 41 #include "llvm/Pass.h" 42 #include "llvm/Support/CodeGen.h" 43 #include "llvm/Support/CommandLine.h" 44 #include "llvm/Target/TargetLoweringObjectFile.h" 45 #include "llvm/Target/TargetOptions.h" 46 #include "llvm/Transforms/CFGuard.h" 47 #include "llvm/Transforms/Scalar.h" 48 #include <memory> 49 #include <string> 50 51 using namespace llvm; 52 53 static cl::opt<bool> EnableCCMP("aarch64-enable-ccmp", 54 cl::desc("Enable the CCMP formation pass"), 55 cl::init(true), cl::Hidden); 56 57 static cl::opt<bool> 58 EnableCondBrTuning("aarch64-enable-cond-br-tune", 59 cl::desc("Enable the conditional branch tuning pass"), 60 cl::init(true), cl::Hidden); 61 62 static cl::opt<bool> EnableMCR("aarch64-enable-mcr", 63 cl::desc("Enable the machine combiner pass"), 64 cl::init(true), cl::Hidden); 65 66 static cl::opt<bool> EnableStPairSuppress("aarch64-enable-stp-suppress", 67 cl::desc("Suppress STP for AArch64"), 68 cl::init(true), cl::Hidden); 69 70 static cl::opt<bool> EnableAdvSIMDScalar( 71 "aarch64-enable-simd-scalar", 72 cl::desc("Enable use of AdvSIMD scalar integer instructions"), 73 cl::init(false), cl::Hidden); 74 75 static cl::opt<bool> 76 EnablePromoteConstant("aarch64-enable-promote-const", 77 cl::desc("Enable the promote constant pass"), 78 cl::init(true), cl::Hidden); 79 80 static cl::opt<bool> EnableCollectLOH( 81 "aarch64-enable-collect-loh", 82 cl::desc("Enable the pass that emits the linker optimization hints (LOH)"), 83 cl::init(true), cl::Hidden); 84 85 static cl::opt<bool> 86 EnableDeadRegisterElimination("aarch64-enable-dead-defs", cl::Hidden, 87 cl::desc("Enable the pass that removes dead" 88 " definitons and replaces stores to" 89 " them with stores to the zero" 90 " register"), 91 cl::init(true)); 92 93 static cl::opt<bool> EnableRedundantCopyElimination( 94 "aarch64-enable-copyelim", 95 cl::desc("Enable the redundant copy elimination pass"), cl::init(true), 96 cl::Hidden); 97 98 static cl::opt<bool> EnableLoadStoreOpt("aarch64-enable-ldst-opt", 99 cl::desc("Enable the load/store pair" 100 " optimization pass"), 101 cl::init(true), cl::Hidden); 102 103 static cl::opt<bool> EnableAtomicTidy( 104 "aarch64-enable-atomic-cfg-tidy", cl::Hidden, 105 cl::desc("Run SimplifyCFG after expanding atomic operations" 106 " to make use of cmpxchg flow-based information"), 107 cl::init(true)); 108 109 static cl::opt<bool> 110 EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, 111 cl::desc("Run early if-conversion"), 112 cl::init(true)); 113 114 static cl::opt<bool> 115 EnableCondOpt("aarch64-enable-condopt", 116 cl::desc("Enable the condition optimizer pass"), 117 cl::init(true), cl::Hidden); 118 119 static cl::opt<bool> 120 EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden, 121 cl::desc("Enable optimizations on complex GEPs"), 122 cl::init(false)); 123 124 static cl::opt<bool> 125 BranchRelaxation("aarch64-enable-branch-relax", cl::Hidden, cl::init(true), 126 cl::desc("Relax out of range conditional branches")); 127 128 static cl::opt<bool> EnableCompressJumpTables( 129 "aarch64-enable-compress-jump-tables", cl::Hidden, cl::init(true), 130 cl::desc("Use smallest entry possible for jump tables")); 131 132 // FIXME: Unify control over GlobalMerge. 133 static cl::opt<cl::boolOrDefault> 134 EnableGlobalMerge("aarch64-enable-global-merge", cl::Hidden, 135 cl::desc("Enable the global merge pass")); 136 137 static cl::opt<bool> 138 EnableLoopDataPrefetch("aarch64-enable-loop-data-prefetch", cl::Hidden, 139 cl::desc("Enable the loop data prefetch pass"), 140 cl::init(true)); 141 142 static cl::opt<int> EnableGlobalISelAtO( 143 "aarch64-enable-global-isel-at-O", cl::Hidden, 144 cl::desc("Enable GlobalISel at or below an opt level (-1 to disable)"), 145 cl::init(0)); 146 147 static cl::opt<bool> 148 EnableSVEIntrinsicOpts("aarch64-enable-sve-intrinsic-opts", cl::Hidden, 149 cl::desc("Enable SVE intrinsic opts"), 150 cl::init(true)); 151 152 static cl::opt<bool> EnableFalkorHWPFFix("aarch64-enable-falkor-hwpf-fix", 153 cl::init(true), cl::Hidden); 154 155 static cl::opt<bool> 156 EnableBranchTargets("aarch64-enable-branch-targets", cl::Hidden, 157 cl::desc("Enable the AArch64 branch target pass"), 158 cl::init(true)); 159 160 static cl::opt<unsigned> SVEVectorBitsMaxOpt( 161 "aarch64-sve-vector-bits-max", 162 cl::desc("Assume SVE vector registers are at most this big, " 163 "with zero meaning no maximum size is assumed."), 164 cl::init(0), cl::Hidden); 165 166 static cl::opt<unsigned> SVEVectorBitsMinOpt( 167 "aarch64-sve-vector-bits-min", 168 cl::desc("Assume SVE vector registers are at least this big, " 169 "with zero meaning no minimum size is assumed."), 170 cl::init(0), cl::Hidden); 171 172 extern cl::opt<bool> EnableHomogeneousPrologEpilog; 173 174 static cl::opt<bool> EnableGISelLoadStoreOptPreLegal( 175 "aarch64-enable-gisel-ldst-prelegal", 176 cl::desc("Enable GlobalISel's pre-legalizer load/store optimization pass"), 177 cl::init(true), cl::Hidden); 178 179 static cl::opt<bool> EnableGISelLoadStoreOptPostLegal( 180 "aarch64-enable-gisel-ldst-postlegal", 181 cl::desc("Enable GlobalISel's post-legalizer load/store optimization pass"), 182 cl::init(false), cl::Hidden); 183 184 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Target() { 185 // Register the target. 186 RegisterTargetMachine<AArch64leTargetMachine> X(getTheAArch64leTarget()); 187 RegisterTargetMachine<AArch64beTargetMachine> Y(getTheAArch64beTarget()); 188 RegisterTargetMachine<AArch64leTargetMachine> Z(getTheARM64Target()); 189 RegisterTargetMachine<AArch64leTargetMachine> W(getTheARM64_32Target()); 190 RegisterTargetMachine<AArch64leTargetMachine> V(getTheAArch64_32Target()); 191 auto PR = PassRegistry::getPassRegistry(); 192 initializeGlobalISel(*PR); 193 initializeAArch64A53Fix835769Pass(*PR); 194 initializeAArch64A57FPLoadBalancingPass(*PR); 195 initializeAArch64AdvSIMDScalarPass(*PR); 196 initializeAArch64BranchTargetsPass(*PR); 197 initializeAArch64CollectLOHPass(*PR); 198 initializeAArch64CompressJumpTablesPass(*PR); 199 initializeAArch64ConditionalComparesPass(*PR); 200 initializeAArch64ConditionOptimizerPass(*PR); 201 initializeAArch64DeadRegisterDefinitionsPass(*PR); 202 initializeAArch64ExpandPseudoPass(*PR); 203 initializeAArch64LoadStoreOptPass(*PR); 204 initializeAArch64MIPeepholeOptPass(*PR); 205 initializeAArch64SIMDInstrOptPass(*PR); 206 initializeAArch64O0PreLegalizerCombinerPass(*PR); 207 initializeAArch64PreLegalizerCombinerPass(*PR); 208 initializeAArch64PostLegalizerCombinerPass(*PR); 209 initializeAArch64PostLegalizerLoweringPass(*PR); 210 initializeAArch64PostSelectOptimizePass(*PR); 211 initializeAArch64PromoteConstantPass(*PR); 212 initializeAArch64RedundantCopyEliminationPass(*PR); 213 initializeAArch64StorePairSuppressPass(*PR); 214 initializeFalkorHWPFFixPass(*PR); 215 initializeFalkorMarkStridedAccessesLegacyPass(*PR); 216 initializeLDTLSCleanupPass(*PR); 217 initializeSVEIntrinsicOptsPass(*PR); 218 initializeAArch64SpeculationHardeningPass(*PR); 219 initializeAArch64SLSHardeningPass(*PR); 220 initializeAArch64StackTaggingPass(*PR); 221 initializeAArch64StackTaggingPreRAPass(*PR); 222 initializeAArch64LowerHomogeneousPrologEpilogPass(*PR); 223 } 224 225 //===----------------------------------------------------------------------===// 226 // AArch64 Lowering public interface. 227 //===----------------------------------------------------------------------===// 228 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 229 if (TT.isOSBinFormatMachO()) 230 return std::make_unique<AArch64_MachoTargetObjectFile>(); 231 if (TT.isOSBinFormatCOFF()) 232 return std::make_unique<AArch64_COFFTargetObjectFile>(); 233 234 return std::make_unique<AArch64_ELFTargetObjectFile>(); 235 } 236 237 // Helper function to build a DataLayout string 238 static std::string computeDataLayout(const Triple &TT, 239 const MCTargetOptions &Options, 240 bool LittleEndian) { 241 if (TT.isOSBinFormatMachO()) { 242 if (TT.getArch() == Triple::aarch64_32) 243 return "e-m:o-p:32:32-i64:64-i128:128-n32:64-S128"; 244 return "e-m:o-i64:64-i128:128-n32:64-S128"; 245 } 246 if (TT.isOSBinFormatCOFF()) 247 return "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"; 248 std::string Endian = LittleEndian ? "e" : "E"; 249 std::string Ptr32 = TT.getEnvironment() == Triple::GNUILP32 ? "-p:32:32" : ""; 250 return Endian + "-m:e" + Ptr32 + 251 "-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"; 252 } 253 254 static StringRef computeDefaultCPU(const Triple &TT, StringRef CPU) { 255 if (CPU.empty() && TT.isArm64e()) 256 return "apple-a12"; 257 return CPU; 258 } 259 260 static Reloc::Model getEffectiveRelocModel(const Triple &TT, 261 Optional<Reloc::Model> RM) { 262 // AArch64 Darwin and Windows are always PIC. 263 if (TT.isOSDarwin() || TT.isOSWindows()) 264 return Reloc::PIC_; 265 // On ELF platforms the default static relocation model has a smart enough 266 // linker to cope with referencing external symbols defined in a shared 267 // library. Hence DynamicNoPIC doesn't need to be promoted to PIC. 268 if (!RM.hasValue() || *RM == Reloc::DynamicNoPIC) 269 return Reloc::Static; 270 return *RM; 271 } 272 273 static CodeModel::Model 274 getEffectiveAArch64CodeModel(const Triple &TT, Optional<CodeModel::Model> CM, 275 bool JIT) { 276 if (CM) { 277 if (*CM != CodeModel::Small && *CM != CodeModel::Tiny && 278 *CM != CodeModel::Large) { 279 report_fatal_error( 280 "Only small, tiny and large code models are allowed on AArch64"); 281 } else if (*CM == CodeModel::Tiny && !TT.isOSBinFormatELF()) 282 report_fatal_error("tiny code model is only supported on ELF"); 283 return *CM; 284 } 285 // The default MCJIT memory managers make no guarantees about where they can 286 // find an executable page; JITed code needs to be able to refer to globals 287 // no matter how far away they are. 288 // We should set the CodeModel::Small for Windows ARM64 in JIT mode, 289 // since with large code model LLVM generating 4 MOV instructions, and 290 // Windows doesn't support relocating these long branch (4 MOVs). 291 if (JIT && !TT.isOSWindows()) 292 return CodeModel::Large; 293 return CodeModel::Small; 294 } 295 296 /// Create an AArch64 architecture model. 297 /// 298 AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT, 299 StringRef CPU, StringRef FS, 300 const TargetOptions &Options, 301 Optional<Reloc::Model> RM, 302 Optional<CodeModel::Model> CM, 303 CodeGenOpt::Level OL, bool JIT, 304 bool LittleEndian) 305 : LLVMTargetMachine(T, 306 computeDataLayout(TT, Options.MCOptions, LittleEndian), 307 TT, computeDefaultCPU(TT, CPU), FS, Options, 308 getEffectiveRelocModel(TT, RM), 309 getEffectiveAArch64CodeModel(TT, CM, JIT), OL), 310 TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian) { 311 initAsmInfo(); 312 313 if (TT.isOSBinFormatMachO()) { 314 this->Options.TrapUnreachable = true; 315 this->Options.NoTrapAfterNoreturn = true; 316 } 317 318 if (getMCAsmInfo()->usesWindowsCFI()) { 319 // Unwinding can get confused if the last instruction in an 320 // exception-handling region (function, funclet, try block, etc.) 321 // is a call. 322 // 323 // FIXME: We could elide the trap if the next instruction would be in 324 // the same region anyway. 325 this->Options.TrapUnreachable = true; 326 } 327 328 if (this->Options.TLSSize == 0) // default 329 this->Options.TLSSize = 24; 330 if ((getCodeModel() == CodeModel::Small || 331 getCodeModel() == CodeModel::Kernel) && 332 this->Options.TLSSize > 32) 333 // for the small (and kernel) code model, the maximum TLS size is 4GiB 334 this->Options.TLSSize = 32; 335 else if (getCodeModel() == CodeModel::Tiny && this->Options.TLSSize > 24) 336 // for the tiny code model, the maximum TLS size is 1MiB (< 16MiB) 337 this->Options.TLSSize = 24; 338 339 // Enable GlobalISel at or below EnableGlobalISelAt0, unless this is 340 // MachO/CodeModel::Large, which GlobalISel does not support. 341 if (getOptLevel() <= EnableGlobalISelAtO && 342 TT.getArch() != Triple::aarch64_32 && 343 TT.getEnvironment() != Triple::GNUILP32 && 344 !(getCodeModel() == CodeModel::Large && TT.isOSBinFormatMachO())) { 345 setGlobalISel(true); 346 setGlobalISelAbort(GlobalISelAbortMode::Disable); 347 } 348 349 // AArch64 supports the MachineOutliner. 350 setMachineOutliner(true); 351 352 // AArch64 supports default outlining behaviour. 353 setSupportsDefaultOutlining(true); 354 355 // AArch64 supports the debug entry values. 356 setSupportsDebugEntryValues(true); 357 } 358 359 AArch64TargetMachine::~AArch64TargetMachine() = default; 360 361 const AArch64Subtarget * 362 AArch64TargetMachine::getSubtargetImpl(const Function &F) const { 363 Attribute CPUAttr = F.getFnAttribute("target-cpu"); 364 Attribute TuneAttr = F.getFnAttribute("tune-cpu"); 365 Attribute FSAttr = F.getFnAttribute("target-features"); 366 367 std::string CPU = 368 CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU; 369 std::string TuneCPU = 370 TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU; 371 std::string FS = 372 FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS; 373 374 SmallString<512> Key; 375 376 unsigned MinSVEVectorSize = 0; 377 unsigned MaxSVEVectorSize = 0; 378 Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange); 379 if (VScaleRangeAttr.isValid()) { 380 Optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax(); 381 MinSVEVectorSize = VScaleRangeAttr.getVScaleRangeMin() * 128; 382 MaxSVEVectorSize = VScaleMax ? VScaleMax.getValue() * 128 : 0; 383 } else { 384 MinSVEVectorSize = SVEVectorBitsMinOpt; 385 MaxSVEVectorSize = SVEVectorBitsMaxOpt; 386 } 387 388 assert(MinSVEVectorSize % 128 == 0 && 389 "SVE requires vector length in multiples of 128!"); 390 assert(MaxSVEVectorSize % 128 == 0 && 391 "SVE requires vector length in multiples of 128!"); 392 assert((MaxSVEVectorSize >= MinSVEVectorSize || MaxSVEVectorSize == 0) && 393 "Minimum SVE vector size should not be larger than its maximum!"); 394 395 // Sanitize user input in case of no asserts 396 if (MaxSVEVectorSize == 0) 397 MinSVEVectorSize = (MinSVEVectorSize / 128) * 128; 398 else { 399 MinSVEVectorSize = 400 (std::min(MinSVEVectorSize, MaxSVEVectorSize) / 128) * 128; 401 MaxSVEVectorSize = 402 (std::max(MinSVEVectorSize, MaxSVEVectorSize) / 128) * 128; 403 } 404 405 Key += "SVEMin"; 406 Key += std::to_string(MinSVEVectorSize); 407 Key += "SVEMax"; 408 Key += std::to_string(MaxSVEVectorSize); 409 Key += CPU; 410 Key += TuneCPU; 411 Key += FS; 412 413 auto &I = SubtargetMap[Key]; 414 if (!I) { 415 // This needs to be done before we create a new subtarget since any 416 // creation will depend on the TM and the code generation flags on the 417 // function that reside in TargetOptions. 418 resetTargetOptions(F); 419 I = std::make_unique<AArch64Subtarget>(TargetTriple, CPU, TuneCPU, FS, 420 *this, isLittle, MinSVEVectorSize, 421 MaxSVEVectorSize); 422 } 423 return I.get(); 424 } 425 426 void AArch64leTargetMachine::anchor() { } 427 428 AArch64leTargetMachine::AArch64leTargetMachine( 429 const Target &T, const Triple &TT, StringRef CPU, StringRef FS, 430 const TargetOptions &Options, Optional<Reloc::Model> RM, 431 Optional<CodeModel::Model> CM, CodeGenOpt::Level OL, bool JIT) 432 : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, JIT, true) {} 433 434 void AArch64beTargetMachine::anchor() { } 435 436 AArch64beTargetMachine::AArch64beTargetMachine( 437 const Target &T, const Triple &TT, StringRef CPU, StringRef FS, 438 const TargetOptions &Options, Optional<Reloc::Model> RM, 439 Optional<CodeModel::Model> CM, CodeGenOpt::Level OL, bool JIT) 440 : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, JIT, false) {} 441 442 namespace { 443 444 /// AArch64 Code Generator Pass Configuration Options. 445 class AArch64PassConfig : public TargetPassConfig { 446 public: 447 AArch64PassConfig(AArch64TargetMachine &TM, PassManagerBase &PM) 448 : TargetPassConfig(TM, PM) { 449 if (TM.getOptLevel() != CodeGenOpt::None) 450 substitutePass(&PostRASchedulerID, &PostMachineSchedulerID); 451 } 452 453 AArch64TargetMachine &getAArch64TargetMachine() const { 454 return getTM<AArch64TargetMachine>(); 455 } 456 457 ScheduleDAGInstrs * 458 createMachineScheduler(MachineSchedContext *C) const override { 459 const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>(); 460 ScheduleDAGMILive *DAG = createGenericSchedLive(C); 461 DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); 462 DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); 463 if (ST.hasFusion()) 464 DAG->addMutation(createAArch64MacroFusionDAGMutation()); 465 return DAG; 466 } 467 468 ScheduleDAGInstrs * 469 createPostMachineScheduler(MachineSchedContext *C) const override { 470 const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>(); 471 if (ST.hasFusion()) { 472 // Run the Macro Fusion after RA again since literals are expanded from 473 // pseudos then (v. addPreSched2()). 474 ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 475 DAG->addMutation(createAArch64MacroFusionDAGMutation()); 476 return DAG; 477 } 478 479 return nullptr; 480 } 481 482 void addIRPasses() override; 483 bool addPreISel() override; 484 void addCodeGenPrepare() override; 485 bool addInstSelector() override; 486 bool addIRTranslator() override; 487 void addPreLegalizeMachineIR() override; 488 bool addLegalizeMachineIR() override; 489 void addPreRegBankSelect() override; 490 bool addRegBankSelect() override; 491 void addPreGlobalInstructionSelect() override; 492 bool addGlobalInstructionSelect() override; 493 void addMachineSSAOptimization() override; 494 bool addILPOpts() override; 495 void addPreRegAlloc() override; 496 void addPostRegAlloc() override; 497 void addPreSched2() override; 498 void addPreEmitPass() override; 499 void addPreEmitPass2() override; 500 501 std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 502 }; 503 504 } // end anonymous namespace 505 506 TargetTransformInfo 507 AArch64TargetMachine::getTargetTransformInfo(const Function &F) { 508 return TargetTransformInfo(AArch64TTIImpl(this, F)); 509 } 510 511 TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) { 512 return new AArch64PassConfig(*this, PM); 513 } 514 515 std::unique_ptr<CSEConfigBase> AArch64PassConfig::getCSEConfig() const { 516 return getStandardCSEConfigForOpt(TM->getOptLevel()); 517 } 518 519 void AArch64PassConfig::addIRPasses() { 520 // Always expand atomic operations, we don't deal with atomicrmw or cmpxchg 521 // ourselves. 522 addPass(createAtomicExpandPass()); 523 524 // Expand any SVE vector library calls that we can't code generate directly. 525 if (EnableSVEIntrinsicOpts && TM->getOptLevel() == CodeGenOpt::Aggressive) 526 addPass(createSVEIntrinsicOptsPass()); 527 528 // Cmpxchg instructions are often used with a subsequent comparison to 529 // determine whether it succeeded. We can exploit existing control-flow in 530 // ldrex/strex loops to simplify this, but it needs tidying up. 531 if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy) 532 addPass(createCFGSimplificationPass(SimplifyCFGOptions() 533 .forwardSwitchCondToPhi(true) 534 .convertSwitchRangeToICmp(true) 535 .convertSwitchToLookupTable(true) 536 .needCanonicalLoops(false) 537 .hoistCommonInsts(true) 538 .sinkCommonInsts(true))); 539 540 // Run LoopDataPrefetch 541 // 542 // Run this before LSR to remove the multiplies involved in computing the 543 // pointer values N iterations ahead. 544 if (TM->getOptLevel() != CodeGenOpt::None) { 545 if (EnableLoopDataPrefetch) 546 addPass(createLoopDataPrefetchPass()); 547 if (EnableFalkorHWPFFix) 548 addPass(createFalkorMarkStridedAccessesPass()); 549 } 550 551 TargetPassConfig::addIRPasses(); 552 553 addPass(createAArch64StackTaggingPass( 554 /*IsOptNone=*/TM->getOptLevel() == CodeGenOpt::None)); 555 556 // Match interleaved memory accesses to ldN/stN intrinsics. 557 if (TM->getOptLevel() != CodeGenOpt::None) { 558 addPass(createInterleavedLoadCombinePass()); 559 addPass(createInterleavedAccessPass()); 560 } 561 562 if (TM->getOptLevel() == CodeGenOpt::Aggressive && EnableGEPOpt) { 563 // Call SeparateConstOffsetFromGEP pass to extract constants within indices 564 // and lower a GEP with multiple indices to either arithmetic operations or 565 // multiple GEPs with single index. 566 addPass(createSeparateConstOffsetFromGEPPass(true)); 567 // Call EarlyCSE pass to find and remove subexpressions in the lowered 568 // result. 569 addPass(createEarlyCSEPass()); 570 // Do loop invariant code motion in case part of the lowered result is 571 // invariant. 572 addPass(createLICMPass()); 573 } 574 575 // Add Control Flow Guard checks. 576 if (TM->getTargetTriple().isOSWindows()) 577 addPass(createCFGuardCheckPass()); 578 } 579 580 // Pass Pipeline Configuration 581 bool AArch64PassConfig::addPreISel() { 582 // Run promote constant before global merge, so that the promoted constants 583 // get a chance to be merged 584 if (TM->getOptLevel() != CodeGenOpt::None && EnablePromoteConstant) 585 addPass(createAArch64PromoteConstantPass()); 586 // FIXME: On AArch64, this depends on the type. 587 // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes(). 588 // and the offset has to be a multiple of the related size in bytes. 589 if ((TM->getOptLevel() != CodeGenOpt::None && 590 EnableGlobalMerge == cl::BOU_UNSET) || 591 EnableGlobalMerge == cl::BOU_TRUE) { 592 bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) && 593 (EnableGlobalMerge == cl::BOU_UNSET); 594 595 // Merging of extern globals is enabled by default on non-Mach-O as we 596 // expect it to be generally either beneficial or harmless. On Mach-O it 597 // is disabled as we emit the .subsections_via_symbols directive which 598 // means that merging extern globals is not safe. 599 bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO(); 600 601 // FIXME: extern global merging is only enabled when we optimise for size 602 // because there are some regressions with it also enabled for performance. 603 if (!OnlyOptimizeForSize) 604 MergeExternalByDefault = false; 605 606 addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize, 607 MergeExternalByDefault)); 608 } 609 610 return false; 611 } 612 613 void AArch64PassConfig::addCodeGenPrepare() { 614 if (getOptLevel() != CodeGenOpt::None) 615 addPass(createTypePromotionPass()); 616 TargetPassConfig::addCodeGenPrepare(); 617 } 618 619 bool AArch64PassConfig::addInstSelector() { 620 addPass(createAArch64ISelDag(getAArch64TargetMachine(), getOptLevel())); 621 622 // For ELF, cleanup any local-dynamic TLS accesses (i.e. combine as many 623 // references to _TLS_MODULE_BASE_ as possible. 624 if (TM->getTargetTriple().isOSBinFormatELF() && 625 getOptLevel() != CodeGenOpt::None) 626 addPass(createAArch64CleanupLocalDynamicTLSPass()); 627 628 return false; 629 } 630 631 bool AArch64PassConfig::addIRTranslator() { 632 addPass(new IRTranslator(getOptLevel())); 633 return false; 634 } 635 636 void AArch64PassConfig::addPreLegalizeMachineIR() { 637 if (getOptLevel() == CodeGenOpt::None) 638 addPass(createAArch64O0PreLegalizerCombiner()); 639 else { 640 addPass(createAArch64PreLegalizerCombiner()); 641 if (EnableGISelLoadStoreOptPreLegal) 642 addPass(new LoadStoreOpt()); 643 } 644 } 645 646 bool AArch64PassConfig::addLegalizeMachineIR() { 647 addPass(new Legalizer()); 648 return false; 649 } 650 651 void AArch64PassConfig::addPreRegBankSelect() { 652 bool IsOptNone = getOptLevel() == CodeGenOpt::None; 653 if (!IsOptNone) { 654 addPass(createAArch64PostLegalizerCombiner(IsOptNone)); 655 if (EnableGISelLoadStoreOptPostLegal) 656 addPass(new LoadStoreOpt()); 657 } 658 addPass(createAArch64PostLegalizerLowering()); 659 } 660 661 bool AArch64PassConfig::addRegBankSelect() { 662 addPass(new RegBankSelect()); 663 return false; 664 } 665 666 void AArch64PassConfig::addPreGlobalInstructionSelect() { 667 addPass(new Localizer()); 668 } 669 670 bool AArch64PassConfig::addGlobalInstructionSelect() { 671 addPass(new InstructionSelect(getOptLevel())); 672 if (getOptLevel() != CodeGenOpt::None) 673 addPass(createAArch64PostSelectOptimize()); 674 return false; 675 } 676 677 void AArch64PassConfig::addMachineSSAOptimization() { 678 // Run default MachineSSAOptimization first. 679 TargetPassConfig::addMachineSSAOptimization(); 680 681 if (TM->getOptLevel() != CodeGenOpt::None) 682 addPass(createAArch64MIPeepholeOptPass()); 683 } 684 685 bool AArch64PassConfig::addILPOpts() { 686 if (EnableCondOpt) 687 addPass(createAArch64ConditionOptimizerPass()); 688 if (EnableCCMP) 689 addPass(createAArch64ConditionalCompares()); 690 if (EnableMCR) 691 addPass(&MachineCombinerID); 692 if (EnableCondBrTuning) 693 addPass(createAArch64CondBrTuning()); 694 if (EnableEarlyIfConversion) 695 addPass(&EarlyIfConverterID); 696 if (EnableStPairSuppress) 697 addPass(createAArch64StorePairSuppressPass()); 698 addPass(createAArch64SIMDInstrOptPass()); 699 if (TM->getOptLevel() != CodeGenOpt::None) 700 addPass(createAArch64StackTaggingPreRAPass()); 701 return true; 702 } 703 704 void AArch64PassConfig::addPreRegAlloc() { 705 // Change dead register definitions to refer to the zero register. 706 if (TM->getOptLevel() != CodeGenOpt::None && EnableDeadRegisterElimination) 707 addPass(createAArch64DeadRegisterDefinitions()); 708 709 // Use AdvSIMD scalar instructions whenever profitable. 710 if (TM->getOptLevel() != CodeGenOpt::None && EnableAdvSIMDScalar) { 711 addPass(createAArch64AdvSIMDScalar()); 712 // The AdvSIMD pass may produce copies that can be rewritten to 713 // be register coalescer friendly. 714 addPass(&PeepholeOptimizerID); 715 } 716 } 717 718 void AArch64PassConfig::addPostRegAlloc() { 719 // Remove redundant copy instructions. 720 if (TM->getOptLevel() != CodeGenOpt::None && EnableRedundantCopyElimination) 721 addPass(createAArch64RedundantCopyEliminationPass()); 722 723 if (TM->getOptLevel() != CodeGenOpt::None && usingDefaultRegAlloc()) 724 // Improve performance for some FP/SIMD code for A57. 725 addPass(createAArch64A57FPLoadBalancing()); 726 } 727 728 void AArch64PassConfig::addPreSched2() { 729 // Lower homogeneous frame instructions 730 if (EnableHomogeneousPrologEpilog) 731 addPass(createAArch64LowerHomogeneousPrologEpilogPass()); 732 // Expand some pseudo instructions to allow proper scheduling. 733 addPass(createAArch64ExpandPseudoPass()); 734 // Use load/store pair instructions when possible. 735 if (TM->getOptLevel() != CodeGenOpt::None) { 736 if (EnableLoadStoreOpt) 737 addPass(createAArch64LoadStoreOptimizationPass()); 738 } 739 740 // The AArch64SpeculationHardeningPass destroys dominator tree and natural 741 // loop info, which is needed for the FalkorHWPFFixPass and also later on. 742 // Therefore, run the AArch64SpeculationHardeningPass before the 743 // FalkorHWPFFixPass to avoid recomputing dominator tree and natural loop 744 // info. 745 addPass(createAArch64SpeculationHardeningPass()); 746 747 addPass(createAArch64IndirectThunks()); 748 addPass(createAArch64SLSHardeningPass()); 749 750 if (TM->getOptLevel() != CodeGenOpt::None) { 751 if (EnableFalkorHWPFFix) 752 addPass(createFalkorHWPFFixPass()); 753 } 754 } 755 756 void AArch64PassConfig::addPreEmitPass() { 757 // Machine Block Placement might have created new opportunities when run 758 // at O3, where the Tail Duplication Threshold is set to 4 instructions. 759 // Run the load/store optimizer once more. 760 if (TM->getOptLevel() >= CodeGenOpt::Aggressive && EnableLoadStoreOpt) 761 addPass(createAArch64LoadStoreOptimizationPass()); 762 763 addPass(createAArch64A53Fix835769()); 764 765 if (EnableBranchTargets) 766 addPass(createAArch64BranchTargetsPass()); 767 768 // Relax conditional branch instructions if they're otherwise out of 769 // range of their destination. 770 if (BranchRelaxation) 771 addPass(&BranchRelaxationPassID); 772 773 if (TM->getTargetTriple().isOSWindows()) { 774 // Identify valid longjmp targets for Windows Control Flow Guard. 775 addPass(createCFGuardLongjmpPass()); 776 // Identify valid eh continuation targets for Windows EHCont Guard. 777 addPass(createEHContGuardCatchretPass()); 778 } 779 780 if (TM->getOptLevel() != CodeGenOpt::None && EnableCompressJumpTables) 781 addPass(createAArch64CompressJumpTablesPass()); 782 783 if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH && 784 TM->getTargetTriple().isOSBinFormatMachO()) 785 addPass(createAArch64CollectLOHPass()); 786 } 787 788 void AArch64PassConfig::addPreEmitPass2() { 789 // SVE bundles move prefixes with destructive operations. BLR_RVMARKER pseudo 790 // instructions are lowered to bundles as well. 791 addPass(createUnpackMachineBundles(nullptr)); 792 } 793 794 yaml::MachineFunctionInfo * 795 AArch64TargetMachine::createDefaultFuncInfoYAML() const { 796 return new yaml::AArch64FunctionInfo(); 797 } 798 799 yaml::MachineFunctionInfo * 800 AArch64TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const { 801 const auto *MFI = MF.getInfo<AArch64FunctionInfo>(); 802 return new yaml::AArch64FunctionInfo(*MFI); 803 } 804 805 bool AArch64TargetMachine::parseMachineFunctionInfo( 806 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS, 807 SMDiagnostic &Error, SMRange &SourceRange) const { 808 const auto &YamlMFI = 809 reinterpret_cast<const yaml::AArch64FunctionInfo &>(MFI); 810 MachineFunction &MF = PFS.MF; 811 MF.getInfo<AArch64FunctionInfo>()->initializeBaseYamlFields(YamlMFI); 812 return false; 813 } 814