1 //===-- AArch64TargetMachine.cpp - Define TargetMachine for AArch64 -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // 10 //===----------------------------------------------------------------------===// 11 12 #include "AArch64TargetMachine.h" 13 #include "AArch64.h" 14 #include "AArch64MachineFunctionInfo.h" 15 #include "AArch64MachineScheduler.h" 16 #include "AArch64MacroFusion.h" 17 #include "AArch64Subtarget.h" 18 #include "AArch64TargetObjectFile.h" 19 #include "AArch64TargetTransformInfo.h" 20 #include "MCTargetDesc/AArch64MCTargetDesc.h" 21 #include "TargetInfo/AArch64TargetInfo.h" 22 #include "llvm/ADT/STLExtras.h" 23 #include "llvm/ADT/Triple.h" 24 #include "llvm/Analysis/TargetTransformInfo.h" 25 #include "llvm/CodeGen/CFIFixup.h" 26 #include "llvm/CodeGen/CSEConfigBase.h" 27 #include "llvm/CodeGen/GlobalISel/CSEInfo.h" 28 #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 29 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 30 #include "llvm/CodeGen/GlobalISel/Legalizer.h" 31 #include "llvm/CodeGen/GlobalISel/LoadStoreOpt.h" 32 #include "llvm/CodeGen/GlobalISel/Localizer.h" 33 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 34 #include "llvm/CodeGen/MIRParser/MIParser.h" 35 #include "llvm/CodeGen/MachineScheduler.h" 36 #include "llvm/CodeGen/Passes.h" 37 #include "llvm/CodeGen/TargetInstrInfo.h" 38 #include "llvm/CodeGen/TargetPassConfig.h" 39 #include "llvm/IR/Attributes.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/InitializePasses.h" 42 #include "llvm/MC/MCAsmInfo.h" 43 #include "llvm/MC/MCTargetOptions.h" 44 #include "llvm/MC/TargetRegistry.h" 45 #include "llvm/Pass.h" 46 #include "llvm/Support/CodeGen.h" 47 #include "llvm/Support/CommandLine.h" 48 #include "llvm/Target/TargetLoweringObjectFile.h" 49 #include "llvm/Target/TargetOptions.h" 50 #include "llvm/Transforms/CFGuard.h" 51 #include "llvm/Transforms/Scalar.h" 52 #include <memory> 53 #include <string> 54 55 using namespace llvm; 56 57 static cl::opt<bool> EnableCCMP("aarch64-enable-ccmp", 58 cl::desc("Enable the CCMP formation pass"), 59 cl::init(true), cl::Hidden); 60 61 static cl::opt<bool> 62 EnableCondBrTuning("aarch64-enable-cond-br-tune", 63 cl::desc("Enable the conditional branch tuning pass"), 64 cl::init(true), cl::Hidden); 65 66 static cl::opt<bool> EnableAArch64CopyPropagation( 67 "aarch64-enable-copy-propagation", 68 cl::desc("Enable the copy propagation with AArch64 copy instr"), 69 cl::init(true), cl::Hidden); 70 71 static cl::opt<bool> EnableMCR("aarch64-enable-mcr", 72 cl::desc("Enable the machine combiner pass"), 73 cl::init(true), cl::Hidden); 74 75 static cl::opt<bool> EnableStPairSuppress("aarch64-enable-stp-suppress", 76 cl::desc("Suppress STP for AArch64"), 77 cl::init(true), cl::Hidden); 78 79 static cl::opt<bool> EnableAdvSIMDScalar( 80 "aarch64-enable-simd-scalar", 81 cl::desc("Enable use of AdvSIMD scalar integer instructions"), 82 cl::init(false), cl::Hidden); 83 84 static cl::opt<bool> 85 EnablePromoteConstant("aarch64-enable-promote-const", 86 cl::desc("Enable the promote constant pass"), 87 cl::init(true), cl::Hidden); 88 89 static cl::opt<bool> EnableCollectLOH( 90 "aarch64-enable-collect-loh", 91 cl::desc("Enable the pass that emits the linker optimization hints (LOH)"), 92 cl::init(true), cl::Hidden); 93 94 static cl::opt<bool> 95 EnableDeadRegisterElimination("aarch64-enable-dead-defs", cl::Hidden, 96 cl::desc("Enable the pass that removes dead" 97 " definitons and replaces stores to" 98 " them with stores to the zero" 99 " register"), 100 cl::init(true)); 101 102 static cl::opt<bool> EnableRedundantCopyElimination( 103 "aarch64-enable-copyelim", 104 cl::desc("Enable the redundant copy elimination pass"), cl::init(true), 105 cl::Hidden); 106 107 static cl::opt<bool> EnableLoadStoreOpt("aarch64-enable-ldst-opt", 108 cl::desc("Enable the load/store pair" 109 " optimization pass"), 110 cl::init(true), cl::Hidden); 111 112 static cl::opt<bool> EnableAtomicTidy( 113 "aarch64-enable-atomic-cfg-tidy", cl::Hidden, 114 cl::desc("Run SimplifyCFG after expanding atomic operations" 115 " to make use of cmpxchg flow-based information"), 116 cl::init(true)); 117 118 static cl::opt<bool> 119 EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, 120 cl::desc("Run early if-conversion"), 121 cl::init(true)); 122 123 static cl::opt<bool> 124 EnableCondOpt("aarch64-enable-condopt", 125 cl::desc("Enable the condition optimizer pass"), 126 cl::init(true), cl::Hidden); 127 128 static cl::opt<bool> 129 EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden, 130 cl::desc("Enable optimizations on complex GEPs"), 131 cl::init(true)); 132 133 static cl::opt<bool> 134 BranchRelaxation("aarch64-enable-branch-relax", cl::Hidden, cl::init(true), 135 cl::desc("Relax out of range conditional branches")); 136 137 static cl::opt<bool> EnableCompressJumpTables( 138 "aarch64-enable-compress-jump-tables", cl::Hidden, cl::init(true), 139 cl::desc("Use smallest entry possible for jump tables")); 140 141 // FIXME: Unify control over GlobalMerge. 142 static cl::opt<cl::boolOrDefault> 143 EnableGlobalMerge("aarch64-enable-global-merge", cl::Hidden, 144 cl::desc("Enable the global merge pass")); 145 146 static cl::opt<bool> 147 EnableLoopDataPrefetch("aarch64-enable-loop-data-prefetch", cl::Hidden, 148 cl::desc("Enable the loop data prefetch pass"), 149 cl::init(true)); 150 151 static cl::opt<int> EnableGlobalISelAtO( 152 "aarch64-enable-global-isel-at-O", cl::Hidden, 153 cl::desc("Enable GlobalISel at or below an opt level (-1 to disable)"), 154 cl::init(0)); 155 156 static cl::opt<bool> 157 EnableSVEIntrinsicOpts("aarch64-enable-sve-intrinsic-opts", cl::Hidden, 158 cl::desc("Enable SVE intrinsic opts"), 159 cl::init(true)); 160 161 static cl::opt<bool> EnableFalkorHWPFFix("aarch64-enable-falkor-hwpf-fix", 162 cl::init(true), cl::Hidden); 163 164 static cl::opt<bool> 165 EnableBranchTargets("aarch64-enable-branch-targets", cl::Hidden, 166 cl::desc("Enable the AArch64 branch target pass"), 167 cl::init(true)); 168 169 static cl::opt<unsigned> SVEVectorBitsMaxOpt( 170 "aarch64-sve-vector-bits-max", 171 cl::desc("Assume SVE vector registers are at most this big, " 172 "with zero meaning no maximum size is assumed."), 173 cl::init(0), cl::Hidden); 174 175 static cl::opt<unsigned> SVEVectorBitsMinOpt( 176 "aarch64-sve-vector-bits-min", 177 cl::desc("Assume SVE vector registers are at least this big, " 178 "with zero meaning no minimum size is assumed."), 179 cl::init(0), cl::Hidden); 180 181 extern cl::opt<bool> EnableHomogeneousPrologEpilog; 182 183 static cl::opt<bool> EnableGISelLoadStoreOptPreLegal( 184 "aarch64-enable-gisel-ldst-prelegal", 185 cl::desc("Enable GlobalISel's pre-legalizer load/store optimization pass"), 186 cl::init(true), cl::Hidden); 187 188 static cl::opt<bool> EnableGISelLoadStoreOptPostLegal( 189 "aarch64-enable-gisel-ldst-postlegal", 190 cl::desc("Enable GlobalISel's post-legalizer load/store optimization pass"), 191 cl::init(false), cl::Hidden); 192 193 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Target() { 194 // Register the target. 195 RegisterTargetMachine<AArch64leTargetMachine> X(getTheAArch64leTarget()); 196 RegisterTargetMachine<AArch64beTargetMachine> Y(getTheAArch64beTarget()); 197 RegisterTargetMachine<AArch64leTargetMachine> Z(getTheARM64Target()); 198 RegisterTargetMachine<AArch64leTargetMachine> W(getTheARM64_32Target()); 199 RegisterTargetMachine<AArch64leTargetMachine> V(getTheAArch64_32Target()); 200 auto PR = PassRegistry::getPassRegistry(); 201 initializeGlobalISel(*PR); 202 initializeAArch64A53Fix835769Pass(*PR); 203 initializeAArch64A57FPLoadBalancingPass(*PR); 204 initializeAArch64AdvSIMDScalarPass(*PR); 205 initializeAArch64BranchTargetsPass(*PR); 206 initializeAArch64CollectLOHPass(*PR); 207 initializeAArch64CompressJumpTablesPass(*PR); 208 initializeAArch64ConditionalComparesPass(*PR); 209 initializeAArch64ConditionOptimizerPass(*PR); 210 initializeAArch64DeadRegisterDefinitionsPass(*PR); 211 initializeAArch64ExpandPseudoPass(*PR); 212 initializeAArch64LoadStoreOptPass(*PR); 213 initializeAArch64MIPeepholeOptPass(*PR); 214 initializeAArch64SIMDInstrOptPass(*PR); 215 initializeAArch64O0PreLegalizerCombinerPass(*PR); 216 initializeAArch64PreLegalizerCombinerPass(*PR); 217 initializeAArch64PostLegalizerCombinerPass(*PR); 218 initializeAArch64PostLegalizerLoweringPass(*PR); 219 initializeAArch64PostSelectOptimizePass(*PR); 220 initializeAArch64PromoteConstantPass(*PR); 221 initializeAArch64RedundantCopyEliminationPass(*PR); 222 initializeAArch64StorePairSuppressPass(*PR); 223 initializeFalkorHWPFFixPass(*PR); 224 initializeFalkorMarkStridedAccessesLegacyPass(*PR); 225 initializeLDTLSCleanupPass(*PR); 226 initializeSVEIntrinsicOptsPass(*PR); 227 initializeAArch64SpeculationHardeningPass(*PR); 228 initializeAArch64SLSHardeningPass(*PR); 229 initializeAArch64StackTaggingPass(*PR); 230 initializeAArch64StackTaggingPreRAPass(*PR); 231 initializeAArch64LowerHomogeneousPrologEpilogPass(*PR); 232 } 233 234 //===----------------------------------------------------------------------===// 235 // AArch64 Lowering public interface. 236 //===----------------------------------------------------------------------===// 237 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 238 if (TT.isOSBinFormatMachO()) 239 return std::make_unique<AArch64_MachoTargetObjectFile>(); 240 if (TT.isOSBinFormatCOFF()) 241 return std::make_unique<AArch64_COFFTargetObjectFile>(); 242 243 return std::make_unique<AArch64_ELFTargetObjectFile>(); 244 } 245 246 // Helper function to build a DataLayout string 247 static std::string computeDataLayout(const Triple &TT, 248 const MCTargetOptions &Options, 249 bool LittleEndian) { 250 if (TT.isOSBinFormatMachO()) { 251 if (TT.getArch() == Triple::aarch64_32) 252 return "e-m:o-p:32:32-i64:64-i128:128-n32:64-S128"; 253 return "e-m:o-i64:64-i128:128-n32:64-S128"; 254 } 255 if (TT.isOSBinFormatCOFF()) 256 return "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"; 257 std::string Endian = LittleEndian ? "e" : "E"; 258 std::string Ptr32 = TT.getEnvironment() == Triple::GNUILP32 ? "-p:32:32" : ""; 259 return Endian + "-m:e" + Ptr32 + 260 "-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"; 261 } 262 263 static StringRef computeDefaultCPU(const Triple &TT, StringRef CPU) { 264 if (CPU.empty() && TT.isArm64e()) 265 return "apple-a12"; 266 return CPU; 267 } 268 269 static Reloc::Model getEffectiveRelocModel(const Triple &TT, 270 Optional<Reloc::Model> RM) { 271 // AArch64 Darwin and Windows are always PIC. 272 if (TT.isOSDarwin() || TT.isOSWindows()) 273 return Reloc::PIC_; 274 // On ELF platforms the default static relocation model has a smart enough 275 // linker to cope with referencing external symbols defined in a shared 276 // library. Hence DynamicNoPIC doesn't need to be promoted to PIC. 277 if (!RM || *RM == Reloc::DynamicNoPIC) 278 return Reloc::Static; 279 return *RM; 280 } 281 282 static CodeModel::Model 283 getEffectiveAArch64CodeModel(const Triple &TT, Optional<CodeModel::Model> CM, 284 bool JIT) { 285 if (CM) { 286 if (*CM != CodeModel::Small && *CM != CodeModel::Tiny && 287 *CM != CodeModel::Large) { 288 report_fatal_error( 289 "Only small, tiny and large code models are allowed on AArch64"); 290 } else if (*CM == CodeModel::Tiny && !TT.isOSBinFormatELF()) 291 report_fatal_error("tiny code model is only supported on ELF"); 292 return *CM; 293 } 294 // The default MCJIT memory managers make no guarantees about where they can 295 // find an executable page; JITed code needs to be able to refer to globals 296 // no matter how far away they are. 297 // We should set the CodeModel::Small for Windows ARM64 in JIT mode, 298 // since with large code model LLVM generating 4 MOV instructions, and 299 // Windows doesn't support relocating these long branch (4 MOVs). 300 if (JIT && !TT.isOSWindows()) 301 return CodeModel::Large; 302 return CodeModel::Small; 303 } 304 305 /// Create an AArch64 architecture model. 306 /// 307 AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT, 308 StringRef CPU, StringRef FS, 309 const TargetOptions &Options, 310 Optional<Reloc::Model> RM, 311 Optional<CodeModel::Model> CM, 312 CodeGenOpt::Level OL, bool JIT, 313 bool LittleEndian) 314 : LLVMTargetMachine(T, 315 computeDataLayout(TT, Options.MCOptions, LittleEndian), 316 TT, computeDefaultCPU(TT, CPU), FS, Options, 317 getEffectiveRelocModel(TT, RM), 318 getEffectiveAArch64CodeModel(TT, CM, JIT), OL), 319 TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian) { 320 initAsmInfo(); 321 322 if (TT.isOSBinFormatMachO()) { 323 this->Options.TrapUnreachable = true; 324 this->Options.NoTrapAfterNoreturn = true; 325 } 326 327 if (getMCAsmInfo()->usesWindowsCFI()) { 328 // Unwinding can get confused if the last instruction in an 329 // exception-handling region (function, funclet, try block, etc.) 330 // is a call. 331 // 332 // FIXME: We could elide the trap if the next instruction would be in 333 // the same region anyway. 334 this->Options.TrapUnreachable = true; 335 } 336 337 if (this->Options.TLSSize == 0) // default 338 this->Options.TLSSize = 24; 339 if ((getCodeModel() == CodeModel::Small || 340 getCodeModel() == CodeModel::Kernel) && 341 this->Options.TLSSize > 32) 342 // for the small (and kernel) code model, the maximum TLS size is 4GiB 343 this->Options.TLSSize = 32; 344 else if (getCodeModel() == CodeModel::Tiny && this->Options.TLSSize > 24) 345 // for the tiny code model, the maximum TLS size is 1MiB (< 16MiB) 346 this->Options.TLSSize = 24; 347 348 // Enable GlobalISel at or below EnableGlobalISelAt0, unless this is 349 // MachO/CodeModel::Large, which GlobalISel does not support. 350 if (getOptLevel() <= EnableGlobalISelAtO && 351 TT.getArch() != Triple::aarch64_32 && 352 TT.getEnvironment() != Triple::GNUILP32 && 353 !(getCodeModel() == CodeModel::Large && TT.isOSBinFormatMachO())) { 354 setGlobalISel(true); 355 setGlobalISelAbort(GlobalISelAbortMode::Disable); 356 } 357 358 // AArch64 supports the MachineOutliner. 359 setMachineOutliner(true); 360 361 // AArch64 supports default outlining behaviour. 362 setSupportsDefaultOutlining(true); 363 364 // AArch64 supports the debug entry values. 365 setSupportsDebugEntryValues(true); 366 367 // AArch64 supports fixing up the DWARF unwind information. 368 if (!getMCAsmInfo()->usesWindowsCFI()) 369 setCFIFixup(true); 370 } 371 372 AArch64TargetMachine::~AArch64TargetMachine() = default; 373 374 const AArch64Subtarget * 375 AArch64TargetMachine::getSubtargetImpl(const Function &F) const { 376 Attribute CPUAttr = F.getFnAttribute("target-cpu"); 377 Attribute TuneAttr = F.getFnAttribute("tune-cpu"); 378 Attribute FSAttr = F.getFnAttribute("target-features"); 379 380 std::string CPU = 381 CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU; 382 std::string TuneCPU = 383 TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU; 384 std::string FS = 385 FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS; 386 387 SmallString<512> Key; 388 389 unsigned MinSVEVectorSize = 0; 390 unsigned MaxSVEVectorSize = 0; 391 Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange); 392 if (VScaleRangeAttr.isValid()) { 393 Optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax(); 394 MinSVEVectorSize = VScaleRangeAttr.getVScaleRangeMin() * 128; 395 MaxSVEVectorSize = VScaleMax ? *VScaleMax * 128 : 0; 396 } else { 397 MinSVEVectorSize = SVEVectorBitsMinOpt; 398 MaxSVEVectorSize = SVEVectorBitsMaxOpt; 399 } 400 401 assert(MinSVEVectorSize % 128 == 0 && 402 "SVE requires vector length in multiples of 128!"); 403 assert(MaxSVEVectorSize % 128 == 0 && 404 "SVE requires vector length in multiples of 128!"); 405 assert((MaxSVEVectorSize >= MinSVEVectorSize || MaxSVEVectorSize == 0) && 406 "Minimum SVE vector size should not be larger than its maximum!"); 407 408 // Sanitize user input in case of no asserts 409 if (MaxSVEVectorSize == 0) 410 MinSVEVectorSize = (MinSVEVectorSize / 128) * 128; 411 else { 412 MinSVEVectorSize = 413 (std::min(MinSVEVectorSize, MaxSVEVectorSize) / 128) * 128; 414 MaxSVEVectorSize = 415 (std::max(MinSVEVectorSize, MaxSVEVectorSize) / 128) * 128; 416 } 417 418 Key += "SVEMin"; 419 Key += std::to_string(MinSVEVectorSize); 420 Key += "SVEMax"; 421 Key += std::to_string(MaxSVEVectorSize); 422 Key += CPU; 423 Key += TuneCPU; 424 Key += FS; 425 426 auto &I = SubtargetMap[Key]; 427 if (!I) { 428 // This needs to be done before we create a new subtarget since any 429 // creation will depend on the TM and the code generation flags on the 430 // function that reside in TargetOptions. 431 resetTargetOptions(F); 432 I = std::make_unique<AArch64Subtarget>(TargetTriple, CPU, TuneCPU, FS, 433 *this, isLittle, MinSVEVectorSize, 434 MaxSVEVectorSize); 435 } 436 return I.get(); 437 } 438 439 void AArch64leTargetMachine::anchor() { } 440 441 AArch64leTargetMachine::AArch64leTargetMachine( 442 const Target &T, const Triple &TT, StringRef CPU, StringRef FS, 443 const TargetOptions &Options, Optional<Reloc::Model> RM, 444 Optional<CodeModel::Model> CM, CodeGenOpt::Level OL, bool JIT) 445 : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, JIT, true) {} 446 447 void AArch64beTargetMachine::anchor() { } 448 449 AArch64beTargetMachine::AArch64beTargetMachine( 450 const Target &T, const Triple &TT, StringRef CPU, StringRef FS, 451 const TargetOptions &Options, Optional<Reloc::Model> RM, 452 Optional<CodeModel::Model> CM, CodeGenOpt::Level OL, bool JIT) 453 : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, JIT, false) {} 454 455 namespace { 456 457 /// AArch64 Code Generator Pass Configuration Options. 458 class AArch64PassConfig : public TargetPassConfig { 459 public: 460 AArch64PassConfig(AArch64TargetMachine &TM, PassManagerBase &PM) 461 : TargetPassConfig(TM, PM) { 462 if (TM.getOptLevel() != CodeGenOpt::None) 463 substitutePass(&PostRASchedulerID, &PostMachineSchedulerID); 464 } 465 466 AArch64TargetMachine &getAArch64TargetMachine() const { 467 return getTM<AArch64TargetMachine>(); 468 } 469 470 ScheduleDAGInstrs * 471 createMachineScheduler(MachineSchedContext *C) const override { 472 const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>(); 473 ScheduleDAGMILive *DAG = createGenericSchedLive(C); 474 DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); 475 DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); 476 if (ST.hasFusion()) 477 DAG->addMutation(createAArch64MacroFusionDAGMutation()); 478 return DAG; 479 } 480 481 ScheduleDAGInstrs * 482 createPostMachineScheduler(MachineSchedContext *C) const override { 483 const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>(); 484 ScheduleDAGMI *DAG = 485 new ScheduleDAGMI(C, std::make_unique<AArch64PostRASchedStrategy>(C), 486 /* RemoveKillFlags=*/true); 487 if (ST.hasFusion()) { 488 // Run the Macro Fusion after RA again since literals are expanded from 489 // pseudos then (v. addPreSched2()). 490 DAG->addMutation(createAArch64MacroFusionDAGMutation()); 491 return DAG; 492 } 493 494 return DAG; 495 } 496 497 void addIRPasses() override; 498 bool addPreISel() override; 499 void addCodeGenPrepare() override; 500 bool addInstSelector() override; 501 bool addIRTranslator() override; 502 void addPreLegalizeMachineIR() override; 503 bool addLegalizeMachineIR() override; 504 void addPreRegBankSelect() override; 505 bool addRegBankSelect() override; 506 void addPreGlobalInstructionSelect() override; 507 bool addGlobalInstructionSelect() override; 508 void addMachineSSAOptimization() override; 509 bool addILPOpts() override; 510 void addPreRegAlloc() override; 511 void addPostRegAlloc() override; 512 void addPreSched2() override; 513 void addPreEmitPass() override; 514 void addPreEmitPass2() override; 515 516 std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 517 }; 518 519 } // end anonymous namespace 520 521 TargetTransformInfo 522 AArch64TargetMachine::getTargetTransformInfo(const Function &F) const { 523 return TargetTransformInfo(AArch64TTIImpl(this, F)); 524 } 525 526 TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) { 527 return new AArch64PassConfig(*this, PM); 528 } 529 530 std::unique_ptr<CSEConfigBase> AArch64PassConfig::getCSEConfig() const { 531 return getStandardCSEConfigForOpt(TM->getOptLevel()); 532 } 533 534 void AArch64PassConfig::addIRPasses() { 535 // Always expand atomic operations, we don't deal with atomicrmw or cmpxchg 536 // ourselves. 537 addPass(createAtomicExpandPass()); 538 539 // Expand any SVE vector library calls that we can't code generate directly. 540 if (EnableSVEIntrinsicOpts && TM->getOptLevel() == CodeGenOpt::Aggressive) 541 addPass(createSVEIntrinsicOptsPass()); 542 543 // Cmpxchg instructions are often used with a subsequent comparison to 544 // determine whether it succeeded. We can exploit existing control-flow in 545 // ldrex/strex loops to simplify this, but it needs tidying up. 546 if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy) 547 addPass(createCFGSimplificationPass(SimplifyCFGOptions() 548 .forwardSwitchCondToPhi(true) 549 .convertSwitchRangeToICmp(true) 550 .convertSwitchToLookupTable(true) 551 .needCanonicalLoops(false) 552 .hoistCommonInsts(true) 553 .sinkCommonInsts(true))); 554 555 // Run LoopDataPrefetch 556 // 557 // Run this before LSR to remove the multiplies involved in computing the 558 // pointer values N iterations ahead. 559 if (TM->getOptLevel() != CodeGenOpt::None) { 560 if (EnableLoopDataPrefetch) 561 addPass(createLoopDataPrefetchPass()); 562 if (EnableFalkorHWPFFix) 563 addPass(createFalkorMarkStridedAccessesPass()); 564 } 565 566 if (TM->getOptLevel() == CodeGenOpt::Aggressive && EnableGEPOpt) { 567 // Call SeparateConstOffsetFromGEP pass to extract constants within indices 568 // and lower a GEP with multiple indices to either arithmetic operations or 569 // multiple GEPs with single index. 570 addPass(createSeparateConstOffsetFromGEPPass(true)); 571 // Call EarlyCSE pass to find and remove subexpressions in the lowered 572 // result. 573 addPass(createEarlyCSEPass()); 574 // Do loop invariant code motion in case part of the lowered result is 575 // invariant. 576 addPass(createLICMPass()); 577 } 578 579 TargetPassConfig::addIRPasses(); 580 581 addPass(createAArch64StackTaggingPass( 582 /*IsOptNone=*/TM->getOptLevel() == CodeGenOpt::None)); 583 584 // Match interleaved memory accesses to ldN/stN intrinsics. 585 if (TM->getOptLevel() != CodeGenOpt::None) { 586 addPass(createInterleavedLoadCombinePass()); 587 addPass(createInterleavedAccessPass()); 588 } 589 590 // Add Control Flow Guard checks. 591 if (TM->getTargetTriple().isOSWindows()) 592 addPass(createCFGuardCheckPass()); 593 594 if (TM->Options.JMCInstrument) 595 addPass(createJMCInstrumenterPass()); 596 } 597 598 // Pass Pipeline Configuration 599 bool AArch64PassConfig::addPreISel() { 600 // Run promote constant before global merge, so that the promoted constants 601 // get a chance to be merged 602 if (TM->getOptLevel() != CodeGenOpt::None && EnablePromoteConstant) 603 addPass(createAArch64PromoteConstantPass()); 604 // FIXME: On AArch64, this depends on the type. 605 // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes(). 606 // and the offset has to be a multiple of the related size in bytes. 607 if ((TM->getOptLevel() != CodeGenOpt::None && 608 EnableGlobalMerge == cl::BOU_UNSET) || 609 EnableGlobalMerge == cl::BOU_TRUE) { 610 bool OnlyOptimizeForSize = (TM->getOptLevel() < CodeGenOpt::Aggressive) && 611 (EnableGlobalMerge == cl::BOU_UNSET); 612 613 // Merging of extern globals is enabled by default on non-Mach-O as we 614 // expect it to be generally either beneficial or harmless. On Mach-O it 615 // is disabled as we emit the .subsections_via_symbols directive which 616 // means that merging extern globals is not safe. 617 bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO(); 618 619 // FIXME: extern global merging is only enabled when we optimise for size 620 // because there are some regressions with it also enabled for performance. 621 if (!OnlyOptimizeForSize) 622 MergeExternalByDefault = false; 623 624 addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize, 625 MergeExternalByDefault)); 626 } 627 628 return false; 629 } 630 631 void AArch64PassConfig::addCodeGenPrepare() { 632 if (getOptLevel() != CodeGenOpt::None) 633 addPass(createTypePromotionPass()); 634 TargetPassConfig::addCodeGenPrepare(); 635 } 636 637 bool AArch64PassConfig::addInstSelector() { 638 addPass(createAArch64ISelDag(getAArch64TargetMachine(), getOptLevel())); 639 640 // For ELF, cleanup any local-dynamic TLS accesses (i.e. combine as many 641 // references to _TLS_MODULE_BASE_ as possible. 642 if (TM->getTargetTriple().isOSBinFormatELF() && 643 getOptLevel() != CodeGenOpt::None) 644 addPass(createAArch64CleanupLocalDynamicTLSPass()); 645 646 return false; 647 } 648 649 bool AArch64PassConfig::addIRTranslator() { 650 addPass(new IRTranslator(getOptLevel())); 651 return false; 652 } 653 654 void AArch64PassConfig::addPreLegalizeMachineIR() { 655 if (getOptLevel() == CodeGenOpt::None) 656 addPass(createAArch64O0PreLegalizerCombiner()); 657 else { 658 addPass(createAArch64PreLegalizerCombiner()); 659 if (EnableGISelLoadStoreOptPreLegal) 660 addPass(new LoadStoreOpt()); 661 } 662 } 663 664 bool AArch64PassConfig::addLegalizeMachineIR() { 665 addPass(new Legalizer()); 666 return false; 667 } 668 669 void AArch64PassConfig::addPreRegBankSelect() { 670 bool IsOptNone = getOptLevel() == CodeGenOpt::None; 671 if (!IsOptNone) { 672 addPass(createAArch64PostLegalizerCombiner(IsOptNone)); 673 if (EnableGISelLoadStoreOptPostLegal) 674 addPass(new LoadStoreOpt()); 675 } 676 addPass(createAArch64PostLegalizerLowering()); 677 } 678 679 bool AArch64PassConfig::addRegBankSelect() { 680 addPass(new RegBankSelect()); 681 return false; 682 } 683 684 void AArch64PassConfig::addPreGlobalInstructionSelect() { 685 addPass(new Localizer()); 686 } 687 688 bool AArch64PassConfig::addGlobalInstructionSelect() { 689 addPass(new InstructionSelect(getOptLevel())); 690 if (getOptLevel() != CodeGenOpt::None) 691 addPass(createAArch64PostSelectOptimize()); 692 return false; 693 } 694 695 void AArch64PassConfig::addMachineSSAOptimization() { 696 // Run default MachineSSAOptimization first. 697 TargetPassConfig::addMachineSSAOptimization(); 698 699 if (TM->getOptLevel() != CodeGenOpt::None) 700 addPass(createAArch64MIPeepholeOptPass()); 701 } 702 703 bool AArch64PassConfig::addILPOpts() { 704 if (EnableCondOpt) 705 addPass(createAArch64ConditionOptimizerPass()); 706 if (EnableCCMP) 707 addPass(createAArch64ConditionalCompares()); 708 if (EnableMCR) 709 addPass(&MachineCombinerID); 710 if (EnableCondBrTuning) 711 addPass(createAArch64CondBrTuning()); 712 if (EnableEarlyIfConversion) 713 addPass(&EarlyIfConverterID); 714 if (EnableStPairSuppress) 715 addPass(createAArch64StorePairSuppressPass()); 716 addPass(createAArch64SIMDInstrOptPass()); 717 if (TM->getOptLevel() != CodeGenOpt::None) 718 addPass(createAArch64StackTaggingPreRAPass()); 719 return true; 720 } 721 722 void AArch64PassConfig::addPreRegAlloc() { 723 // Change dead register definitions to refer to the zero register. 724 if (TM->getOptLevel() != CodeGenOpt::None && EnableDeadRegisterElimination) 725 addPass(createAArch64DeadRegisterDefinitions()); 726 727 // Use AdvSIMD scalar instructions whenever profitable. 728 if (TM->getOptLevel() != CodeGenOpt::None && EnableAdvSIMDScalar) { 729 addPass(createAArch64AdvSIMDScalar()); 730 // The AdvSIMD pass may produce copies that can be rewritten to 731 // be register coalescer friendly. 732 addPass(&PeepholeOptimizerID); 733 } 734 } 735 736 void AArch64PassConfig::addPostRegAlloc() { 737 // Remove redundant copy instructions. 738 if (TM->getOptLevel() != CodeGenOpt::None && EnableRedundantCopyElimination) 739 addPass(createAArch64RedundantCopyEliminationPass()); 740 741 if (TM->getOptLevel() != CodeGenOpt::None && usingDefaultRegAlloc()) 742 // Improve performance for some FP/SIMD code for A57. 743 addPass(createAArch64A57FPLoadBalancing()); 744 } 745 746 void AArch64PassConfig::addPreSched2() { 747 // Lower homogeneous frame instructions 748 if (EnableHomogeneousPrologEpilog) 749 addPass(createAArch64LowerHomogeneousPrologEpilogPass()); 750 // Expand some pseudo instructions to allow proper scheduling. 751 addPass(createAArch64ExpandPseudoPass()); 752 // Use load/store pair instructions when possible. 753 if (TM->getOptLevel() != CodeGenOpt::None) { 754 if (EnableLoadStoreOpt) 755 addPass(createAArch64LoadStoreOptimizationPass()); 756 } 757 758 // The AArch64SpeculationHardeningPass destroys dominator tree and natural 759 // loop info, which is needed for the FalkorHWPFFixPass and also later on. 760 // Therefore, run the AArch64SpeculationHardeningPass before the 761 // FalkorHWPFFixPass to avoid recomputing dominator tree and natural loop 762 // info. 763 addPass(createAArch64SpeculationHardeningPass()); 764 765 addPass(createAArch64IndirectThunks()); 766 addPass(createAArch64SLSHardeningPass()); 767 768 if (TM->getOptLevel() != CodeGenOpt::None) { 769 if (EnableFalkorHWPFFix) 770 addPass(createFalkorHWPFFixPass()); 771 } 772 } 773 774 void AArch64PassConfig::addPreEmitPass() { 775 // Machine Block Placement might have created new opportunities when run 776 // at O3, where the Tail Duplication Threshold is set to 4 instructions. 777 // Run the load/store optimizer once more. 778 if (TM->getOptLevel() >= CodeGenOpt::Aggressive && EnableLoadStoreOpt) 779 addPass(createAArch64LoadStoreOptimizationPass()); 780 781 if (TM->getOptLevel() >= CodeGenOpt::Aggressive && 782 EnableAArch64CopyPropagation) 783 addPass(createMachineCopyPropagationPass(true)); 784 785 addPass(createAArch64A53Fix835769()); 786 787 if (EnableBranchTargets) 788 addPass(createAArch64BranchTargetsPass()); 789 790 // Relax conditional branch instructions if they're otherwise out of 791 // range of their destination. 792 if (BranchRelaxation) 793 addPass(&BranchRelaxationPassID); 794 795 if (TM->getTargetTriple().isOSWindows()) { 796 // Identify valid longjmp targets for Windows Control Flow Guard. 797 addPass(createCFGuardLongjmpPass()); 798 // Identify valid eh continuation targets for Windows EHCont Guard. 799 addPass(createEHContGuardCatchretPass()); 800 } 801 802 if (TM->getOptLevel() != CodeGenOpt::None && EnableCompressJumpTables) 803 addPass(createAArch64CompressJumpTablesPass()); 804 805 if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH && 806 TM->getTargetTriple().isOSBinFormatMachO()) 807 addPass(createAArch64CollectLOHPass()); 808 } 809 810 void AArch64PassConfig::addPreEmitPass2() { 811 // SVE bundles move prefixes with destructive operations. BLR_RVMARKER pseudo 812 // instructions are lowered to bundles as well. 813 addPass(createUnpackMachineBundles(nullptr)); 814 } 815 816 yaml::MachineFunctionInfo * 817 AArch64TargetMachine::createDefaultFuncInfoYAML() const { 818 return new yaml::AArch64FunctionInfo(); 819 } 820 821 yaml::MachineFunctionInfo * 822 AArch64TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const { 823 const auto *MFI = MF.getInfo<AArch64FunctionInfo>(); 824 return new yaml::AArch64FunctionInfo(*MFI); 825 } 826 827 bool AArch64TargetMachine::parseMachineFunctionInfo( 828 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS, 829 SMDiagnostic &Error, SMRange &SourceRange) const { 830 const auto &YamlMFI = static_cast<const yaml::AArch64FunctionInfo &>(MFI); 831 MachineFunction &MF = PFS.MF; 832 MF.getInfo<AArch64FunctionInfo>()->initializeBaseYamlFields(YamlMFI); 833 return false; 834 } 835