1 //===-- AArch64TargetMachine.cpp - Define TargetMachine for AArch64 -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // 10 //===----------------------------------------------------------------------===// 11 12 #include "AArch64TargetMachine.h" 13 #include "AArch64.h" 14 #include "AArch64LoopIdiomTransform.h" 15 #include "AArch64MachineFunctionInfo.h" 16 #include "AArch64MachineScheduler.h" 17 #include "AArch64MacroFusion.h" 18 #include "AArch64Subtarget.h" 19 #include "AArch64TargetObjectFile.h" 20 #include "AArch64TargetTransformInfo.h" 21 #include "MCTargetDesc/AArch64MCTargetDesc.h" 22 #include "TargetInfo/AArch64TargetInfo.h" 23 #include "llvm/ADT/STLExtras.h" 24 #include "llvm/Analysis/TargetTransformInfo.h" 25 #include "llvm/Analysis/ValueTracking.h" 26 #include "llvm/CodeGen/CFIFixup.h" 27 #include "llvm/CodeGen/CSEConfigBase.h" 28 #include "llvm/CodeGen/GlobalISel/CSEInfo.h" 29 #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 30 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 31 #include "llvm/CodeGen/GlobalISel/Legalizer.h" 32 #include "llvm/CodeGen/GlobalISel/LoadStoreOpt.h" 33 #include "llvm/CodeGen/GlobalISel/Localizer.h" 34 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 35 #include "llvm/CodeGen/MIRParser/MIParser.h" 36 #include "llvm/CodeGen/MachineScheduler.h" 37 #include "llvm/CodeGen/Passes.h" 38 #include "llvm/CodeGen/TargetInstrInfo.h" 39 #include "llvm/CodeGen/TargetPassConfig.h" 40 #include "llvm/IR/Attributes.h" 41 #include "llvm/IR/Function.h" 42 #include "llvm/InitializePasses.h" 43 #include "llvm/MC/MCAsmInfo.h" 44 #include "llvm/MC/MCTargetOptions.h" 45 #include "llvm/MC/TargetRegistry.h" 46 #include "llvm/Pass.h" 47 #include "llvm/Passes/PassBuilder.h" 48 #include "llvm/Support/CodeGen.h" 49 #include "llvm/Support/CommandLine.h" 50 #include "llvm/Target/TargetLoweringObjectFile.h" 51 #include "llvm/Target/TargetOptions.h" 52 #include "llvm/TargetParser/Triple.h" 53 #include "llvm/Transforms/CFGuard.h" 54 #include "llvm/Transforms/Scalar.h" 55 #include <memory> 56 #include <optional> 57 #include <string> 58 59 using namespace llvm; 60 61 static cl::opt<bool> EnableCCMP("aarch64-enable-ccmp", 62 cl::desc("Enable the CCMP formation pass"), 63 cl::init(true), cl::Hidden); 64 65 static cl::opt<bool> 66 EnableCondBrTuning("aarch64-enable-cond-br-tune", 67 cl::desc("Enable the conditional branch tuning pass"), 68 cl::init(true), cl::Hidden); 69 70 static cl::opt<bool> EnableAArch64CopyPropagation( 71 "aarch64-enable-copy-propagation", 72 cl::desc("Enable the copy propagation with AArch64 copy instr"), 73 cl::init(true), cl::Hidden); 74 75 static cl::opt<bool> EnableMCR("aarch64-enable-mcr", 76 cl::desc("Enable the machine combiner pass"), 77 cl::init(true), cl::Hidden); 78 79 static cl::opt<bool> EnableStPairSuppress("aarch64-enable-stp-suppress", 80 cl::desc("Suppress STP for AArch64"), 81 cl::init(true), cl::Hidden); 82 83 static cl::opt<bool> EnableAdvSIMDScalar( 84 "aarch64-enable-simd-scalar", 85 cl::desc("Enable use of AdvSIMD scalar integer instructions"), 86 cl::init(false), cl::Hidden); 87 88 static cl::opt<bool> 89 EnablePromoteConstant("aarch64-enable-promote-const", 90 cl::desc("Enable the promote constant pass"), 91 cl::init(true), cl::Hidden); 92 93 static cl::opt<bool> EnableCollectLOH( 94 "aarch64-enable-collect-loh", 95 cl::desc("Enable the pass that emits the linker optimization hints (LOH)"), 96 cl::init(true), cl::Hidden); 97 98 static cl::opt<bool> 99 EnableDeadRegisterElimination("aarch64-enable-dead-defs", cl::Hidden, 100 cl::desc("Enable the pass that removes dead" 101 " definitons and replaces stores to" 102 " them with stores to the zero" 103 " register"), 104 cl::init(true)); 105 106 static cl::opt<bool> EnableRedundantCopyElimination( 107 "aarch64-enable-copyelim", 108 cl::desc("Enable the redundant copy elimination pass"), cl::init(true), 109 cl::Hidden); 110 111 static cl::opt<bool> EnableLoadStoreOpt("aarch64-enable-ldst-opt", 112 cl::desc("Enable the load/store pair" 113 " optimization pass"), 114 cl::init(true), cl::Hidden); 115 116 static cl::opt<bool> EnableAtomicTidy( 117 "aarch64-enable-atomic-cfg-tidy", cl::Hidden, 118 cl::desc("Run SimplifyCFG after expanding atomic operations" 119 " to make use of cmpxchg flow-based information"), 120 cl::init(true)); 121 122 static cl::opt<bool> 123 EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, 124 cl::desc("Run early if-conversion"), 125 cl::init(true)); 126 127 static cl::opt<bool> 128 EnableCondOpt("aarch64-enable-condopt", 129 cl::desc("Enable the condition optimizer pass"), 130 cl::init(true), cl::Hidden); 131 132 static cl::opt<bool> 133 EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden, 134 cl::desc("Enable optimizations on complex GEPs"), 135 cl::init(false)); 136 137 static cl::opt<bool> 138 EnableSelectOpt("aarch64-select-opt", cl::Hidden, 139 cl::desc("Enable select to branch optimizations"), 140 cl::init(true)); 141 142 static cl::opt<bool> 143 BranchRelaxation("aarch64-enable-branch-relax", cl::Hidden, cl::init(true), 144 cl::desc("Relax out of range conditional branches")); 145 146 static cl::opt<bool> EnableCompressJumpTables( 147 "aarch64-enable-compress-jump-tables", cl::Hidden, cl::init(true), 148 cl::desc("Use smallest entry possible for jump tables")); 149 150 // FIXME: Unify control over GlobalMerge. 151 static cl::opt<cl::boolOrDefault> 152 EnableGlobalMerge("aarch64-enable-global-merge", cl::Hidden, 153 cl::desc("Enable the global merge pass")); 154 155 static cl::opt<bool> 156 EnableLoopDataPrefetch("aarch64-enable-loop-data-prefetch", cl::Hidden, 157 cl::desc("Enable the loop data prefetch pass"), 158 cl::init(true)); 159 160 static cl::opt<int> EnableGlobalISelAtO( 161 "aarch64-enable-global-isel-at-O", cl::Hidden, 162 cl::desc("Enable GlobalISel at or below an opt level (-1 to disable)"), 163 cl::init(0)); 164 165 static cl::opt<bool> 166 EnableSVEIntrinsicOpts("aarch64-enable-sve-intrinsic-opts", cl::Hidden, 167 cl::desc("Enable SVE intrinsic opts"), 168 cl::init(true)); 169 170 static cl::opt<bool> EnableFalkorHWPFFix("aarch64-enable-falkor-hwpf-fix", 171 cl::init(true), cl::Hidden); 172 173 static cl::opt<bool> 174 EnableBranchTargets("aarch64-enable-branch-targets", cl::Hidden, 175 cl::desc("Enable the AArch64 branch target pass"), 176 cl::init(true)); 177 178 static cl::opt<unsigned> SVEVectorBitsMaxOpt( 179 "aarch64-sve-vector-bits-max", 180 cl::desc("Assume SVE vector registers are at most this big, " 181 "with zero meaning no maximum size is assumed."), 182 cl::init(0), cl::Hidden); 183 184 static cl::opt<unsigned> SVEVectorBitsMinOpt( 185 "aarch64-sve-vector-bits-min", 186 cl::desc("Assume SVE vector registers are at least this big, " 187 "with zero meaning no minimum size is assumed."), 188 cl::init(0), cl::Hidden); 189 190 extern cl::opt<bool> EnableHomogeneousPrologEpilog; 191 192 static cl::opt<bool> EnableGISelLoadStoreOptPreLegal( 193 "aarch64-enable-gisel-ldst-prelegal", 194 cl::desc("Enable GlobalISel's pre-legalizer load/store optimization pass"), 195 cl::init(true), cl::Hidden); 196 197 static cl::opt<bool> EnableGISelLoadStoreOptPostLegal( 198 "aarch64-enable-gisel-ldst-postlegal", 199 cl::desc("Enable GlobalISel's post-legalizer load/store optimization pass"), 200 cl::init(false), cl::Hidden); 201 202 static cl::opt<bool> 203 EnableSinkFold("aarch64-enable-sink-fold", 204 cl::desc("Enable sinking and folding of instruction copies"), 205 cl::init(true), cl::Hidden); 206 207 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Target() { 208 // Register the target. 209 RegisterTargetMachine<AArch64leTargetMachine> X(getTheAArch64leTarget()); 210 RegisterTargetMachine<AArch64beTargetMachine> Y(getTheAArch64beTarget()); 211 RegisterTargetMachine<AArch64leTargetMachine> Z(getTheARM64Target()); 212 RegisterTargetMachine<AArch64leTargetMachine> W(getTheARM64_32Target()); 213 RegisterTargetMachine<AArch64leTargetMachine> V(getTheAArch64_32Target()); 214 auto PR = PassRegistry::getPassRegistry(); 215 initializeGlobalISel(*PR); 216 initializeAArch64A53Fix835769Pass(*PR); 217 initializeAArch64A57FPLoadBalancingPass(*PR); 218 initializeAArch64AdvSIMDScalarPass(*PR); 219 initializeAArch64BranchTargetsPass(*PR); 220 initializeAArch64CollectLOHPass(*PR); 221 initializeAArch64CompressJumpTablesPass(*PR); 222 initializeAArch64ConditionalComparesPass(*PR); 223 initializeAArch64ConditionOptimizerPass(*PR); 224 initializeAArch64DeadRegisterDefinitionsPass(*PR); 225 initializeAArch64ExpandPseudoPass(*PR); 226 initializeAArch64LoadStoreOptPass(*PR); 227 initializeAArch64LoopIdiomTransformLegacyPassPass(*PR); 228 initializeAArch64MIPeepholeOptPass(*PR); 229 initializeAArch64SIMDInstrOptPass(*PR); 230 initializeAArch64O0PreLegalizerCombinerPass(*PR); 231 initializeAArch64PreLegalizerCombinerPass(*PR); 232 initializeAArch64PointerAuthPass(*PR); 233 initializeAArch64PostLegalizerCombinerPass(*PR); 234 initializeAArch64PostLegalizerLoweringPass(*PR); 235 initializeAArch64PostSelectOptimizePass(*PR); 236 initializeAArch64PromoteConstantPass(*PR); 237 initializeAArch64RedundantCopyEliminationPass(*PR); 238 initializeAArch64StorePairSuppressPass(*PR); 239 initializeFalkorHWPFFixPass(*PR); 240 initializeFalkorMarkStridedAccessesLegacyPass(*PR); 241 initializeLDTLSCleanupPass(*PR); 242 initializeKCFIPass(*PR); 243 initializeSMEABIPass(*PR); 244 initializeSVEIntrinsicOptsPass(*PR); 245 initializeAArch64SpeculationHardeningPass(*PR); 246 initializeAArch64SLSHardeningPass(*PR); 247 initializeAArch64StackTaggingPass(*PR); 248 initializeAArch64StackTaggingPreRAPass(*PR); 249 initializeAArch64LowerHomogeneousPrologEpilogPass(*PR); 250 initializeAArch64DAGToDAGISelPass(*PR); 251 initializeAArch64GlobalsTaggingPass(*PR); 252 } 253 254 //===----------------------------------------------------------------------===// 255 // AArch64 Lowering public interface. 256 //===----------------------------------------------------------------------===// 257 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 258 if (TT.isOSBinFormatMachO()) 259 return std::make_unique<AArch64_MachoTargetObjectFile>(); 260 if (TT.isOSBinFormatCOFF()) 261 return std::make_unique<AArch64_COFFTargetObjectFile>(); 262 263 return std::make_unique<AArch64_ELFTargetObjectFile>(); 264 } 265 266 // Helper function to build a DataLayout string 267 static std::string computeDataLayout(const Triple &TT, 268 const MCTargetOptions &Options, 269 bool LittleEndian) { 270 if (TT.isOSBinFormatMachO()) { 271 if (TT.getArch() == Triple::aarch64_32) 272 return "e-m:o-p:32:32-i64:64-i128:128-n32:64-S128"; 273 return "e-m:o-i64:64-i128:128-n32:64-S128"; 274 } 275 if (TT.isOSBinFormatCOFF()) 276 return "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"; 277 std::string Endian = LittleEndian ? "e" : "E"; 278 std::string Ptr32 = TT.getEnvironment() == Triple::GNUILP32 ? "-p:32:32" : ""; 279 return Endian + "-m:e" + Ptr32 + 280 "-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"; 281 } 282 283 static StringRef computeDefaultCPU(const Triple &TT, StringRef CPU) { 284 if (CPU.empty() && TT.isArm64e()) 285 return "apple-a12"; 286 return CPU; 287 } 288 289 static Reloc::Model getEffectiveRelocModel(const Triple &TT, 290 std::optional<Reloc::Model> RM) { 291 // AArch64 Darwin and Windows are always PIC. 292 if (TT.isOSDarwin() || TT.isOSWindows()) 293 return Reloc::PIC_; 294 // On ELF platforms the default static relocation model has a smart enough 295 // linker to cope with referencing external symbols defined in a shared 296 // library. Hence DynamicNoPIC doesn't need to be promoted to PIC. 297 if (!RM || *RM == Reloc::DynamicNoPIC) 298 return Reloc::Static; 299 return *RM; 300 } 301 302 static CodeModel::Model 303 getEffectiveAArch64CodeModel(const Triple &TT, 304 std::optional<CodeModel::Model> CM, bool JIT) { 305 if (CM) { 306 if (*CM != CodeModel::Small && *CM != CodeModel::Tiny && 307 *CM != CodeModel::Large) { 308 report_fatal_error( 309 "Only small, tiny and large code models are allowed on AArch64"); 310 } else if (*CM == CodeModel::Tiny && !TT.isOSBinFormatELF()) 311 report_fatal_error("tiny code model is only supported on ELF"); 312 return *CM; 313 } 314 // The default MCJIT memory managers make no guarantees about where they can 315 // find an executable page; JITed code needs to be able to refer to globals 316 // no matter how far away they are. 317 // We should set the CodeModel::Small for Windows ARM64 in JIT mode, 318 // since with large code model LLVM generating 4 MOV instructions, and 319 // Windows doesn't support relocating these long branch (4 MOVs). 320 if (JIT && !TT.isOSWindows()) 321 return CodeModel::Large; 322 return CodeModel::Small; 323 } 324 325 /// Create an AArch64 architecture model. 326 /// 327 AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT, 328 StringRef CPU, StringRef FS, 329 const TargetOptions &Options, 330 std::optional<Reloc::Model> RM, 331 std::optional<CodeModel::Model> CM, 332 CodeGenOptLevel OL, bool JIT, 333 bool LittleEndian) 334 : LLVMTargetMachine(T, 335 computeDataLayout(TT, Options.MCOptions, LittleEndian), 336 TT, computeDefaultCPU(TT, CPU), FS, Options, 337 getEffectiveRelocModel(TT, RM), 338 getEffectiveAArch64CodeModel(TT, CM, JIT), OL), 339 TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian) { 340 initAsmInfo(); 341 342 if (TT.isOSBinFormatMachO()) { 343 this->Options.TrapUnreachable = true; 344 this->Options.NoTrapAfterNoreturn = true; 345 } 346 347 if (getMCAsmInfo()->usesWindowsCFI()) { 348 // Unwinding can get confused if the last instruction in an 349 // exception-handling region (function, funclet, try block, etc.) 350 // is a call. 351 // 352 // FIXME: We could elide the trap if the next instruction would be in 353 // the same region anyway. 354 this->Options.TrapUnreachable = true; 355 } 356 357 if (this->Options.TLSSize == 0) // default 358 this->Options.TLSSize = 24; 359 if ((getCodeModel() == CodeModel::Small || 360 getCodeModel() == CodeModel::Kernel) && 361 this->Options.TLSSize > 32) 362 // for the small (and kernel) code model, the maximum TLS size is 4GiB 363 this->Options.TLSSize = 32; 364 else if (getCodeModel() == CodeModel::Tiny && this->Options.TLSSize > 24) 365 // for the tiny code model, the maximum TLS size is 1MiB (< 16MiB) 366 this->Options.TLSSize = 24; 367 368 // Enable GlobalISel at or below EnableGlobalISelAt0, unless this is 369 // MachO/CodeModel::Large, which GlobalISel does not support. 370 if (static_cast<int>(getOptLevel()) <= EnableGlobalISelAtO && 371 TT.getArch() != Triple::aarch64_32 && 372 TT.getEnvironment() != Triple::GNUILP32 && 373 !(getCodeModel() == CodeModel::Large && TT.isOSBinFormatMachO())) { 374 setGlobalISel(true); 375 setGlobalISelAbort(GlobalISelAbortMode::Disable); 376 } 377 378 // AArch64 supports the MachineOutliner. 379 setMachineOutliner(true); 380 381 // AArch64 supports default outlining behaviour. 382 setSupportsDefaultOutlining(true); 383 384 // AArch64 supports the debug entry values. 385 setSupportsDebugEntryValues(true); 386 387 // AArch64 supports fixing up the DWARF unwind information. 388 if (!getMCAsmInfo()->usesWindowsCFI()) 389 setCFIFixup(true); 390 } 391 392 AArch64TargetMachine::~AArch64TargetMachine() = default; 393 394 const AArch64Subtarget * 395 AArch64TargetMachine::getSubtargetImpl(const Function &F) const { 396 Attribute CPUAttr = F.getFnAttribute("target-cpu"); 397 Attribute TuneAttr = F.getFnAttribute("tune-cpu"); 398 Attribute FSAttr = F.getFnAttribute("target-features"); 399 400 StringRef CPU = CPUAttr.isValid() ? CPUAttr.getValueAsString() : TargetCPU; 401 StringRef TuneCPU = TuneAttr.isValid() ? TuneAttr.getValueAsString() : CPU; 402 StringRef FS = FSAttr.isValid() ? FSAttr.getValueAsString() : TargetFS; 403 bool HasMinSize = F.hasMinSize(); 404 405 bool StreamingSVEMode = F.hasFnAttribute("aarch64_pstate_sm_enabled") || 406 F.hasFnAttribute("aarch64_pstate_sm_body"); 407 bool StreamingCompatibleSVEMode = 408 F.hasFnAttribute("aarch64_pstate_sm_compatible"); 409 410 unsigned MinSVEVectorSize = 0; 411 unsigned MaxSVEVectorSize = 0; 412 if (F.hasFnAttribute(Attribute::VScaleRange)) { 413 ConstantRange CR = getVScaleRange(&F, 64); 414 MinSVEVectorSize = CR.getUnsignedMin().getZExtValue() * 128; 415 MaxSVEVectorSize = CR.getUnsignedMax().getZExtValue() * 128; 416 } else { 417 MinSVEVectorSize = SVEVectorBitsMinOpt; 418 MaxSVEVectorSize = SVEVectorBitsMaxOpt; 419 } 420 421 assert(MinSVEVectorSize % 128 == 0 && 422 "SVE requires vector length in multiples of 128!"); 423 assert(MaxSVEVectorSize % 128 == 0 && 424 "SVE requires vector length in multiples of 128!"); 425 assert((MaxSVEVectorSize >= MinSVEVectorSize || MaxSVEVectorSize == 0) && 426 "Minimum SVE vector size should not be larger than its maximum!"); 427 428 // Sanitize user input in case of no asserts 429 if (MaxSVEVectorSize != 0) { 430 MinSVEVectorSize = std::min(MinSVEVectorSize, MaxSVEVectorSize); 431 MaxSVEVectorSize = std::max(MinSVEVectorSize, MaxSVEVectorSize); 432 } 433 434 SmallString<512> Key; 435 raw_svector_ostream(Key) << "SVEMin" << MinSVEVectorSize << "SVEMax" 436 << MaxSVEVectorSize 437 << "StreamingSVEMode=" << StreamingSVEMode 438 << "StreamingCompatibleSVEMode=" 439 << StreamingCompatibleSVEMode << CPU << TuneCPU << FS 440 << "HasMinSize=" << HasMinSize; 441 442 auto &I = SubtargetMap[Key]; 443 if (!I) { 444 // This needs to be done before we create a new subtarget since any 445 // creation will depend on the TM and the code generation flags on the 446 // function that reside in TargetOptions. 447 resetTargetOptions(F); 448 I = std::make_unique<AArch64Subtarget>( 449 TargetTriple, CPU, TuneCPU, FS, *this, isLittle, MinSVEVectorSize, 450 MaxSVEVectorSize, StreamingSVEMode, StreamingCompatibleSVEMode, 451 HasMinSize); 452 } 453 454 assert((!StreamingSVEMode || I->hasSME()) && 455 "Expected SME to be available"); 456 457 return I.get(); 458 } 459 460 void AArch64leTargetMachine::anchor() { } 461 462 AArch64leTargetMachine::AArch64leTargetMachine( 463 const Target &T, const Triple &TT, StringRef CPU, StringRef FS, 464 const TargetOptions &Options, std::optional<Reloc::Model> RM, 465 std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) 466 : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, JIT, true) {} 467 468 void AArch64beTargetMachine::anchor() { } 469 470 AArch64beTargetMachine::AArch64beTargetMachine( 471 const Target &T, const Triple &TT, StringRef CPU, StringRef FS, 472 const TargetOptions &Options, std::optional<Reloc::Model> RM, 473 std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) 474 : AArch64TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, JIT, false) {} 475 476 namespace { 477 478 /// AArch64 Code Generator Pass Configuration Options. 479 class AArch64PassConfig : public TargetPassConfig { 480 public: 481 AArch64PassConfig(AArch64TargetMachine &TM, PassManagerBase &PM) 482 : TargetPassConfig(TM, PM) { 483 if (TM.getOptLevel() != CodeGenOptLevel::None) 484 substitutePass(&PostRASchedulerID, &PostMachineSchedulerID); 485 setEnableSinkAndFold(EnableSinkFold); 486 } 487 488 AArch64TargetMachine &getAArch64TargetMachine() const { 489 return getTM<AArch64TargetMachine>(); 490 } 491 492 ScheduleDAGInstrs * 493 createMachineScheduler(MachineSchedContext *C) const override { 494 const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>(); 495 ScheduleDAGMILive *DAG = createGenericSchedLive(C); 496 DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); 497 DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); 498 if (ST.hasFusion()) 499 DAG->addMutation(createAArch64MacroFusionDAGMutation()); 500 return DAG; 501 } 502 503 ScheduleDAGInstrs * 504 createPostMachineScheduler(MachineSchedContext *C) const override { 505 const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>(); 506 ScheduleDAGMI *DAG = 507 new ScheduleDAGMI(C, std::make_unique<AArch64PostRASchedStrategy>(C), 508 /* RemoveKillFlags=*/true); 509 if (ST.hasFusion()) { 510 // Run the Macro Fusion after RA again since literals are expanded from 511 // pseudos then (v. addPreSched2()). 512 DAG->addMutation(createAArch64MacroFusionDAGMutation()); 513 return DAG; 514 } 515 516 return DAG; 517 } 518 519 void addIRPasses() override; 520 bool addPreISel() override; 521 void addCodeGenPrepare() override; 522 bool addInstSelector() override; 523 bool addIRTranslator() override; 524 void addPreLegalizeMachineIR() override; 525 bool addLegalizeMachineIR() override; 526 void addPreRegBankSelect() override; 527 bool addRegBankSelect() override; 528 bool addGlobalInstructionSelect() override; 529 void addMachineSSAOptimization() override; 530 bool addILPOpts() override; 531 void addPreRegAlloc() override; 532 void addPostRegAlloc() override; 533 void addPreSched2() override; 534 void addPreEmitPass() override; 535 void addPostBBSections() override; 536 void addPreEmitPass2() override; 537 538 std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 539 }; 540 541 } // end anonymous namespace 542 543 void AArch64TargetMachine::registerPassBuilderCallbacks( 544 PassBuilder &PB, bool PopulateClassToPassNames) { 545 PB.registerLateLoopOptimizationsEPCallback( 546 [=](LoopPassManager &LPM, OptimizationLevel Level) { 547 LPM.addPass(AArch64LoopIdiomTransformPass()); 548 }); 549 } 550 551 TargetTransformInfo 552 AArch64TargetMachine::getTargetTransformInfo(const Function &F) const { 553 return TargetTransformInfo(AArch64TTIImpl(this, F)); 554 } 555 556 TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) { 557 return new AArch64PassConfig(*this, PM); 558 } 559 560 std::unique_ptr<CSEConfigBase> AArch64PassConfig::getCSEConfig() const { 561 return getStandardCSEConfigForOpt(TM->getOptLevel()); 562 } 563 564 void AArch64PassConfig::addIRPasses() { 565 // Always expand atomic operations, we don't deal with atomicrmw or cmpxchg 566 // ourselves. 567 addPass(createAtomicExpandPass()); 568 569 // Expand any SVE vector library calls that we can't code generate directly. 570 if (EnableSVEIntrinsicOpts && 571 TM->getOptLevel() == CodeGenOptLevel::Aggressive) 572 addPass(createSVEIntrinsicOptsPass()); 573 574 // Cmpxchg instructions are often used with a subsequent comparison to 575 // determine whether it succeeded. We can exploit existing control-flow in 576 // ldrex/strex loops to simplify this, but it needs tidying up. 577 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableAtomicTidy) 578 addPass(createCFGSimplificationPass(SimplifyCFGOptions() 579 .forwardSwitchCondToPhi(true) 580 .convertSwitchRangeToICmp(true) 581 .convertSwitchToLookupTable(true) 582 .needCanonicalLoops(false) 583 .hoistCommonInsts(true) 584 .sinkCommonInsts(true))); 585 586 // Run LoopDataPrefetch 587 // 588 // Run this before LSR to remove the multiplies involved in computing the 589 // pointer values N iterations ahead. 590 if (TM->getOptLevel() != CodeGenOptLevel::None) { 591 if (EnableLoopDataPrefetch) 592 addPass(createLoopDataPrefetchPass()); 593 if (EnableFalkorHWPFFix) 594 addPass(createFalkorMarkStridedAccessesPass()); 595 } 596 597 if (TM->getOptLevel() == CodeGenOptLevel::Aggressive && EnableGEPOpt) { 598 // Call SeparateConstOffsetFromGEP pass to extract constants within indices 599 // and lower a GEP with multiple indices to either arithmetic operations or 600 // multiple GEPs with single index. 601 addPass(createSeparateConstOffsetFromGEPPass(true)); 602 // Call EarlyCSE pass to find and remove subexpressions in the lowered 603 // result. 604 addPass(createEarlyCSEPass()); 605 // Do loop invariant code motion in case part of the lowered result is 606 // invariant. 607 addPass(createLICMPass()); 608 } 609 610 TargetPassConfig::addIRPasses(); 611 612 if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSelectOpt) 613 addPass(createSelectOptimizePass()); 614 615 addPass(createAArch64GlobalsTaggingPass()); 616 addPass(createAArch64StackTaggingPass( 617 /*IsOptNone=*/TM->getOptLevel() == CodeGenOptLevel::None)); 618 619 // Match complex arithmetic patterns 620 if (TM->getOptLevel() >= CodeGenOptLevel::Default) 621 addPass(createComplexDeinterleavingPass(TM)); 622 623 // Match interleaved memory accesses to ldN/stN intrinsics. 624 if (TM->getOptLevel() != CodeGenOptLevel::None) { 625 addPass(createInterleavedLoadCombinePass()); 626 addPass(createInterleavedAccessPass()); 627 } 628 629 // Expand any functions marked with SME attributes which require special 630 // changes for the calling convention or that require the lazy-saving 631 // mechanism specified in the SME ABI. 632 addPass(createSMEABIPass()); 633 634 // Add Control Flow Guard checks. 635 if (TM->getTargetTriple().isOSWindows()) 636 addPass(createCFGuardCheckPass()); 637 638 if (TM->Options.JMCInstrument) 639 addPass(createJMCInstrumenterPass()); 640 } 641 642 // Pass Pipeline Configuration 643 bool AArch64PassConfig::addPreISel() { 644 // Run promote constant before global merge, so that the promoted constants 645 // get a chance to be merged 646 if (TM->getOptLevel() != CodeGenOptLevel::None && EnablePromoteConstant) 647 addPass(createAArch64PromoteConstantPass()); 648 // FIXME: On AArch64, this depends on the type. 649 // Basically, the addressable offsets are up to 4095 * Ty.getSizeInBytes(). 650 // and the offset has to be a multiple of the related size in bytes. 651 if ((TM->getOptLevel() != CodeGenOptLevel::None && 652 EnableGlobalMerge == cl::BOU_UNSET) || 653 EnableGlobalMerge == cl::BOU_TRUE) { 654 bool OnlyOptimizeForSize = 655 (TM->getOptLevel() < CodeGenOptLevel::Aggressive) && 656 (EnableGlobalMerge == cl::BOU_UNSET); 657 658 // Merging of extern globals is enabled by default on non-Mach-O as we 659 // expect it to be generally either beneficial or harmless. On Mach-O it 660 // is disabled as we emit the .subsections_via_symbols directive which 661 // means that merging extern globals is not safe. 662 bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO(); 663 664 // FIXME: extern global merging is only enabled when we optimise for size 665 // because there are some regressions with it also enabled for performance. 666 if (!OnlyOptimizeForSize) 667 MergeExternalByDefault = false; 668 669 addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize, 670 MergeExternalByDefault)); 671 } 672 673 return false; 674 } 675 676 void AArch64PassConfig::addCodeGenPrepare() { 677 if (getOptLevel() != CodeGenOptLevel::None) 678 addPass(createTypePromotionLegacyPass()); 679 TargetPassConfig::addCodeGenPrepare(); 680 } 681 682 bool AArch64PassConfig::addInstSelector() { 683 addPass(createAArch64ISelDag(getAArch64TargetMachine(), getOptLevel())); 684 685 // For ELF, cleanup any local-dynamic TLS accesses (i.e. combine as many 686 // references to _TLS_MODULE_BASE_ as possible. 687 if (TM->getTargetTriple().isOSBinFormatELF() && 688 getOptLevel() != CodeGenOptLevel::None) 689 addPass(createAArch64CleanupLocalDynamicTLSPass()); 690 691 return false; 692 } 693 694 bool AArch64PassConfig::addIRTranslator() { 695 addPass(new IRTranslator(getOptLevel())); 696 return false; 697 } 698 699 void AArch64PassConfig::addPreLegalizeMachineIR() { 700 if (getOptLevel() == CodeGenOptLevel::None) { 701 addPass(createAArch64O0PreLegalizerCombiner()); 702 addPass(new Localizer()); 703 } else { 704 addPass(createAArch64PreLegalizerCombiner()); 705 addPass(new Localizer()); 706 if (EnableGISelLoadStoreOptPreLegal) 707 addPass(new LoadStoreOpt()); 708 } 709 } 710 711 bool AArch64PassConfig::addLegalizeMachineIR() { 712 addPass(new Legalizer()); 713 return false; 714 } 715 716 void AArch64PassConfig::addPreRegBankSelect() { 717 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None; 718 if (!IsOptNone) { 719 addPass(createAArch64PostLegalizerCombiner(IsOptNone)); 720 if (EnableGISelLoadStoreOptPostLegal) 721 addPass(new LoadStoreOpt()); 722 } 723 addPass(createAArch64PostLegalizerLowering()); 724 } 725 726 bool AArch64PassConfig::addRegBankSelect() { 727 addPass(new RegBankSelect()); 728 return false; 729 } 730 731 bool AArch64PassConfig::addGlobalInstructionSelect() { 732 addPass(new InstructionSelect(getOptLevel())); 733 if (getOptLevel() != CodeGenOptLevel::None) 734 addPass(createAArch64PostSelectOptimize()); 735 return false; 736 } 737 738 void AArch64PassConfig::addMachineSSAOptimization() { 739 // Run default MachineSSAOptimization first. 740 TargetPassConfig::addMachineSSAOptimization(); 741 742 if (TM->getOptLevel() != CodeGenOptLevel::None) 743 addPass(createAArch64MIPeepholeOptPass()); 744 } 745 746 bool AArch64PassConfig::addILPOpts() { 747 if (EnableCondOpt) 748 addPass(createAArch64ConditionOptimizerPass()); 749 if (EnableCCMP) 750 addPass(createAArch64ConditionalCompares()); 751 if (EnableMCR) 752 addPass(&MachineCombinerID); 753 if (EnableCondBrTuning) 754 addPass(createAArch64CondBrTuning()); 755 if (EnableEarlyIfConversion) 756 addPass(&EarlyIfConverterID); 757 if (EnableStPairSuppress) 758 addPass(createAArch64StorePairSuppressPass()); 759 addPass(createAArch64SIMDInstrOptPass()); 760 if (TM->getOptLevel() != CodeGenOptLevel::None) 761 addPass(createAArch64StackTaggingPreRAPass()); 762 return true; 763 } 764 765 void AArch64PassConfig::addPreRegAlloc() { 766 // Change dead register definitions to refer to the zero register. 767 if (TM->getOptLevel() != CodeGenOptLevel::None && 768 EnableDeadRegisterElimination) 769 addPass(createAArch64DeadRegisterDefinitions()); 770 771 // Use AdvSIMD scalar instructions whenever profitable. 772 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableAdvSIMDScalar) { 773 addPass(createAArch64AdvSIMDScalar()); 774 // The AdvSIMD pass may produce copies that can be rewritten to 775 // be register coalescer friendly. 776 addPass(&PeepholeOptimizerID); 777 } 778 } 779 780 void AArch64PassConfig::addPostRegAlloc() { 781 // Remove redundant copy instructions. 782 if (TM->getOptLevel() != CodeGenOptLevel::None && 783 EnableRedundantCopyElimination) 784 addPass(createAArch64RedundantCopyEliminationPass()); 785 786 if (TM->getOptLevel() != CodeGenOptLevel::None && usingDefaultRegAlloc()) 787 // Improve performance for some FP/SIMD code for A57. 788 addPass(createAArch64A57FPLoadBalancing()); 789 } 790 791 void AArch64PassConfig::addPreSched2() { 792 // Lower homogeneous frame instructions 793 if (EnableHomogeneousPrologEpilog) 794 addPass(createAArch64LowerHomogeneousPrologEpilogPass()); 795 // Expand some pseudo instructions to allow proper scheduling. 796 addPass(createAArch64ExpandPseudoPass()); 797 // Use load/store pair instructions when possible. 798 if (TM->getOptLevel() != CodeGenOptLevel::None) { 799 if (EnableLoadStoreOpt) 800 addPass(createAArch64LoadStoreOptimizationPass()); 801 } 802 // Emit KCFI checks for indirect calls. 803 addPass(createKCFIPass()); 804 805 // The AArch64SpeculationHardeningPass destroys dominator tree and natural 806 // loop info, which is needed for the FalkorHWPFFixPass and also later on. 807 // Therefore, run the AArch64SpeculationHardeningPass before the 808 // FalkorHWPFFixPass to avoid recomputing dominator tree and natural loop 809 // info. 810 addPass(createAArch64SpeculationHardeningPass()); 811 812 addPass(createAArch64IndirectThunks()); 813 addPass(createAArch64SLSHardeningPass()); 814 815 if (TM->getOptLevel() != CodeGenOptLevel::None) { 816 if (EnableFalkorHWPFFix) 817 addPass(createFalkorHWPFFixPass()); 818 } 819 } 820 821 void AArch64PassConfig::addPreEmitPass() { 822 // Machine Block Placement might have created new opportunities when run 823 // at O3, where the Tail Duplication Threshold is set to 4 instructions. 824 // Run the load/store optimizer once more. 825 if (TM->getOptLevel() >= CodeGenOptLevel::Aggressive && EnableLoadStoreOpt) 826 addPass(createAArch64LoadStoreOptimizationPass()); 827 828 if (TM->getOptLevel() >= CodeGenOptLevel::Aggressive && 829 EnableAArch64CopyPropagation) 830 addPass(createMachineCopyPropagationPass(true)); 831 832 addPass(createAArch64A53Fix835769()); 833 834 if (TM->getTargetTriple().isOSWindows()) { 835 // Identify valid longjmp targets for Windows Control Flow Guard. 836 addPass(createCFGuardLongjmpPass()); 837 // Identify valid eh continuation targets for Windows EHCont Guard. 838 addPass(createEHContGuardCatchretPass()); 839 } 840 841 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableCollectLOH && 842 TM->getTargetTriple().isOSBinFormatMachO()) 843 addPass(createAArch64CollectLOHPass()); 844 } 845 846 void AArch64PassConfig::addPostBBSections() { 847 addPass(createAArch64PointerAuthPass()); 848 if (EnableBranchTargets) 849 addPass(createAArch64BranchTargetsPass()); 850 // Relax conditional branch instructions if they're otherwise out of 851 // range of their destination. 852 if (BranchRelaxation) 853 addPass(&BranchRelaxationPassID); 854 855 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableCompressJumpTables) 856 addPass(createAArch64CompressJumpTablesPass()); 857 } 858 859 void AArch64PassConfig::addPreEmitPass2() { 860 // SVE bundles move prefixes with destructive operations. BLR_RVMARKER pseudo 861 // instructions are lowered to bundles as well. 862 addPass(createUnpackMachineBundles(nullptr)); 863 } 864 865 MachineFunctionInfo *AArch64TargetMachine::createMachineFunctionInfo( 866 BumpPtrAllocator &Allocator, const Function &F, 867 const TargetSubtargetInfo *STI) const { 868 return AArch64FunctionInfo::create<AArch64FunctionInfo>( 869 Allocator, F, static_cast<const AArch64Subtarget *>(STI)); 870 } 871 872 yaml::MachineFunctionInfo * 873 AArch64TargetMachine::createDefaultFuncInfoYAML() const { 874 return new yaml::AArch64FunctionInfo(); 875 } 876 877 yaml::MachineFunctionInfo * 878 AArch64TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const { 879 const auto *MFI = MF.getInfo<AArch64FunctionInfo>(); 880 return new yaml::AArch64FunctionInfo(*MFI); 881 } 882 883 bool AArch64TargetMachine::parseMachineFunctionInfo( 884 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS, 885 SMDiagnostic &Error, SMRange &SourceRange) const { 886 const auto &YamlMFI = static_cast<const yaml::AArch64FunctionInfo &>(MFI); 887 MachineFunction &MF = PFS.MF; 888 MF.getInfo<AArch64FunctionInfo>()->initializeBaseYamlFields(YamlMFI); 889 return false; 890 } 891