1 //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the X86 specific subclass of TargetMachine. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "X86TargetMachine.h" 14 #include "MCTargetDesc/X86MCTargetDesc.h" 15 #include "TargetInfo/X86TargetInfo.h" 16 #include "X86.h" 17 #include "X86MachineFunctionInfo.h" 18 #include "X86MacroFusion.h" 19 #include "X86Subtarget.h" 20 #include "X86TargetObjectFile.h" 21 #include "X86TargetTransformInfo.h" 22 #include "llvm/ADT/STLExtras.h" 23 #include "llvm/ADT/SmallString.h" 24 #include "llvm/ADT/StringRef.h" 25 #include "llvm/Analysis/TargetTransformInfo.h" 26 #include "llvm/CodeGen/ExecutionDomainFix.h" 27 #include "llvm/CodeGen/GlobalISel/CSEInfo.h" 28 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 29 #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 30 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 31 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 32 #include "llvm/CodeGen/GlobalISel/Legalizer.h" 33 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 34 #include "llvm/CodeGen/MachineScheduler.h" 35 #include "llvm/CodeGen/Passes.h" 36 #include "llvm/CodeGen/RegAllocRegistry.h" 37 #include "llvm/CodeGen/TargetPassConfig.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/MC/MCAsmInfo.h" 42 #include "llvm/MC/TargetRegistry.h" 43 #include "llvm/Pass.h" 44 #include "llvm/Support/CodeGen.h" 45 #include "llvm/Support/CommandLine.h" 46 #include "llvm/Support/ErrorHandling.h" 47 #include "llvm/Target/TargetLoweringObjectFile.h" 48 #include "llvm/Target/TargetOptions.h" 49 #include "llvm/TargetParser/Triple.h" 50 #include "llvm/Transforms/CFGuard.h" 51 #include <memory> 52 #include <optional> 53 #include <string> 54 55 using namespace llvm; 56 57 static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner", 58 cl::desc("Enable the machine combiner pass"), 59 cl::init(true), cl::Hidden); 60 61 static cl::opt<bool> 62 EnableTileRAPass("x86-tile-ra", 63 cl::desc("Enable the tile register allocation pass"), 64 cl::init(true), cl::Hidden); 65 66 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() { 67 // Register the target. 68 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target()); 69 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target()); 70 71 PassRegistry &PR = *PassRegistry::getPassRegistry(); 72 initializeX86LowerAMXIntrinsicsLegacyPassPass(PR); 73 initializeX86LowerAMXTypeLegacyPassPass(PR); 74 initializeX86PreTileConfigPass(PR); 75 initializeGlobalISel(PR); 76 initializeWinEHStatePassPass(PR); 77 initializeFixupBWInstPassPass(PR); 78 initializeCompressEVEXPassPass(PR); 79 initializeFixupLEAPassPass(PR); 80 initializeFPSPass(PR); 81 initializeX86FixupSetCCPassPass(PR); 82 initializeX86CallFrameOptimizationPass(PR); 83 initializeX86CmovConverterPassPass(PR); 84 initializeX86TileConfigPass(PR); 85 initializeX86FastPreTileConfigPass(PR); 86 initializeX86FastTileConfigPass(PR); 87 initializeKCFIPass(PR); 88 initializeX86LowerTileCopyPass(PR); 89 initializeX86ExpandPseudoPass(PR); 90 initializeX86ExecutionDomainFixPass(PR); 91 initializeX86DomainReassignmentPass(PR); 92 initializeX86AvoidSFBPassPass(PR); 93 initializeX86AvoidTrailingCallPassPass(PR); 94 initializeX86SpeculativeLoadHardeningPassPass(PR); 95 initializeX86SpeculativeExecutionSideEffectSuppressionPass(PR); 96 initializeX86FlagsCopyLoweringPassPass(PR); 97 initializeX86LoadValueInjectionLoadHardeningPassPass(PR); 98 initializeX86LoadValueInjectionRetHardeningPassPass(PR); 99 initializeX86OptimizeLEAPassPass(PR); 100 initializeX86PartialReductionPass(PR); 101 initializePseudoProbeInserterPass(PR); 102 initializeX86ReturnThunksPass(PR); 103 initializeX86DAGToDAGISelPass(PR); 104 initializeX86ArgumentStackSlotPassPass(PR); 105 } 106 107 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 108 if (TT.isOSBinFormatMachO()) { 109 if (TT.getArch() == Triple::x86_64) 110 return std::make_unique<X86_64MachoTargetObjectFile>(); 111 return std::make_unique<TargetLoweringObjectFileMachO>(); 112 } 113 114 if (TT.isOSBinFormatCOFF()) 115 return std::make_unique<TargetLoweringObjectFileCOFF>(); 116 117 if (TT.getArch() == Triple::x86_64) 118 return std::make_unique<X86_64ELFTargetObjectFile>(); 119 return std::make_unique<X86ELFTargetObjectFile>(); 120 } 121 122 static std::string computeDataLayout(const Triple &TT) { 123 // X86 is little endian 124 std::string Ret = "e"; 125 126 Ret += DataLayout::getManglingComponent(TT); 127 // X86 and x32 have 32 bit pointers. 128 if (!TT.isArch64Bit() || TT.isX32() || TT.isOSNaCl()) 129 Ret += "-p:32:32"; 130 131 // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers. 132 Ret += "-p270:32:32-p271:32:32-p272:64:64"; 133 134 // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. 135 // 128 bit integers are not specified in the 32-bit ABIs but are used 136 // internally for lowering f128, so we match the alignment to that. 137 if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl()) 138 Ret += "-i64:64-i128:128"; 139 else if (TT.isOSIAMCU()) 140 Ret += "-i64:32-f64:32"; 141 else 142 Ret += "-i128:128-f64:32:64"; 143 144 // Some ABIs align long double to 128 bits, others to 32. 145 if (TT.isOSNaCl() || TT.isOSIAMCU()) 146 ; // No f80 147 else if (TT.isArch64Bit() || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment()) 148 Ret += "-f80:128"; 149 else 150 Ret += "-f80:32"; 151 152 if (TT.isOSIAMCU()) 153 Ret += "-f128:32"; 154 155 // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. 156 if (TT.isArch64Bit()) 157 Ret += "-n8:16:32:64"; 158 else 159 Ret += "-n8:16:32"; 160 161 // The stack is aligned to 32 bits on some ABIs and 128 bits on others. 162 if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU()) 163 Ret += "-a:0:32-S32"; 164 else 165 Ret += "-S128"; 166 167 return Ret; 168 } 169 170 static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT, 171 std::optional<Reloc::Model> RM) { 172 bool is64Bit = TT.getArch() == Triple::x86_64; 173 if (!RM) { 174 // JIT codegen should use static relocations by default, since it's 175 // typically executed in process and not relocatable. 176 if (JIT) 177 return Reloc::Static; 178 179 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode. 180 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we 181 // use static relocation model by default. 182 if (TT.isOSDarwin()) { 183 if (is64Bit) 184 return Reloc::PIC_; 185 return Reloc::DynamicNoPIC; 186 } 187 if (TT.isOSWindows() && is64Bit) 188 return Reloc::PIC_; 189 return Reloc::Static; 190 } 191 192 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC 193 // is defined as a model for code which may be used in static or dynamic 194 // executables but not necessarily a shared library. On X86-32 we just 195 // compile in -static mode, in x86-64 we use PIC. 196 if (*RM == Reloc::DynamicNoPIC) { 197 if (is64Bit) 198 return Reloc::PIC_; 199 if (!TT.isOSDarwin()) 200 return Reloc::Static; 201 } 202 203 // If we are on Darwin, disallow static relocation model in X86-64 mode, since 204 // the Mach-O file format doesn't support it. 205 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit) 206 return Reloc::PIC_; 207 208 return *RM; 209 } 210 211 static CodeModel::Model 212 getEffectiveX86CodeModel(std::optional<CodeModel::Model> CM, bool JIT, 213 bool Is64Bit) { 214 if (CM) { 215 if (*CM == CodeModel::Tiny) 216 report_fatal_error("Target does not support the tiny CodeModel", false); 217 return *CM; 218 } 219 if (JIT) 220 return Is64Bit ? CodeModel::Large : CodeModel::Small; 221 return CodeModel::Small; 222 } 223 224 /// Create an X86 target. 225 /// 226 X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, 227 StringRef CPU, StringRef FS, 228 const TargetOptions &Options, 229 std::optional<Reloc::Model> RM, 230 std::optional<CodeModel::Model> CM, 231 CodeGenOptLevel OL, bool JIT) 232 : LLVMTargetMachine( 233 T, computeDataLayout(TT), TT, CPU, FS, Options, 234 getEffectiveRelocModel(TT, JIT, RM), 235 getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64), 236 OL), 237 TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) { 238 // On PS4/PS5, the "return address" of a 'noreturn' call must still be within 239 // the calling function, and TrapUnreachable is an easy way to get that. 240 if (TT.isPS() || TT.isOSBinFormatMachO()) { 241 this->Options.TrapUnreachable = true; 242 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO(); 243 } 244 245 setMachineOutliner(true); 246 247 // x86 supports the debug entry values. 248 setSupportsDebugEntryValues(true); 249 250 initAsmInfo(); 251 } 252 253 X86TargetMachine::~X86TargetMachine() = default; 254 255 const X86Subtarget * 256 X86TargetMachine::getSubtargetImpl(const Function &F) const { 257 Attribute CPUAttr = F.getFnAttribute("target-cpu"); 258 Attribute TuneAttr = F.getFnAttribute("tune-cpu"); 259 Attribute FSAttr = F.getFnAttribute("target-features"); 260 261 StringRef CPU = 262 CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU; 263 // "x86-64" is a default target setting for many front ends. In these cases, 264 // they actually request for "generic" tuning unless the "tune-cpu" was 265 // specified. 266 StringRef TuneCPU = TuneAttr.isValid() ? TuneAttr.getValueAsString() 267 : CPU == "x86-64" ? "generic" 268 : (StringRef)CPU; 269 StringRef FS = 270 FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS; 271 272 SmallString<512> Key; 273 // The additions here are ordered so that the definitely short strings are 274 // added first so we won't exceed the small size. We append the 275 // much longer FS string at the end so that we only heap allocate at most 276 // one time. 277 278 // Extract prefer-vector-width attribute. 279 unsigned PreferVectorWidthOverride = 0; 280 Attribute PreferVecWidthAttr = F.getFnAttribute("prefer-vector-width"); 281 if (PreferVecWidthAttr.isValid()) { 282 StringRef Val = PreferVecWidthAttr.getValueAsString(); 283 unsigned Width; 284 if (!Val.getAsInteger(0, Width)) { 285 Key += 'p'; 286 Key += Val; 287 PreferVectorWidthOverride = Width; 288 } 289 } 290 291 // Extract min-legal-vector-width attribute. 292 unsigned RequiredVectorWidth = UINT32_MAX; 293 Attribute MinLegalVecWidthAttr = F.getFnAttribute("min-legal-vector-width"); 294 if (MinLegalVecWidthAttr.isValid()) { 295 StringRef Val = MinLegalVecWidthAttr.getValueAsString(); 296 unsigned Width; 297 if (!Val.getAsInteger(0, Width)) { 298 Key += 'm'; 299 Key += Val; 300 RequiredVectorWidth = Width; 301 } 302 } 303 304 // Add CPU to the Key. 305 Key += CPU; 306 307 // Add tune CPU to the Key. 308 Key += TuneCPU; 309 310 // Keep track of the start of the feature portion of the string. 311 unsigned FSStart = Key.size(); 312 313 // FIXME: This is related to the code below to reset the target options, 314 // we need to know whether or not the soft float flag is set on the 315 // function before we can generate a subtarget. We also need to use 316 // it as a key for the subtarget since that can be the only difference 317 // between two functions. 318 bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsBool(); 319 // If the soft float attribute is set on the function turn on the soft float 320 // subtarget feature. 321 if (SoftFloat) 322 Key += FS.empty() ? "+soft-float" : "+soft-float,"; 323 324 Key += FS; 325 326 // We may have added +soft-float to the features so move the StringRef to 327 // point to the full string in the Key. 328 FS = Key.substr(FSStart); 329 330 auto &I = SubtargetMap[Key]; 331 if (!I) { 332 // This needs to be done before we create a new subtarget since any 333 // creation will depend on the TM and the code generation flags on the 334 // function that reside in TargetOptions. 335 resetTargetOptions(F); 336 I = std::make_unique<X86Subtarget>( 337 TargetTriple, CPU, TuneCPU, FS, *this, 338 MaybeAlign(F.getParent()->getOverrideStackAlignment()), 339 PreferVectorWidthOverride, RequiredVectorWidth); 340 } 341 return I.get(); 342 } 343 344 bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS, 345 unsigned DestAS) const { 346 assert(SrcAS != DestAS && "Expected different address spaces!"); 347 if (getPointerSize(SrcAS) != getPointerSize(DestAS)) 348 return false; 349 return SrcAS < 256 && DestAS < 256; 350 } 351 352 //===----------------------------------------------------------------------===// 353 // X86 TTI query. 354 //===----------------------------------------------------------------------===// 355 356 TargetTransformInfo 357 X86TargetMachine::getTargetTransformInfo(const Function &F) const { 358 return TargetTransformInfo(X86TTIImpl(this, F)); 359 } 360 361 //===----------------------------------------------------------------------===// 362 // Pass Pipeline Configuration 363 //===----------------------------------------------------------------------===// 364 365 namespace { 366 367 /// X86 Code Generator Pass Configuration Options. 368 class X86PassConfig : public TargetPassConfig { 369 public: 370 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM) 371 : TargetPassConfig(TM, PM) {} 372 373 X86TargetMachine &getX86TargetMachine() const { 374 return getTM<X86TargetMachine>(); 375 } 376 377 ScheduleDAGInstrs * 378 createMachineScheduler(MachineSchedContext *C) const override { 379 ScheduleDAGMILive *DAG = createGenericSchedLive(C); 380 DAG->addMutation(createX86MacroFusionDAGMutation()); 381 return DAG; 382 } 383 384 ScheduleDAGInstrs * 385 createPostMachineScheduler(MachineSchedContext *C) const override { 386 ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 387 DAG->addMutation(createX86MacroFusionDAGMutation()); 388 return DAG; 389 } 390 391 void addIRPasses() override; 392 bool addInstSelector() override; 393 bool addIRTranslator() override; 394 bool addLegalizeMachineIR() override; 395 bool addRegBankSelect() override; 396 bool addGlobalInstructionSelect() override; 397 bool addILPOpts() override; 398 bool addPreISel() override; 399 void addMachineSSAOptimization() override; 400 void addPreRegAlloc() override; 401 bool addPostFastRegAllocRewrite() override; 402 void addPostRegAlloc() override; 403 void addPreEmitPass() override; 404 void addPreEmitPass2() override; 405 void addPreSched2() override; 406 bool addRegAssignAndRewriteOptimized() override; 407 408 std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 409 }; 410 411 class X86ExecutionDomainFix : public ExecutionDomainFix { 412 public: 413 static char ID; 414 X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {} 415 StringRef getPassName() const override { 416 return "X86 Execution Dependency Fix"; 417 } 418 }; 419 char X86ExecutionDomainFix::ID; 420 421 } // end anonymous namespace 422 423 INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix", 424 "X86 Execution Domain Fix", false, false) 425 INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) 426 INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix", 427 "X86 Execution Domain Fix", false, false) 428 429 TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) { 430 return new X86PassConfig(*this, PM); 431 } 432 433 MachineFunctionInfo *X86TargetMachine::createMachineFunctionInfo( 434 BumpPtrAllocator &Allocator, const Function &F, 435 const TargetSubtargetInfo *STI) const { 436 return X86MachineFunctionInfo::create<X86MachineFunctionInfo>(Allocator, F, 437 STI); 438 } 439 440 void X86PassConfig::addIRPasses() { 441 addPass(createAtomicExpandPass()); 442 443 // We add both pass anyway and when these two passes run, we skip the pass 444 // based on the option level and option attribute. 445 addPass(createX86LowerAMXIntrinsicsPass()); 446 addPass(createX86LowerAMXTypePass()); 447 448 TargetPassConfig::addIRPasses(); 449 450 if (TM->getOptLevel() != CodeGenOptLevel::None) { 451 addPass(createInterleavedAccessPass()); 452 addPass(createX86PartialReductionPass()); 453 } 454 455 // Add passes that handle indirect branch removal and insertion of a retpoline 456 // thunk. These will be a no-op unless a function subtarget has the retpoline 457 // feature enabled. 458 addPass(createIndirectBrExpandPass()); 459 460 // Add Control Flow Guard checks. 461 const Triple &TT = TM->getTargetTriple(); 462 if (TT.isOSWindows()) { 463 if (TT.getArch() == Triple::x86_64) { 464 addPass(createCFGuardDispatchPass()); 465 } else { 466 addPass(createCFGuardCheckPass()); 467 } 468 } 469 470 if (TM->Options.JMCInstrument) 471 addPass(createJMCInstrumenterPass()); 472 } 473 474 bool X86PassConfig::addInstSelector() { 475 // Install an instruction selector. 476 addPass(createX86ISelDag(getX86TargetMachine(), getOptLevel())); 477 478 // For ELF, cleanup any local-dynamic TLS accesses. 479 if (TM->getTargetTriple().isOSBinFormatELF() && 480 getOptLevel() != CodeGenOptLevel::None) 481 addPass(createCleanupLocalDynamicTLSPass()); 482 483 addPass(createX86GlobalBaseRegPass()); 484 addPass(createX86ArgumentStackSlotPass()); 485 return false; 486 } 487 488 bool X86PassConfig::addIRTranslator() { 489 addPass(new IRTranslator(getOptLevel())); 490 return false; 491 } 492 493 bool X86PassConfig::addLegalizeMachineIR() { 494 addPass(new Legalizer()); 495 return false; 496 } 497 498 bool X86PassConfig::addRegBankSelect() { 499 addPass(new RegBankSelect()); 500 return false; 501 } 502 503 bool X86PassConfig::addGlobalInstructionSelect() { 504 addPass(new InstructionSelect(getOptLevel())); 505 return false; 506 } 507 508 bool X86PassConfig::addILPOpts() { 509 addPass(&EarlyIfConverterID); 510 if (EnableMachineCombinerPass) 511 addPass(&MachineCombinerID); 512 addPass(createX86CmovConverterPass()); 513 return true; 514 } 515 516 bool X86PassConfig::addPreISel() { 517 // Only add this pass for 32-bit x86 Windows. 518 const Triple &TT = TM->getTargetTriple(); 519 if (TT.isOSWindows() && TT.getArch() == Triple::x86) 520 addPass(createX86WinEHStatePass()); 521 return true; 522 } 523 524 void X86PassConfig::addPreRegAlloc() { 525 if (getOptLevel() != CodeGenOptLevel::None) { 526 addPass(&LiveRangeShrinkID); 527 addPass(createX86FixupSetCC()); 528 addPass(createX86OptimizeLEAs()); 529 addPass(createX86CallFrameOptimization()); 530 addPass(createX86AvoidStoreForwardingBlocks()); 531 } 532 533 addPass(createX86SpeculativeLoadHardeningPass()); 534 addPass(createX86FlagsCopyLoweringPass()); 535 addPass(createX86DynAllocaExpander()); 536 537 if (getOptLevel() != CodeGenOptLevel::None) 538 addPass(createX86PreTileConfigPass()); 539 else 540 addPass(createX86FastPreTileConfigPass()); 541 } 542 543 void X86PassConfig::addMachineSSAOptimization() { 544 addPass(createX86DomainReassignmentPass()); 545 TargetPassConfig::addMachineSSAOptimization(); 546 } 547 548 void X86PassConfig::addPostRegAlloc() { 549 addPass(createX86LowerTileCopyPass()); 550 addPass(createX86FloatingPointStackifierPass()); 551 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back 552 // to using the Speculative Execution Side Effect Suppression pass for 553 // mitigation. This is to prevent slow downs due to 554 // analyses needed by the LVIHardening pass when compiling at -O0. 555 if (getOptLevel() != CodeGenOptLevel::None) 556 addPass(createX86LoadValueInjectionLoadHardeningPass()); 557 } 558 559 void X86PassConfig::addPreSched2() { 560 addPass(createX86ExpandPseudoPass()); 561 addPass(createKCFIPass()); 562 } 563 564 void X86PassConfig::addPreEmitPass() { 565 if (getOptLevel() != CodeGenOptLevel::None) { 566 addPass(new X86ExecutionDomainFix()); 567 addPass(createBreakFalseDeps()); 568 } 569 570 addPass(createX86IndirectBranchTrackingPass()); 571 572 addPass(createX86IssueVZeroUpperPass()); 573 574 if (getOptLevel() != CodeGenOptLevel::None) { 575 addPass(createX86FixupBWInsts()); 576 addPass(createX86PadShortFunctions()); 577 addPass(createX86FixupLEAs()); 578 addPass(createX86FixupInstTuning()); 579 addPass(createX86FixupVectorConstants()); 580 } 581 addPass(createX86CompressEVEXPass()); 582 addPass(createX86DiscriminateMemOpsPass()); 583 addPass(createX86InsertPrefetchPass()); 584 addPass(createX86InsertX87waitPass()); 585 } 586 587 void X86PassConfig::addPreEmitPass2() { 588 const Triple &TT = TM->getTargetTriple(); 589 const MCAsmInfo *MAI = TM->getMCAsmInfo(); 590 591 // The X86 Speculative Execution Pass must run after all control 592 // flow graph modifying passes. As a result it was listed to run right before 593 // the X86 Retpoline Thunks pass. The reason it must run after control flow 594 // graph modifications is that the model of LFENCE in LLVM has to be updated 595 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the 596 // placement of this pass was hand checked to ensure that the subsequent 597 // passes don't move the code around the LFENCEs in a way that will hurt the 598 // correctness of this pass. This placement has been shown to work based on 599 // hand inspection of the codegen output. 600 addPass(createX86SpeculativeExecutionSideEffectSuppression()); 601 addPass(createX86IndirectThunksPass()); 602 addPass(createX86ReturnThunksPass()); 603 604 // Insert extra int3 instructions after trailing call instructions to avoid 605 // issues in the unwinder. 606 if (TT.isOSWindows() && TT.getArch() == Triple::x86_64) 607 addPass(createX86AvoidTrailingCallPass()); 608 609 // Verify basic block incoming and outgoing cfa offset and register values and 610 // correct CFA calculation rule where needed by inserting appropriate CFI 611 // instructions. 612 if (!TT.isOSDarwin() && 613 (!TT.isOSWindows() || 614 MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) 615 addPass(createCFIInstrInserter()); 616 617 if (TT.isOSWindows()) { 618 // Identify valid longjmp targets for Windows Control Flow Guard. 619 addPass(createCFGuardLongjmpPass()); 620 // Identify valid eh continuation targets for Windows EHCont Guard. 621 addPass(createEHContGuardCatchretPass()); 622 } 623 addPass(createX86LoadValueInjectionRetHardeningPass()); 624 625 // Insert pseudo probe annotation for callsite profiling 626 addPass(createPseudoProbeInserter()); 627 628 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms, 629 // also CALL_RVMARKER. 630 addPass(createUnpackMachineBundles([&TT](const MachineFunction &MF) { 631 // Only run bundle expansion if the module uses kcfi, or there are relevant 632 // ObjC runtime functions present in the module. 633 const Function &F = MF.getFunction(); 634 const Module *M = F.getParent(); 635 return M->getModuleFlag("kcfi") || 636 (TT.isOSDarwin() && 637 (M->getFunction("objc_retainAutoreleasedReturnValue") || 638 M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"))); 639 })); 640 } 641 642 bool X86PassConfig::addPostFastRegAllocRewrite() { 643 addPass(createX86FastTileConfigPass()); 644 return true; 645 } 646 647 std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const { 648 return getStandardCSEConfigForOpt(TM->getOptLevel()); 649 } 650 651 static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, 652 const TargetRegisterClass &RC) { 653 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(&RC); 654 } 655 656 bool X86PassConfig::addRegAssignAndRewriteOptimized() { 657 // Don't support tile RA when RA is specified by command line "-regalloc". 658 if (!isCustomizedRegAlloc() && EnableTileRAPass) { 659 // Allocate tile register first. 660 addPass(createGreedyRegisterAllocator(onlyAllocateTileRegisters)); 661 addPass(createX86TileConfigPass()); 662 } 663 return TargetPassConfig::addRegAssignAndRewriteOptimized(); 664 } 665