1 //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the X86 specific subclass of TargetMachine.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "X86TargetMachine.h"
14 #include "MCTargetDesc/X86MCTargetDesc.h"
15 #include "TargetInfo/X86TargetInfo.h"
16 #include "X86.h"
17 #include "X86MachineFunctionInfo.h"
18 #include "X86MacroFusion.h"
19 #include "X86Subtarget.h"
20 #include "X86TargetObjectFile.h"
21 #include "X86TargetTransformInfo.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/Analysis/TargetTransformInfo.h"
26 #include "llvm/CodeGen/ExecutionDomainFix.h"
27 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
28 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
29 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
30 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
31 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
32 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
33 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
34 #include "llvm/CodeGen/MIRParser/MIParser.h"
35 #include "llvm/CodeGen/MIRYamlMapping.h"
36 #include "llvm/CodeGen/MachineScheduler.h"
37 #include "llvm/CodeGen/Passes.h"
38 #include "llvm/CodeGen/RegAllocRegistry.h"
39 #include "llvm/CodeGen/TargetPassConfig.h"
40 #include "llvm/IR/Attributes.h"
41 #include "llvm/IR/DataLayout.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/MC/MCAsmInfo.h"
44 #include "llvm/MC/TargetRegistry.h"
45 #include "llvm/Pass.h"
46 #include "llvm/Support/CodeGen.h"
47 #include "llvm/Support/CommandLine.h"
48 #include "llvm/Support/ErrorHandling.h"
49 #include "llvm/Target/TargetLoweringObjectFile.h"
50 #include "llvm/Target/TargetOptions.h"
51 #include "llvm/TargetParser/Triple.h"
52 #include "llvm/Transforms/CFGuard.h"
53 #include <memory>
54 #include <optional>
55 #include <string>
56
57 using namespace llvm;
58
59 static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner",
60 cl::desc("Enable the machine combiner pass"),
61 cl::init(true), cl::Hidden);
62
63 static cl::opt<bool>
64 EnableTileRAPass("x86-tile-ra",
65 cl::desc("Enable the tile register allocation pass"),
66 cl::init(true), cl::Hidden);
67
LLVMInitializeX86Target()68 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
69 // Register the target.
70 RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target());
71 RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target());
72
73 PassRegistry &PR = *PassRegistry::getPassRegistry();
74 initializeX86LowerAMXIntrinsicsLegacyPassPass(PR);
75 initializeX86LowerAMXTypeLegacyPassPass(PR);
76 initializeX86PreTileConfigPass(PR);
77 initializeGlobalISel(PR);
78 initializeWinEHStatePassPass(PR);
79 initializeFixupBWInstPassPass(PR);
80 initializeCompressEVEXPassPass(PR);
81 initializeFixupLEAPassPass(PR);
82 initializeFPSPass(PR);
83 initializeX86FixupSetCCPassPass(PR);
84 initializeX86CallFrameOptimizationPass(PR);
85 initializeX86CmovConverterPassPass(PR);
86 initializeX86TileConfigPass(PR);
87 initializeX86FastPreTileConfigPass(PR);
88 initializeX86FastTileConfigPass(PR);
89 initializeKCFIPass(PR);
90 initializeX86LowerTileCopyPass(PR);
91 initializeX86ExpandPseudoPass(PR);
92 initializeX86ExecutionDomainFixPass(PR);
93 initializeX86DomainReassignmentPass(PR);
94 initializeX86AvoidSFBPassPass(PR);
95 initializeX86AvoidTrailingCallPassPass(PR);
96 initializeX86SpeculativeLoadHardeningPassPass(PR);
97 initializeX86SpeculativeExecutionSideEffectSuppressionPass(PR);
98 initializeX86FlagsCopyLoweringPassPass(PR);
99 initializeX86LoadValueInjectionLoadHardeningPassPass(PR);
100 initializeX86LoadValueInjectionRetHardeningPassPass(PR);
101 initializeX86OptimizeLEAPassPass(PR);
102 initializeX86PartialReductionPass(PR);
103 initializePseudoProbeInserterPass(PR);
104 initializeX86ReturnThunksPass(PR);
105 initializeX86DAGToDAGISelLegacyPass(PR);
106 initializeX86ArgumentStackSlotPassPass(PR);
107 initializeX86FixupInstTuningPassPass(PR);
108 initializeX86FixupVectorConstantsPassPass(PR);
109 }
110
createTLOF(const Triple & TT)111 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
112 if (TT.isOSBinFormatMachO()) {
113 if (TT.getArch() == Triple::x86_64)
114 return std::make_unique<X86_64MachoTargetObjectFile>();
115 return std::make_unique<TargetLoweringObjectFileMachO>();
116 }
117
118 if (TT.isOSBinFormatCOFF())
119 return std::make_unique<TargetLoweringObjectFileCOFF>();
120
121 if (TT.getArch() == Triple::x86_64)
122 return std::make_unique<X86_64ELFTargetObjectFile>();
123 return std::make_unique<X86ELFTargetObjectFile>();
124 }
125
computeDataLayout(const Triple & TT)126 static std::string computeDataLayout(const Triple &TT) {
127 // X86 is little endian
128 std::string Ret = "e";
129
130 Ret += DataLayout::getManglingComponent(TT);
131 // X86 and x32 have 32 bit pointers.
132 if (!TT.isArch64Bit() || TT.isX32() || TT.isOSNaCl())
133 Ret += "-p:32:32";
134
135 // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers.
136 Ret += "-p270:32:32-p271:32:32-p272:64:64";
137
138 // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32.
139 // 128 bit integers are not specified in the 32-bit ABIs but are used
140 // internally for lowering f128, so we match the alignment to that.
141 if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl())
142 Ret += "-i64:64-i128:128";
143 else if (TT.isOSIAMCU())
144 Ret += "-i64:32-f64:32";
145 else
146 Ret += "-i128:128-f64:32:64";
147
148 // Some ABIs align long double to 128 bits, others to 32.
149 if (TT.isOSNaCl() || TT.isOSIAMCU())
150 ; // No f80
151 else if (TT.isArch64Bit() || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment())
152 Ret += "-f80:128";
153 else
154 Ret += "-f80:32";
155
156 if (TT.isOSIAMCU())
157 Ret += "-f128:32";
158
159 // The registers can hold 8, 16, 32 or, in x86-64, 64 bits.
160 if (TT.isArch64Bit())
161 Ret += "-n8:16:32:64";
162 else
163 Ret += "-n8:16:32";
164
165 // The stack is aligned to 32 bits on some ABIs and 128 bits on others.
166 if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU())
167 Ret += "-a:0:32-S32";
168 else
169 Ret += "-S128";
170
171 return Ret;
172 }
173
getEffectiveRelocModel(const Triple & TT,bool JIT,std::optional<Reloc::Model> RM)174 static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT,
175 std::optional<Reloc::Model> RM) {
176 bool is64Bit = TT.getArch() == Triple::x86_64;
177 if (!RM) {
178 // JIT codegen should use static relocations by default, since it's
179 // typically executed in process and not relocatable.
180 if (JIT)
181 return Reloc::Static;
182
183 // Darwin defaults to PIC in 64 bit mode and dynamic-no-pic in 32 bit mode.
184 // Win64 requires rip-rel addressing, thus we force it to PIC. Otherwise we
185 // use static relocation model by default.
186 if (TT.isOSDarwin()) {
187 if (is64Bit)
188 return Reloc::PIC_;
189 return Reloc::DynamicNoPIC;
190 }
191 if (TT.isOSWindows() && is64Bit)
192 return Reloc::PIC_;
193 return Reloc::Static;
194 }
195
196 // ELF and X86-64 don't have a distinct DynamicNoPIC model. DynamicNoPIC
197 // is defined as a model for code which may be used in static or dynamic
198 // executables but not necessarily a shared library. On X86-32 we just
199 // compile in -static mode, in x86-64 we use PIC.
200 if (*RM == Reloc::DynamicNoPIC) {
201 if (is64Bit)
202 return Reloc::PIC_;
203 if (!TT.isOSDarwin())
204 return Reloc::Static;
205 }
206
207 // If we are on Darwin, disallow static relocation model in X86-64 mode, since
208 // the Mach-O file format doesn't support it.
209 if (*RM == Reloc::Static && TT.isOSDarwin() && is64Bit)
210 return Reloc::PIC_;
211
212 return *RM;
213 }
214
215 static CodeModel::Model
getEffectiveX86CodeModel(const Triple & TT,std::optional<CodeModel::Model> CM,bool JIT)216 getEffectiveX86CodeModel(const Triple &TT, std::optional<CodeModel::Model> CM,
217 bool JIT) {
218 bool Is64Bit = TT.getArch() == Triple::x86_64;
219 if (CM) {
220 if (*CM == CodeModel::Tiny)
221 report_fatal_error("Target does not support the tiny CodeModel", false);
222 return *CM;
223 }
224 if (JIT)
225 return Is64Bit ? CodeModel::Large : CodeModel::Small;
226 return CodeModel::Small;
227 }
228
229 /// Create an X86 target.
230 ///
X86TargetMachine(const Target & T,const Triple & TT,StringRef CPU,StringRef FS,const TargetOptions & Options,std::optional<Reloc::Model> RM,std::optional<CodeModel::Model> CM,CodeGenOptLevel OL,bool JIT)231 X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
232 StringRef CPU, StringRef FS,
233 const TargetOptions &Options,
234 std::optional<Reloc::Model> RM,
235 std::optional<CodeModel::Model> CM,
236 CodeGenOptLevel OL, bool JIT)
237 : LLVMTargetMachine(
238 T, computeDataLayout(TT), TT, CPU, FS, Options,
239 getEffectiveRelocModel(TT, JIT, RM),
240 getEffectiveX86CodeModel(TT, CM, JIT),
241 OL),
242 TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) {
243 // On PS4/PS5, the "return address" of a 'noreturn' call must still be within
244 // the calling function, and TrapUnreachable is an easy way to get that.
245 if (TT.isPS() || TT.isOSBinFormatMachO()) {
246 this->Options.TrapUnreachable = true;
247 this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
248 }
249
250 setMachineOutliner(true);
251
252 // x86 supports the debug entry values.
253 setSupportsDebugEntryValues(true);
254
255 initAsmInfo();
256 }
257
258 X86TargetMachine::~X86TargetMachine() = default;
259
260 const X86Subtarget *
getSubtargetImpl(const Function & F) const261 X86TargetMachine::getSubtargetImpl(const Function &F) const {
262 Attribute CPUAttr = F.getFnAttribute("target-cpu");
263 Attribute TuneAttr = F.getFnAttribute("tune-cpu");
264 Attribute FSAttr = F.getFnAttribute("target-features");
265
266 StringRef CPU =
267 CPUAttr.isValid() ? CPUAttr.getValueAsString() : (StringRef)TargetCPU;
268 // "x86-64" is a default target setting for many front ends. In these cases,
269 // they actually request for "generic" tuning unless the "tune-cpu" was
270 // specified.
271 StringRef TuneCPU = TuneAttr.isValid() ? TuneAttr.getValueAsString()
272 : CPU == "x86-64" ? "generic"
273 : (StringRef)CPU;
274 StringRef FS =
275 FSAttr.isValid() ? FSAttr.getValueAsString() : (StringRef)TargetFS;
276
277 SmallString<512> Key;
278 // The additions here are ordered so that the definitely short strings are
279 // added first so we won't exceed the small size. We append the
280 // much longer FS string at the end so that we only heap allocate at most
281 // one time.
282
283 // Extract prefer-vector-width attribute.
284 unsigned PreferVectorWidthOverride = 0;
285 Attribute PreferVecWidthAttr = F.getFnAttribute("prefer-vector-width");
286 if (PreferVecWidthAttr.isValid()) {
287 StringRef Val = PreferVecWidthAttr.getValueAsString();
288 unsigned Width;
289 if (!Val.getAsInteger(0, Width)) {
290 Key += 'p';
291 Key += Val;
292 PreferVectorWidthOverride = Width;
293 }
294 }
295
296 // Extract min-legal-vector-width attribute.
297 unsigned RequiredVectorWidth = UINT32_MAX;
298 Attribute MinLegalVecWidthAttr = F.getFnAttribute("min-legal-vector-width");
299 if (MinLegalVecWidthAttr.isValid()) {
300 StringRef Val = MinLegalVecWidthAttr.getValueAsString();
301 unsigned Width;
302 if (!Val.getAsInteger(0, Width)) {
303 Key += 'm';
304 Key += Val;
305 RequiredVectorWidth = Width;
306 }
307 }
308
309 // Add CPU to the Key.
310 Key += CPU;
311
312 // Add tune CPU to the Key.
313 Key += TuneCPU;
314
315 // Keep track of the start of the feature portion of the string.
316 unsigned FSStart = Key.size();
317
318 // FIXME: This is related to the code below to reset the target options,
319 // we need to know whether or not the soft float flag is set on the
320 // function before we can generate a subtarget. We also need to use
321 // it as a key for the subtarget since that can be the only difference
322 // between two functions.
323 bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsBool();
324 // If the soft float attribute is set on the function turn on the soft float
325 // subtarget feature.
326 if (SoftFloat)
327 Key += FS.empty() ? "+soft-float" : "+soft-float,";
328
329 Key += FS;
330
331 // We may have added +soft-float to the features so move the StringRef to
332 // point to the full string in the Key.
333 FS = Key.substr(FSStart);
334
335 auto &I = SubtargetMap[Key];
336 if (!I) {
337 // This needs to be done before we create a new subtarget since any
338 // creation will depend on the TM and the code generation flags on the
339 // function that reside in TargetOptions.
340 resetTargetOptions(F);
341 I = std::make_unique<X86Subtarget>(
342 TargetTriple, CPU, TuneCPU, FS, *this,
343 MaybeAlign(F.getParent()->getOverrideStackAlignment()),
344 PreferVectorWidthOverride, RequiredVectorWidth);
345 }
346 return I.get();
347 }
348
createDefaultFuncInfoYAML() const349 yaml::MachineFunctionInfo *X86TargetMachine::createDefaultFuncInfoYAML() const {
350 return new yaml::X86MachineFunctionInfo();
351 }
352
353 yaml::MachineFunctionInfo *
convertFuncInfoToYAML(const MachineFunction & MF) const354 X86TargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
355 const auto *MFI = MF.getInfo<X86MachineFunctionInfo>();
356 return new yaml::X86MachineFunctionInfo(*MFI);
357 }
358
parseMachineFunctionInfo(const yaml::MachineFunctionInfo & MFI,PerFunctionMIParsingState & PFS,SMDiagnostic & Error,SMRange & SourceRange) const359 bool X86TargetMachine::parseMachineFunctionInfo(
360 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
361 SMDiagnostic &Error, SMRange &SourceRange) const {
362 const auto &YamlMFI = static_cast<const yaml::X86MachineFunctionInfo &>(MFI);
363 PFS.MF.getInfo<X86MachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
364 return false;
365 }
366
isNoopAddrSpaceCast(unsigned SrcAS,unsigned DestAS) const367 bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
368 unsigned DestAS) const {
369 assert(SrcAS != DestAS && "Expected different address spaces!");
370 if (getPointerSize(SrcAS) != getPointerSize(DestAS))
371 return false;
372 return SrcAS < 256 && DestAS < 256;
373 }
374
375 //===----------------------------------------------------------------------===//
376 // X86 TTI query.
377 //===----------------------------------------------------------------------===//
378
379 TargetTransformInfo
getTargetTransformInfo(const Function & F) const380 X86TargetMachine::getTargetTransformInfo(const Function &F) const {
381 return TargetTransformInfo(X86TTIImpl(this, F));
382 }
383
384 //===----------------------------------------------------------------------===//
385 // Pass Pipeline Configuration
386 //===----------------------------------------------------------------------===//
387
388 namespace {
389
390 /// X86 Code Generator Pass Configuration Options.
391 class X86PassConfig : public TargetPassConfig {
392 public:
X86PassConfig(X86TargetMachine & TM,PassManagerBase & PM)393 X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM)
394 : TargetPassConfig(TM, PM) {}
395
getX86TargetMachine() const396 X86TargetMachine &getX86TargetMachine() const {
397 return getTM<X86TargetMachine>();
398 }
399
400 ScheduleDAGInstrs *
createMachineScheduler(MachineSchedContext * C) const401 createMachineScheduler(MachineSchedContext *C) const override {
402 ScheduleDAGMILive *DAG = createGenericSchedLive(C);
403 DAG->addMutation(createX86MacroFusionDAGMutation());
404 return DAG;
405 }
406
407 ScheduleDAGInstrs *
createPostMachineScheduler(MachineSchedContext * C) const408 createPostMachineScheduler(MachineSchedContext *C) const override {
409 ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
410 DAG->addMutation(createX86MacroFusionDAGMutation());
411 return DAG;
412 }
413
414 void addIRPasses() override;
415 bool addInstSelector() override;
416 bool addIRTranslator() override;
417 bool addLegalizeMachineIR() override;
418 bool addRegBankSelect() override;
419 bool addGlobalInstructionSelect() override;
420 bool addILPOpts() override;
421 bool addPreISel() override;
422 void addMachineSSAOptimization() override;
423 void addPreRegAlloc() override;
424 bool addPostFastRegAllocRewrite() override;
425 void addPostRegAlloc() override;
426 void addPreEmitPass() override;
427 void addPreEmitPass2() override;
428 void addPreSched2() override;
429 bool addRegAssignAndRewriteOptimized() override;
430
431 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
432 };
433
434 class X86ExecutionDomainFix : public ExecutionDomainFix {
435 public:
436 static char ID;
X86ExecutionDomainFix()437 X86ExecutionDomainFix() : ExecutionDomainFix(ID, X86::VR128XRegClass) {}
getPassName() const438 StringRef getPassName() const override {
439 return "X86 Execution Dependency Fix";
440 }
441 };
442 char X86ExecutionDomainFix::ID;
443
444 } // end anonymous namespace
445
446 INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix",
447 "X86 Execution Domain Fix", false, false)
INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis)448 INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis)
449 INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix",
450 "X86 Execution Domain Fix", false, false)
451
452 TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
453 return new X86PassConfig(*this, PM);
454 }
455
createMachineFunctionInfo(BumpPtrAllocator & Allocator,const Function & F,const TargetSubtargetInfo * STI) const456 MachineFunctionInfo *X86TargetMachine::createMachineFunctionInfo(
457 BumpPtrAllocator &Allocator, const Function &F,
458 const TargetSubtargetInfo *STI) const {
459 return X86MachineFunctionInfo::create<X86MachineFunctionInfo>(Allocator, F,
460 STI);
461 }
462
addIRPasses()463 void X86PassConfig::addIRPasses() {
464 addPass(createAtomicExpandLegacyPass());
465
466 // We add both pass anyway and when these two passes run, we skip the pass
467 // based on the option level and option attribute.
468 addPass(createX86LowerAMXIntrinsicsPass());
469 addPass(createX86LowerAMXTypePass());
470
471 TargetPassConfig::addIRPasses();
472
473 if (TM->getOptLevel() != CodeGenOptLevel::None) {
474 addPass(createInterleavedAccessPass());
475 addPass(createX86PartialReductionPass());
476 }
477
478 // Add passes that handle indirect branch removal and insertion of a retpoline
479 // thunk. These will be a no-op unless a function subtarget has the retpoline
480 // feature enabled.
481 addPass(createIndirectBrExpandPass());
482
483 // Add Control Flow Guard checks.
484 const Triple &TT = TM->getTargetTriple();
485 if (TT.isOSWindows()) {
486 if (TT.getArch() == Triple::x86_64) {
487 addPass(createCFGuardDispatchPass());
488 } else {
489 addPass(createCFGuardCheckPass());
490 }
491 }
492
493 if (TM->Options.JMCInstrument)
494 addPass(createJMCInstrumenterPass());
495 }
496
addInstSelector()497 bool X86PassConfig::addInstSelector() {
498 // Install an instruction selector.
499 addPass(createX86ISelDag(getX86TargetMachine(), getOptLevel()));
500
501 // For ELF, cleanup any local-dynamic TLS accesses.
502 if (TM->getTargetTriple().isOSBinFormatELF() &&
503 getOptLevel() != CodeGenOptLevel::None)
504 addPass(createCleanupLocalDynamicTLSPass());
505
506 addPass(createX86GlobalBaseRegPass());
507 addPass(createX86ArgumentStackSlotPass());
508 return false;
509 }
510
addIRTranslator()511 bool X86PassConfig::addIRTranslator() {
512 addPass(new IRTranslator(getOptLevel()));
513 return false;
514 }
515
addLegalizeMachineIR()516 bool X86PassConfig::addLegalizeMachineIR() {
517 addPass(new Legalizer());
518 return false;
519 }
520
addRegBankSelect()521 bool X86PassConfig::addRegBankSelect() {
522 addPass(new RegBankSelect());
523 return false;
524 }
525
addGlobalInstructionSelect()526 bool X86PassConfig::addGlobalInstructionSelect() {
527 addPass(new InstructionSelect(getOptLevel()));
528 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
529 if (isGlobalISelAbortEnabled())
530 addPass(createX86GlobalBaseRegPass());
531 return false;
532 }
533
addILPOpts()534 bool X86PassConfig::addILPOpts() {
535 addPass(&EarlyIfConverterID);
536 if (EnableMachineCombinerPass)
537 addPass(&MachineCombinerID);
538 addPass(createX86CmovConverterPass());
539 return true;
540 }
541
addPreISel()542 bool X86PassConfig::addPreISel() {
543 // Only add this pass for 32-bit x86 Windows.
544 const Triple &TT = TM->getTargetTriple();
545 if (TT.isOSWindows() && TT.getArch() == Triple::x86)
546 addPass(createX86WinEHStatePass());
547 return true;
548 }
549
addPreRegAlloc()550 void X86PassConfig::addPreRegAlloc() {
551 if (getOptLevel() != CodeGenOptLevel::None) {
552 addPass(&LiveRangeShrinkID);
553 addPass(createX86WinFixupBufferSecurityCheckPass());
554 addPass(createX86FixupSetCC());
555 addPass(createX86OptimizeLEAs());
556 addPass(createX86CallFrameOptimization());
557 addPass(createX86AvoidStoreForwardingBlocks());
558 }
559
560 addPass(createX86SpeculativeLoadHardeningPass());
561 addPass(createX86FlagsCopyLoweringPass());
562 addPass(createX86DynAllocaExpander());
563
564 if (getOptLevel() != CodeGenOptLevel::None)
565 addPass(createX86PreTileConfigPass());
566 else
567 addPass(createX86FastPreTileConfigPass());
568 }
569
addMachineSSAOptimization()570 void X86PassConfig::addMachineSSAOptimization() {
571 addPass(createX86DomainReassignmentPass());
572 TargetPassConfig::addMachineSSAOptimization();
573 }
574
addPostRegAlloc()575 void X86PassConfig::addPostRegAlloc() {
576 addPass(createX86LowerTileCopyPass());
577 addPass(createX86FloatingPointStackifierPass());
578 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
579 // to using the Speculative Execution Side Effect Suppression pass for
580 // mitigation. This is to prevent slow downs due to
581 // analyses needed by the LVIHardening pass when compiling at -O0.
582 if (getOptLevel() != CodeGenOptLevel::None)
583 addPass(createX86LoadValueInjectionLoadHardeningPass());
584 }
585
addPreSched2()586 void X86PassConfig::addPreSched2() {
587 addPass(createX86ExpandPseudoPass());
588 addPass(createKCFIPass());
589 }
590
addPreEmitPass()591 void X86PassConfig::addPreEmitPass() {
592 if (getOptLevel() != CodeGenOptLevel::None) {
593 addPass(new X86ExecutionDomainFix());
594 addPass(createBreakFalseDeps());
595 }
596
597 addPass(createX86IndirectBranchTrackingPass());
598
599 addPass(createX86IssueVZeroUpperPass());
600
601 if (getOptLevel() != CodeGenOptLevel::None) {
602 addPass(createX86FixupBWInsts());
603 addPass(createX86PadShortFunctions());
604 addPass(createX86FixupLEAs());
605 addPass(createX86FixupInstTuning());
606 addPass(createX86FixupVectorConstants());
607 }
608 addPass(createX86CompressEVEXPass());
609 addPass(createX86DiscriminateMemOpsPass());
610 addPass(createX86InsertPrefetchPass());
611 addPass(createX86InsertX87waitPass());
612 }
613
addPreEmitPass2()614 void X86PassConfig::addPreEmitPass2() {
615 const Triple &TT = TM->getTargetTriple();
616 const MCAsmInfo *MAI = TM->getMCAsmInfo();
617
618 // The X86 Speculative Execution Pass must run after all control
619 // flow graph modifying passes. As a result it was listed to run right before
620 // the X86 Retpoline Thunks pass. The reason it must run after control flow
621 // graph modifications is that the model of LFENCE in LLVM has to be updated
622 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
623 // placement of this pass was hand checked to ensure that the subsequent
624 // passes don't move the code around the LFENCEs in a way that will hurt the
625 // correctness of this pass. This placement has been shown to work based on
626 // hand inspection of the codegen output.
627 addPass(createX86SpeculativeExecutionSideEffectSuppression());
628 addPass(createX86IndirectThunksPass());
629 addPass(createX86ReturnThunksPass());
630
631 // Insert extra int3 instructions after trailing call instructions to avoid
632 // issues in the unwinder.
633 if (TT.isOSWindows() && TT.getArch() == Triple::x86_64)
634 addPass(createX86AvoidTrailingCallPass());
635
636 // Verify basic block incoming and outgoing cfa offset and register values and
637 // correct CFA calculation rule where needed by inserting appropriate CFI
638 // instructions.
639 if (!TT.isOSDarwin() &&
640 (!TT.isOSWindows() ||
641 MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI))
642 addPass(createCFIInstrInserter());
643
644 if (TT.isOSWindows()) {
645 // Identify valid longjmp targets for Windows Control Flow Guard.
646 addPass(createCFGuardLongjmpPass());
647 // Identify valid eh continuation targets for Windows EHCont Guard.
648 addPass(createEHContGuardCatchretPass());
649 }
650 addPass(createX86LoadValueInjectionRetHardeningPass());
651
652 // Insert pseudo probe annotation for callsite profiling
653 addPass(createPseudoProbeInserter());
654
655 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
656 // also CALL_RVMARKER.
657 addPass(createUnpackMachineBundles([&TT](const MachineFunction &MF) {
658 // Only run bundle expansion if the module uses kcfi, or there are relevant
659 // ObjC runtime functions present in the module.
660 const Function &F = MF.getFunction();
661 const Module *M = F.getParent();
662 return M->getModuleFlag("kcfi") ||
663 (TT.isOSDarwin() &&
664 (M->getFunction("objc_retainAutoreleasedReturnValue") ||
665 M->getFunction("objc_unsafeClaimAutoreleasedReturnValue")));
666 }));
667 }
668
addPostFastRegAllocRewrite()669 bool X86PassConfig::addPostFastRegAllocRewrite() {
670 addPass(createX86FastTileConfigPass());
671 return true;
672 }
673
getCSEConfig() const674 std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
675 return getStandardCSEConfigForOpt(TM->getOptLevel());
676 }
677
onlyAllocateTileRegisters(const TargetRegisterInfo & TRI,const MachineRegisterInfo & MRI,const Register Reg)678 static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
679 const MachineRegisterInfo &MRI,
680 const Register Reg) {
681 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
682 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
683 }
684
addRegAssignAndRewriteOptimized()685 bool X86PassConfig::addRegAssignAndRewriteOptimized() {
686 // Don't support tile RA when RA is specified by command line "-regalloc".
687 if (!isCustomizedRegAlloc() && EnableTileRAPass) {
688 // Allocate tile register first.
689 addPass(createGreedyRegisterAllocator(onlyAllocateTileRegisters));
690 addPass(createX86TileConfigPass());
691 }
692 return TargetPassConfig::addRegAssignAndRewriteOptimized();
693 }
694