1 //===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This tablegen backend emits code for use by the GlobalISel instruction 11 /// selector. See include/llvm/Target/GlobalISel/Target.td. 12 /// 13 /// This file analyzes the patterns recognized by the SelectionDAGISel tablegen 14 /// backend, filters out the ones that are unsupported, maps 15 /// SelectionDAG-specific constructs to their GlobalISel counterpart 16 /// (when applicable: MVT to LLT; SDNode to generic Instruction). 17 /// 18 /// Not all patterns are supported: pass the tablegen invocation 19 /// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped, 20 /// as well as why. 21 /// 22 /// The generated file defines a single method: 23 /// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const; 24 /// intended to be used in InstructionSelector::select as the first-step 25 /// selector for the patterns that don't require complex C++. 26 /// 27 /// FIXME: We'll probably want to eventually define a base 28 /// "TargetGenInstructionSelector" class. 29 /// 30 //===----------------------------------------------------------------------===// 31 32 #include "CodeGenDAGPatterns.h" 33 #include "CodeGenInstruction.h" 34 #include "CodeGenIntrinsics.h" 35 #include "CodeGenRegisters.h" 36 #include "CodeGenTarget.h" 37 #include "GlobalISelMatchTable.h" 38 #include "GlobalISelMatchTableExecutorEmitter.h" 39 #include "InfoByHwMode.h" 40 #include "SubtargetFeatureInfo.h" 41 #include "llvm/ADT/Statistic.h" 42 #include "llvm/CodeGen/LowLevelType.h" 43 #include "llvm/CodeGen/MachineValueType.h" 44 #include "llvm/Support/CodeGenCoverage.h" 45 #include "llvm/Support/CommandLine.h" 46 #include "llvm/Support/Error.h" 47 #include "llvm/Support/SaveAndRestore.h" 48 #include "llvm/Support/ScopedPrinter.h" 49 #include "llvm/TableGen/Error.h" 50 #include "llvm/TableGen/Record.h" 51 #include "llvm/TableGen/TableGenBackend.h" 52 #include <numeric> 53 #include <string> 54 55 using namespace llvm; 56 using namespace llvm::gi; 57 58 using action_iterator = RuleMatcher::action_iterator; 59 60 #define DEBUG_TYPE "gisel-emitter" 61 62 STATISTIC(NumPatternTotal, "Total number of patterns"); 63 STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG"); 64 STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped"); 65 STATISTIC(NumPatternsTested, 66 "Number of patterns executed according to coverage information"); 67 68 cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel"); 69 70 static cl::opt<bool> WarnOnSkippedPatterns( 71 "warn-on-skipped-patterns", 72 cl::desc("Explain why a pattern was skipped for inclusion " 73 "in the GlobalISel selector"), 74 cl::init(false), cl::cat(GlobalISelEmitterCat)); 75 76 static cl::opt<bool> GenerateCoverage( 77 "instrument-gisel-coverage", 78 cl::desc("Generate coverage instrumentation for GlobalISel"), 79 cl::init(false), cl::cat(GlobalISelEmitterCat)); 80 81 static cl::opt<std::string> UseCoverageFile( 82 "gisel-coverage-file", cl::init(""), 83 cl::desc("Specify file to retrieve coverage information from"), 84 cl::cat(GlobalISelEmitterCat)); 85 86 static cl::opt<bool> OptimizeMatchTable( 87 "optimize-match-table", 88 cl::desc("Generate an optimized version of the match table"), 89 cl::init(true), cl::cat(GlobalISelEmitterCat)); 90 91 namespace { 92 93 static std::string explainPredicates(const TreePatternNode *N) { 94 std::string Explanation; 95 StringRef Separator = ""; 96 for (const TreePredicateCall &Call : N->getPredicateCalls()) { 97 const TreePredicateFn &P = Call.Fn; 98 Explanation += 99 (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str(); 100 Separator = ", "; 101 102 if (P.isAlwaysTrue()) 103 Explanation += " always-true"; 104 if (P.isImmediatePattern()) 105 Explanation += " immediate"; 106 107 if (P.isUnindexed()) 108 Explanation += " unindexed"; 109 110 if (P.isNonExtLoad()) 111 Explanation += " non-extload"; 112 if (P.isAnyExtLoad()) 113 Explanation += " extload"; 114 if (P.isSignExtLoad()) 115 Explanation += " sextload"; 116 if (P.isZeroExtLoad()) 117 Explanation += " zextload"; 118 119 if (P.isNonTruncStore()) 120 Explanation += " non-truncstore"; 121 if (P.isTruncStore()) 122 Explanation += " truncstore"; 123 124 if (Record *VT = P.getMemoryVT()) 125 Explanation += (" MemVT=" + VT->getName()).str(); 126 if (Record *VT = P.getScalarMemoryVT()) 127 Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str(); 128 129 if (ListInit *AddrSpaces = P.getAddressSpaces()) { 130 raw_string_ostream OS(Explanation); 131 OS << " AddressSpaces=["; 132 133 StringRef AddrSpaceSeparator; 134 for (Init *Val : AddrSpaces->getValues()) { 135 IntInit *IntVal = dyn_cast<IntInit>(Val); 136 if (!IntVal) 137 continue; 138 139 OS << AddrSpaceSeparator << IntVal->getValue(); 140 AddrSpaceSeparator = ", "; 141 } 142 143 OS << ']'; 144 } 145 146 int64_t MinAlign = P.getMinAlignment(); 147 if (MinAlign > 0) 148 Explanation += " MinAlign=" + utostr(MinAlign); 149 150 if (P.isAtomicOrderingMonotonic()) 151 Explanation += " monotonic"; 152 if (P.isAtomicOrderingAcquire()) 153 Explanation += " acquire"; 154 if (P.isAtomicOrderingRelease()) 155 Explanation += " release"; 156 if (P.isAtomicOrderingAcquireRelease()) 157 Explanation += " acq_rel"; 158 if (P.isAtomicOrderingSequentiallyConsistent()) 159 Explanation += " seq_cst"; 160 if (P.isAtomicOrderingAcquireOrStronger()) 161 Explanation += " >=acquire"; 162 if (P.isAtomicOrderingWeakerThanAcquire()) 163 Explanation += " <acquire"; 164 if (P.isAtomicOrderingReleaseOrStronger()) 165 Explanation += " >=release"; 166 if (P.isAtomicOrderingWeakerThanRelease()) 167 Explanation += " <release"; 168 } 169 return Explanation; 170 } 171 172 std::string explainOperator(Record *Operator) { 173 if (Operator->isSubClassOf("SDNode")) 174 return (" (" + Operator->getValueAsString("Opcode") + ")").str(); 175 176 if (Operator->isSubClassOf("Intrinsic")) 177 return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str(); 178 179 if (Operator->isSubClassOf("ComplexPattern")) 180 return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() + 181 ")") 182 .str(); 183 184 if (Operator->isSubClassOf("SDNodeXForm")) 185 return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() + 186 ")") 187 .str(); 188 189 return (" (Operator " + Operator->getName() + " not understood)").str(); 190 } 191 192 /// Helper function to let the emitter report skip reason error messages. 193 static Error failedImport(const Twine &Reason) { 194 return make_error<StringError>(Reason, inconvertibleErrorCode()); 195 } 196 197 static Error isTrivialOperatorNode(const TreePatternNode *N) { 198 std::string Explanation; 199 std::string Separator; 200 201 bool HasUnsupportedPredicate = false; 202 for (const TreePredicateCall &Call : N->getPredicateCalls()) { 203 const TreePredicateFn &Predicate = Call.Fn; 204 205 if (Predicate.isAlwaysTrue()) 206 continue; 207 208 if (Predicate.isImmediatePattern()) 209 continue; 210 211 if (Predicate.hasNoUse()) 212 continue; 213 214 if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() || 215 Predicate.isSignExtLoad() || Predicate.isZeroExtLoad()) 216 continue; 217 218 if (Predicate.isNonTruncStore() || Predicate.isTruncStore()) 219 continue; 220 221 if (Predicate.isLoad() && Predicate.getMemoryVT()) 222 continue; 223 224 if (Predicate.isLoad() || Predicate.isStore()) { 225 if (Predicate.isUnindexed()) 226 continue; 227 } 228 229 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) { 230 const ListInit *AddrSpaces = Predicate.getAddressSpaces(); 231 if (AddrSpaces && !AddrSpaces->empty()) 232 continue; 233 234 if (Predicate.getMinAlignment() > 0) 235 continue; 236 } 237 238 if (Predicate.isAtomic() && Predicate.getMemoryVT()) 239 continue; 240 241 if (Predicate.isAtomic() && 242 (Predicate.isAtomicOrderingMonotonic() || 243 Predicate.isAtomicOrderingAcquire() || 244 Predicate.isAtomicOrderingRelease() || 245 Predicate.isAtomicOrderingAcquireRelease() || 246 Predicate.isAtomicOrderingSequentiallyConsistent() || 247 Predicate.isAtomicOrderingAcquireOrStronger() || 248 Predicate.isAtomicOrderingWeakerThanAcquire() || 249 Predicate.isAtomicOrderingReleaseOrStronger() || 250 Predicate.isAtomicOrderingWeakerThanRelease())) 251 continue; 252 253 if (Predicate.hasGISelPredicateCode()) 254 continue; 255 256 HasUnsupportedPredicate = true; 257 Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")"; 258 Separator = ", "; 259 Explanation += (Separator + "first-failing:" + 260 Predicate.getOrigPatFragRecord()->getRecord()->getName()) 261 .str(); 262 break; 263 } 264 265 if (!HasUnsupportedPredicate) 266 return Error::success(); 267 268 return failedImport(Explanation); 269 } 270 271 static Record *getInitValueAsRegClass(Init *V) { 272 if (DefInit *VDefInit = dyn_cast<DefInit>(V)) { 273 if (VDefInit->getDef()->isSubClassOf("RegisterOperand")) 274 return VDefInit->getDef()->getValueAsDef("RegClass"); 275 if (VDefInit->getDef()->isSubClassOf("RegisterClass")) 276 return VDefInit->getDef(); 277 } 278 return nullptr; 279 } 280 281 static std::string getScopedName(unsigned Scope, const std::string &Name) { 282 return ("pred:" + Twine(Scope) + ":" + Name).str(); 283 } 284 285 static std::string getMangledRootDefName(StringRef DefOperandName) { 286 return ("DstI[" + DefOperandName + "]").str(); 287 } 288 289 //===- GlobalISelEmitter class --------------------------------------------===// 290 291 static Expected<LLTCodeGen> getInstResultType(const TreePatternNode *Dst) { 292 ArrayRef<TypeSetByHwMode> ChildTypes = Dst->getExtTypes(); 293 if (ChildTypes.size() != 1) 294 return failedImport("Dst pattern child has multiple results"); 295 296 std::optional<LLTCodeGen> MaybeOpTy; 297 if (ChildTypes.front().isMachineValueType()) { 298 MaybeOpTy = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy); 299 } 300 301 if (!MaybeOpTy) 302 return failedImport("Dst operand has an unsupported type"); 303 return *MaybeOpTy; 304 } 305 306 class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter { 307 public: 308 explicit GlobalISelEmitter(RecordKeeper &RK); 309 310 void emitAdditionalImpl(raw_ostream &OS) override; 311 312 void emitMIPredicateFns(raw_ostream &OS) override; 313 void emitI64ImmPredicateFns(raw_ostream &OS) override; 314 void emitAPFloatImmPredicateFns(raw_ostream &OS) override; 315 void emitAPIntImmPredicateFns(raw_ostream &OS) override; 316 void emitTestSimplePredicate(raw_ostream &OS) override; 317 void emitRunCustomAction(raw_ostream &OS) override; 318 319 void postProcessRule(RuleMatcher &M); 320 321 const CodeGenTarget &getTarget() const override { return Target; } 322 StringRef getClassName() const override { return ClassName; } 323 324 void run(raw_ostream &OS); 325 326 private: 327 std::string ClassName; 328 329 const RecordKeeper &RK; 330 const CodeGenDAGPatterns CGP; 331 const CodeGenTarget &Target; 332 CodeGenRegBank &CGRegs; 333 334 std::vector<Record *> AllPatFrags; 335 336 /// Keep track of the equivalence between SDNodes and Instruction by mapping 337 /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to 338 /// check for attributes on the relation such as CheckMMOIsNonAtomic. 339 /// This is defined using 'GINodeEquiv' in the target description. 340 DenseMap<Record *, Record *> NodeEquivs; 341 342 /// Keep track of the equivalence between ComplexPattern's and 343 /// GIComplexOperandMatcher. Map entries are specified by subclassing 344 /// GIComplexPatternEquiv. 345 DenseMap<const Record *, const Record *> ComplexPatternEquivs; 346 347 /// Keep track of the equivalence between SDNodeXForm's and 348 /// GICustomOperandRenderer. Map entries are specified by subclassing 349 /// GISDNodeXFormEquiv. 350 DenseMap<const Record *, const Record *> SDNodeXFormEquivs; 351 352 /// Keep track of Scores of PatternsToMatch similar to how the DAG does. 353 /// This adds compatibility for RuleMatchers to use this for ordering rules. 354 DenseMap<uint64_t, int> RuleMatcherScores; 355 356 // Rule coverage information. 357 std::optional<CodeGenCoverage> RuleCoverage; 358 359 /// Variables used to help with collecting of named operands for predicates 360 /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set 361 /// to the number of named operands that predicate expects. Store locations in 362 /// StoreIdxForName correspond to the order in which operand names appear in 363 /// predicate's argument list. 364 /// When we visit named operand and WaitingForNamedOperands is not zero, add 365 /// matcher that will record operand and decrease counter. 366 unsigned WaitingForNamedOperands = 0; 367 StringMap<unsigned> StoreIdxForName; 368 369 void gatherOpcodeValues(); 370 void gatherTypeIDValues(); 371 void gatherNodeEquivs(); 372 373 Record *findNodeEquiv(Record *N) const; 374 const CodeGenInstruction *getEquivNode(Record &Equiv, 375 const TreePatternNode *N) const; 376 377 Error importRulePredicates(RuleMatcher &M, ArrayRef<Record *> Predicates); 378 Expected<InstructionMatcher &> 379 createAndImportSelDAGMatcher(RuleMatcher &Rule, 380 InstructionMatcher &InsnMatcher, 381 const TreePatternNode *Src, unsigned &TempOpIdx); 382 Error importComplexPatternOperandMatcher(OperandMatcher &OM, Record *R, 383 unsigned &TempOpIdx) const; 384 Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher, 385 const TreePatternNode *SrcChild, 386 bool OperandIsAPointer, bool OperandIsImmArg, 387 unsigned OpIdx, unsigned &TempOpIdx); 388 389 Expected<BuildMIAction &> createAndImportInstructionRenderer( 390 RuleMatcher &M, InstructionMatcher &InsnMatcher, 391 const TreePatternNode *Src, const TreePatternNode *Dst); 392 Expected<action_iterator> createAndImportSubInstructionRenderer( 393 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst, 394 const TreePatternNode *Src, unsigned TempReg); 395 Expected<action_iterator> 396 createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M, 397 const TreePatternNode *Dst); 398 399 Expected<action_iterator> importExplicitDefRenderers( 400 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, 401 const TreePatternNode *Src, const TreePatternNode *Dst); 402 403 Expected<action_iterator> importExplicitUseRenderers( 404 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, 405 const llvm::TreePatternNode *Dst, const TreePatternNode *Src); 406 Expected<action_iterator> importExplicitUseRenderer( 407 action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder, 408 const TreePatternNode *DstChild, const TreePatternNode *Src); 409 Error importDefaultOperandRenderers(action_iterator InsertPt, RuleMatcher &M, 410 BuildMIAction &DstMIBuilder, 411 const DAGDefaultOperand &DefaultOp) const; 412 Error 413 importImplicitDefRenderers(BuildMIAction &DstMIBuilder, 414 const std::vector<Record *> &ImplicitDefs) const; 415 416 /// Analyze pattern \p P, returning a matcher for it if possible. 417 /// Otherwise, return an Error explaining why we don't support it. 418 Expected<RuleMatcher> runOnPattern(const PatternToMatch &P); 419 420 void declareSubtargetFeature(Record *Predicate); 421 422 unsigned declareHwModeCheck(StringRef HwModeFeatures); 423 424 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize, 425 bool WithCoverage); 426 427 /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned 428 /// CodeGenRegisterClass will support the CodeGenRegisterClass of 429 /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode. 430 /// If no register class is found, return std::nullopt. 431 std::optional<const CodeGenRegisterClass *> 432 inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty, 433 const TreePatternNode *SuperRegNode, 434 const TreePatternNode *SubRegIdxNode); 435 std::optional<CodeGenSubRegIndex *> 436 inferSubRegIndexForNode(const TreePatternNode *SubRegIdxNode); 437 438 /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode. 439 /// Return std::nullopt if no such class exists. 440 std::optional<const CodeGenRegisterClass *> 441 inferSuperRegisterClass(const TypeSetByHwMode &Ty, 442 const TreePatternNode *SubRegIdxNode); 443 444 /// Return the CodeGenRegisterClass associated with \p Leaf if it has one. 445 std::optional<const CodeGenRegisterClass *> 446 getRegClassFromLeaf(const TreePatternNode *Leaf); 447 448 /// Return a CodeGenRegisterClass for \p N if one can be found. Return 449 /// std::nullopt otherwise. 450 std::optional<const CodeGenRegisterClass *> 451 inferRegClassFromPattern(const TreePatternNode *N); 452 453 /// Return the size of the MemoryVT in this predicate, if possible. 454 std::optional<unsigned> 455 getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate); 456 457 // Add builtin predicates. 458 Expected<InstructionMatcher &> 459 addBuiltinPredicates(const Record *SrcGIEquivOrNull, 460 const TreePredicateFn &Predicate, 461 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher); 462 }; 463 464 StringRef getPatFragPredicateEnumName(Record *R) { return R->getName(); } 465 466 void GlobalISelEmitter::gatherOpcodeValues() { 467 InstructionOpcodeMatcher::initOpcodeValuesMap(Target); 468 } 469 470 void GlobalISelEmitter::gatherTypeIDValues() { 471 LLTOperandMatcher::initTypeIDValuesMap(); 472 } 473 474 void GlobalISelEmitter::gatherNodeEquivs() { 475 assert(NodeEquivs.empty()); 476 for (Record *Equiv : RK.getAllDerivedDefinitions("GINodeEquiv")) 477 NodeEquivs[Equiv->getValueAsDef("Node")] = Equiv; 478 479 assert(ComplexPatternEquivs.empty()); 480 for (Record *Equiv : RK.getAllDerivedDefinitions("GIComplexPatternEquiv")) { 481 Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent"); 482 if (!SelDAGEquiv) 483 continue; 484 ComplexPatternEquivs[SelDAGEquiv] = Equiv; 485 } 486 487 assert(SDNodeXFormEquivs.empty()); 488 for (Record *Equiv : RK.getAllDerivedDefinitions("GISDNodeXFormEquiv")) { 489 Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent"); 490 if (!SelDAGEquiv) 491 continue; 492 SDNodeXFormEquivs[SelDAGEquiv] = Equiv; 493 } 494 } 495 496 Record *GlobalISelEmitter::findNodeEquiv(Record *N) const { 497 return NodeEquivs.lookup(N); 498 } 499 500 const CodeGenInstruction * 501 GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode *N) const { 502 if (N->getNumChildren() >= 1) { 503 // setcc operation maps to two different G_* instructions based on the type. 504 if (!Equiv.isValueUnset("IfFloatingPoint") && 505 MVT(N->getChild(0)->getSimpleType(0)).isFloatingPoint()) 506 return &Target.getInstruction(Equiv.getValueAsDef("IfFloatingPoint")); 507 } 508 509 if (!Equiv.isValueUnset("IfConvergent") && 510 N->getIntrinsicInfo(CGP)->isConvergent) 511 return &Target.getInstruction(Equiv.getValueAsDef("IfConvergent")); 512 513 for (const TreePredicateCall &Call : N->getPredicateCalls()) { 514 const TreePredicateFn &Predicate = Call.Fn; 515 if (!Equiv.isValueUnset("IfSignExtend") && 516 (Predicate.isLoad() || Predicate.isAtomic()) && 517 Predicate.isSignExtLoad()) 518 return &Target.getInstruction(Equiv.getValueAsDef("IfSignExtend")); 519 if (!Equiv.isValueUnset("IfZeroExtend") && 520 (Predicate.isLoad() || Predicate.isAtomic()) && 521 Predicate.isZeroExtLoad()) 522 return &Target.getInstruction(Equiv.getValueAsDef("IfZeroExtend")); 523 } 524 525 return &Target.getInstruction(Equiv.getValueAsDef("I")); 526 } 527 528 GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK) 529 : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK), 530 Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) { 531 ClassName = Target.getName().str() + "InstructionSelector"; 532 } 533 534 //===- Emitter ------------------------------------------------------------===// 535 536 Error GlobalISelEmitter::importRulePredicates(RuleMatcher &M, 537 ArrayRef<Record *> Predicates) { 538 for (Record *Pred : Predicates) { 539 if (Pred->getValueAsString("CondString").empty()) 540 continue; 541 declareSubtargetFeature(Pred); 542 M.addRequiredFeature(Pred); 543 } 544 545 return Error::success(); 546 } 547 548 std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate( 549 const TreePredicateFn &Predicate) { 550 std::optional<LLTCodeGen> MemTyOrNone = 551 MVTToLLT(getValueType(Predicate.getMemoryVT())); 552 553 if (!MemTyOrNone) 554 return std::nullopt; 555 556 // Align so unusual types like i1 don't get rounded down. 557 return llvm::alignTo( 558 static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), 8); 559 } 560 561 Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates( 562 const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate, 563 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) { 564 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) { 565 if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) { 566 SmallVector<unsigned, 4> ParsedAddrSpaces; 567 568 for (Init *Val : AddrSpaces->getValues()) { 569 IntInit *IntVal = dyn_cast<IntInit>(Val); 570 if (!IntVal) 571 return failedImport("Address space is not an integer"); 572 ParsedAddrSpaces.push_back(IntVal->getValue()); 573 } 574 575 if (!ParsedAddrSpaces.empty()) { 576 InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>( 577 0, ParsedAddrSpaces); 578 return InsnMatcher; 579 } 580 } 581 582 int64_t MinAlign = Predicate.getMinAlignment(); 583 if (MinAlign > 0) { 584 InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(0, MinAlign); 585 return InsnMatcher; 586 } 587 } 588 589 // G_LOAD is used for both non-extending and any-extending loads. 590 if (Predicate.isLoad() && Predicate.isNonExtLoad()) { 591 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( 592 0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0); 593 return InsnMatcher; 594 } 595 if (Predicate.isLoad() && Predicate.isAnyExtLoad()) { 596 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( 597 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0); 598 return InsnMatcher; 599 } 600 601 if (Predicate.isStore()) { 602 if (Predicate.isTruncStore()) { 603 if (Predicate.getMemoryVT() != nullptr) { 604 // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size. 605 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate); 606 if (!MemSizeInBits) 607 return failedImport("MemVT could not be converted to LLT"); 608 609 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0, *MemSizeInBits / 610 8); 611 } else { 612 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( 613 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0); 614 } 615 return InsnMatcher; 616 } 617 if (Predicate.isNonTruncStore()) { 618 // We need to check the sizes match here otherwise we could incorrectly 619 // match truncating stores with non-truncating ones. 620 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( 621 0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0); 622 } 623 } 624 625 assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value"); 626 // No check required. We already did it by swapping the opcode. 627 if (!SrcGIEquivOrNull->isValueUnset("IfSignExtend") && 628 Predicate.isSignExtLoad()) 629 return InsnMatcher; 630 631 // No check required. We already did it by swapping the opcode. 632 if (!SrcGIEquivOrNull->isValueUnset("IfZeroExtend") && 633 Predicate.isZeroExtLoad()) 634 return InsnMatcher; 635 636 // No check required. G_STORE by itself is a non-extending store. 637 if (Predicate.isNonTruncStore()) 638 return InsnMatcher; 639 640 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) { 641 if (Predicate.getMemoryVT() != nullptr) { 642 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate); 643 if (!MemSizeInBits) 644 return failedImport("MemVT could not be converted to LLT"); 645 646 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0, 647 *MemSizeInBits / 8); 648 return InsnMatcher; 649 } 650 } 651 652 if (Predicate.isLoad() || Predicate.isStore()) { 653 // No check required. A G_LOAD/G_STORE is an unindexed load. 654 if (Predicate.isUnindexed()) 655 return InsnMatcher; 656 } 657 658 if (Predicate.isAtomic()) { 659 if (Predicate.isAtomicOrderingMonotonic()) { 660 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Monotonic"); 661 return InsnMatcher; 662 } 663 if (Predicate.isAtomicOrderingAcquire()) { 664 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Acquire"); 665 return InsnMatcher; 666 } 667 if (Predicate.isAtomicOrderingRelease()) { 668 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Release"); 669 return InsnMatcher; 670 } 671 if (Predicate.isAtomicOrderingAcquireRelease()) { 672 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( 673 "AcquireRelease"); 674 return InsnMatcher; 675 } 676 if (Predicate.isAtomicOrderingSequentiallyConsistent()) { 677 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( 678 "SequentiallyConsistent"); 679 return InsnMatcher; 680 } 681 } 682 683 if (Predicate.isAtomicOrderingAcquireOrStronger()) { 684 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( 685 "Acquire", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); 686 return InsnMatcher; 687 } 688 if (Predicate.isAtomicOrderingWeakerThanAcquire()) { 689 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( 690 "Acquire", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan); 691 return InsnMatcher; 692 } 693 694 if (Predicate.isAtomicOrderingReleaseOrStronger()) { 695 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( 696 "Release", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); 697 return InsnMatcher; 698 } 699 if (Predicate.isAtomicOrderingWeakerThanRelease()) { 700 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( 701 "Release", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan); 702 return InsnMatcher; 703 } 704 HasAddedMatcher = false; 705 return InsnMatcher; 706 } 707 708 Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( 709 RuleMatcher &Rule, InstructionMatcher &InsnMatcher, 710 const TreePatternNode *Src, unsigned &TempOpIdx) { 711 const auto SavedFlags = Rule.setGISelFlags(Src->getGISelFlagsRecord()); 712 713 Record *SrcGIEquivOrNull = nullptr; 714 const CodeGenInstruction *SrcGIOrNull = nullptr; 715 716 // Start with the defined operands (i.e., the results of the root operator). 717 if (Src->isLeaf()) { 718 Init *SrcInit = Src->getLeafValue(); 719 if (isa<IntInit>(SrcInit)) { 720 InsnMatcher.addPredicate<InstructionOpcodeMatcher>( 721 &Target.getInstruction(RK.getDef("G_CONSTANT"))); 722 } else 723 return failedImport( 724 "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); 725 } else { 726 SrcGIEquivOrNull = findNodeEquiv(Src->getOperator()); 727 if (!SrcGIEquivOrNull) 728 return failedImport("Pattern operator lacks an equivalent Instruction" + 729 explainOperator(Src->getOperator())); 730 SrcGIOrNull = getEquivNode(*SrcGIEquivOrNull, Src); 731 732 // The operators look good: match the opcode 733 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull); 734 } 735 736 unsigned OpIdx = 0; 737 for (const TypeSetByHwMode &VTy : Src->getExtTypes()) { 738 // Results don't have a name unless they are the root node. The caller will 739 // set the name if appropriate. 740 const bool OperandIsAPointer = 741 SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(OpIdx); 742 OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); 743 if (auto Error = OM.addTypeCheckPredicate(VTy, OperandIsAPointer)) 744 return failedImport(toString(std::move(Error)) + 745 " for result of Src pattern operator"); 746 } 747 748 for (const TreePredicateCall &Call : Src->getPredicateCalls()) { 749 const TreePredicateFn &Predicate = Call.Fn; 750 bool HasAddedBuiltinMatcher = true; 751 if (Predicate.isAlwaysTrue()) 752 continue; 753 754 if (Predicate.isImmediatePattern()) { 755 InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(Predicate); 756 continue; 757 } 758 759 auto InsnMatcherOrError = addBuiltinPredicates( 760 SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedBuiltinMatcher); 761 if (auto Error = InsnMatcherOrError.takeError()) 762 return std::move(Error); 763 764 // FIXME: This should be part of addBuiltinPredicates(). If we add this at 765 // the start of addBuiltinPredicates() without returning, then there might 766 // be cases where we hit the last return before which the 767 // HasAddedBuiltinMatcher will be set to false. The predicate could be 768 // missed if we add it in the middle or at the end due to return statements 769 // after the addPredicate<>() calls. 770 if (Predicate.hasNoUse()) { 771 InsnMatcher.addPredicate<NoUsePredicateMatcher>(); 772 HasAddedBuiltinMatcher = true; 773 } 774 775 if (Predicate.hasGISelPredicateCode()) { 776 if (Predicate.usesOperands()) { 777 assert(WaitingForNamedOperands == 0 && 778 "previous predicate didn't find all operands or " 779 "nested predicate that uses operands"); 780 TreePattern *TP = Predicate.getOrigPatFragRecord(); 781 WaitingForNamedOperands = TP->getNumArgs(); 782 for (unsigned i = 0; i < WaitingForNamedOperands; ++i) 783 StoreIdxForName[getScopedName(Call.Scope, TP->getArgName(i))] = i; 784 } 785 InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(Predicate); 786 continue; 787 } 788 if (!HasAddedBuiltinMatcher) { 789 return failedImport("Src pattern child has predicate (" + 790 explainPredicates(Src) + ")"); 791 } 792 } 793 794 if (SrcGIEquivOrNull && 795 SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) 796 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic"); 797 else if (SrcGIEquivOrNull && 798 SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { 799 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( 800 "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); 801 } 802 803 if (Src->isLeaf()) { 804 Init *SrcInit = Src->getLeafValue(); 805 if (IntInit *SrcIntInit = dyn_cast<IntInit>(SrcInit)) { 806 OperandMatcher &OM = 807 InsnMatcher.addOperand(OpIdx++, Src->getName(), TempOpIdx); 808 OM.addPredicate<LiteralIntOperandMatcher>(SrcIntInit->getValue()); 809 } else 810 return failedImport( 811 "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); 812 } else { 813 assert(SrcGIOrNull && 814 "Expected to have already found an equivalent Instruction"); 815 if (SrcGIOrNull->TheDef->getName() == "G_CONSTANT" || 816 SrcGIOrNull->TheDef->getName() == "G_FCONSTANT") { 817 // imm/fpimm still have operands but we don't need to do anything with it 818 // here since we don't support ImmLeaf predicates yet. However, we still 819 // need to note the hidden operand to get GIM_CheckNumOperands correct. 820 InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); 821 return InsnMatcher; 822 } 823 824 // Special case because the operand order is changed from setcc. The 825 // predicate operand needs to be swapped from the last operand to the first 826 // source. 827 828 unsigned NumChildren = Src->getNumChildren(); 829 bool IsFCmp = SrcGIOrNull->TheDef->getName() == "G_FCMP"; 830 831 if (IsFCmp || SrcGIOrNull->TheDef->getName() == "G_ICMP") { 832 const TreePatternNode *SrcChild = Src->getChild(NumChildren - 1); 833 if (SrcChild->isLeaf()) { 834 DefInit *DI = dyn_cast<DefInit>(SrcChild->getLeafValue()); 835 Record *CCDef = DI ? DI->getDef() : nullptr; 836 if (!CCDef || !CCDef->isSubClassOf("CondCode")) 837 return failedImport("Unable to handle CondCode"); 838 839 OperandMatcher &OM = 840 InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx); 841 StringRef PredType = IsFCmp ? CCDef->getValueAsString("FCmpPredicate") 842 : CCDef->getValueAsString("ICmpPredicate"); 843 844 if (!PredType.empty()) { 845 OM.addPredicate<CmpPredicateOperandMatcher>(std::string(PredType)); 846 // Process the other 2 operands normally. 847 --NumChildren; 848 } 849 } 850 } 851 852 // Match the used operands (i.e. the children of the operator). 853 bool IsIntrinsic = 854 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" || 855 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" || 856 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_CONVERGENT" || 857 SrcGIOrNull->TheDef->getName() == 858 "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS"; 859 const CodeGenIntrinsic *II = Src->getIntrinsicInfo(CGP); 860 if (IsIntrinsic && !II) 861 return failedImport("Expected IntInit containing intrinsic ID)"); 862 863 for (unsigned i = 0; i != NumChildren; ++i) { 864 const TreePatternNode *SrcChild = Src->getChild(i); 865 866 // We need to determine the meaning of a literal integer based on the 867 // context. If this is a field required to be an immediate (such as an 868 // immarg intrinsic argument), the required predicates are different than 869 // a constant which may be materialized in a register. If we have an 870 // argument that is required to be an immediate, we should not emit an LLT 871 // type check, and should not be looking for a G_CONSTANT defined 872 // register. 873 bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i); 874 875 // SelectionDAG allows pointers to be represented with iN since it doesn't 876 // distinguish between pointers and integers but they are different types 877 // in GlobalISel. Coerce integers to pointers to address space 0 if the 878 // context indicates a pointer. 879 // 880 bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i); 881 882 if (IsIntrinsic) { 883 // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately 884 // following the defs is an intrinsic ID. 885 if (i == 0) { 886 OperandMatcher &OM = 887 InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx); 888 OM.addPredicate<IntrinsicIDOperandMatcher>(II); 889 continue; 890 } 891 892 // We have to check intrinsics for llvm_anyptr_ty and immarg parameters. 893 // 894 // Note that we have to look at the i-1th parameter, because we don't 895 // have the intrinsic ID in the intrinsic's parameter list. 896 OperandIsAPointer |= II->isParamAPointer(i - 1); 897 OperandIsImmArg |= II->isParamImmArg(i - 1); 898 } 899 900 if (auto Error = 901 importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer, 902 OperandIsImmArg, OpIdx++, TempOpIdx)) 903 return std::move(Error); 904 } 905 } 906 907 return InsnMatcher; 908 } 909 910 Error GlobalISelEmitter::importComplexPatternOperandMatcher( 911 OperandMatcher &OM, Record *R, unsigned &TempOpIdx) const { 912 const auto &ComplexPattern = ComplexPatternEquivs.find(R); 913 if (ComplexPattern == ComplexPatternEquivs.end()) 914 return failedImport("SelectionDAG ComplexPattern (" + R->getName() + 915 ") not mapped to GlobalISel"); 916 917 OM.addPredicate<ComplexPatternOperandMatcher>(OM, *ComplexPattern->second); 918 TempOpIdx++; 919 return Error::success(); 920 } 921 922 // Get the name to use for a pattern operand. For an anonymous physical register 923 // input, this should use the register name. 924 static StringRef getSrcChildName(const TreePatternNode *SrcChild, 925 Record *&PhysReg) { 926 StringRef SrcChildName = SrcChild->getName(); 927 if (SrcChildName.empty() && SrcChild->isLeaf()) { 928 if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild->getLeafValue())) { 929 auto *ChildRec = ChildDefInit->getDef(); 930 if (ChildRec->isSubClassOf("Register")) { 931 SrcChildName = ChildRec->getName(); 932 PhysReg = ChildRec; 933 } 934 } 935 } 936 937 return SrcChildName; 938 } 939 940 Error GlobalISelEmitter::importChildMatcher( 941 RuleMatcher &Rule, InstructionMatcher &InsnMatcher, 942 const TreePatternNode *SrcChild, bool OperandIsAPointer, 943 bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) { 944 945 Record *PhysReg = nullptr; 946 std::string SrcChildName = std::string(getSrcChildName(SrcChild, PhysReg)); 947 if (!SrcChild->isLeaf() && 948 SrcChild->getOperator()->isSubClassOf("ComplexPattern")) { 949 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is 950 // "MY_PAT:op1:op2" and the ones with same "name" represent same operand. 951 std::string PatternName = std::string(SrcChild->getOperator()->getName()); 952 for (unsigned i = 0; i < SrcChild->getNumChildren(); ++i) { 953 PatternName += ":"; 954 PatternName += SrcChild->getChild(i)->getName(); 955 } 956 SrcChildName = PatternName; 957 } 958 959 OperandMatcher &OM = 960 PhysReg ? InsnMatcher.addPhysRegInput(PhysReg, OpIdx, TempOpIdx) 961 : InsnMatcher.addOperand(OpIdx, SrcChildName, TempOpIdx); 962 if (OM.isSameAsAnotherOperand()) 963 return Error::success(); 964 965 ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild->getExtTypes(); 966 if (ChildTypes.size() != 1) 967 return failedImport("Src pattern child has multiple results"); 968 969 // Check MBB's before the type check since they are not a known type. 970 if (!SrcChild->isLeaf()) { 971 if (SrcChild->getOperator()->isSubClassOf("SDNode")) { 972 auto &ChildSDNI = CGP.getSDNodeInfo(SrcChild->getOperator()); 973 if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { 974 OM.addPredicate<MBBOperandMatcher>(); 975 return Error::success(); 976 } 977 if (SrcChild->getOperator()->getName() == "timm") { 978 OM.addPredicate<ImmOperandMatcher>(); 979 980 // Add predicates, if any 981 for (const TreePredicateCall &Call : SrcChild->getPredicateCalls()) { 982 const TreePredicateFn &Predicate = Call.Fn; 983 984 // Only handle immediate patterns for now 985 if (Predicate.isImmediatePattern()) { 986 OM.addPredicate<OperandImmPredicateMatcher>(Predicate); 987 } 988 } 989 990 return Error::success(); 991 } 992 } 993 } 994 995 // Immediate arguments have no meaningful type to check as they don't have 996 // registers. 997 if (!OperandIsImmArg) { 998 if (auto Error = 999 OM.addTypeCheckPredicate(ChildTypes.front(), OperandIsAPointer)) 1000 return failedImport(toString(std::move(Error)) + " for Src operand (" + 1001 to_string(*SrcChild) + ")"); 1002 } 1003 1004 // Try look up SrcChild for a (named) predicate operand if there is any. 1005 if (WaitingForNamedOperands) { 1006 auto &ScopedNames = SrcChild->getNamesAsPredicateArg(); 1007 if (!ScopedNames.empty()) { 1008 auto PA = ScopedNames.begin(); 1009 std::string Name = getScopedName(PA->getScope(), PA->getIdentifier()); 1010 OM.addPredicate<RecordNamedOperandMatcher>(StoreIdxForName[Name], Name); 1011 --WaitingForNamedOperands; 1012 } 1013 } 1014 1015 // Check for nested instructions. 1016 if (!SrcChild->isLeaf()) { 1017 if (SrcChild->getOperator()->isSubClassOf("ComplexPattern")) { 1018 // When a ComplexPattern is used as an operator, it should do the same 1019 // thing as when used as a leaf. However, the children of the operator 1020 // name the sub-operands that make up the complex operand and we must 1021 // prepare to reference them in the renderer too. 1022 unsigned RendererID = TempOpIdx; 1023 if (auto Error = importComplexPatternOperandMatcher( 1024 OM, SrcChild->getOperator(), TempOpIdx)) 1025 return Error; 1026 1027 for (unsigned i = 0, e = SrcChild->getNumChildren(); i != e; ++i) { 1028 auto *SubOperand = SrcChild->getChild(i); 1029 if (!SubOperand->getName().empty()) { 1030 if (auto Error = Rule.defineComplexSubOperand( 1031 SubOperand->getName(), SrcChild->getOperator(), RendererID, i, 1032 SrcChildName)) 1033 return Error; 1034 } 1035 } 1036 1037 return Error::success(); 1038 } 1039 1040 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>( 1041 InsnMatcher.getRuleMatcher(), SrcChild->getName()); 1042 if (!MaybeInsnOperand) { 1043 // This isn't strictly true. If the user were to provide exactly the same 1044 // matchers as the original operand then we could allow it. However, it's 1045 // simpler to not permit the redundant specification. 1046 return failedImport( 1047 "Nested instruction cannot be the same as another operand"); 1048 } 1049 1050 // Map the node to a gMIR instruction. 1051 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand; 1052 auto InsnMatcherOrError = createAndImportSelDAGMatcher( 1053 Rule, InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx); 1054 if (auto Error = InsnMatcherOrError.takeError()) 1055 return Error; 1056 1057 return Error::success(); 1058 } 1059 1060 if (SrcChild->hasAnyPredicate()) 1061 return failedImport("Src pattern child has unsupported predicate"); 1062 1063 // Check for constant immediates. 1064 if (auto *ChildInt = dyn_cast<IntInit>(SrcChild->getLeafValue())) { 1065 if (OperandIsImmArg) { 1066 // Checks for argument directly in operand list 1067 OM.addPredicate<LiteralIntOperandMatcher>(ChildInt->getValue()); 1068 } else { 1069 // Checks for materialized constant 1070 OM.addPredicate<ConstantIntOperandMatcher>(ChildInt->getValue()); 1071 } 1072 return Error::success(); 1073 } 1074 1075 // Check for def's like register classes or ComplexPattern's. 1076 if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild->getLeafValue())) { 1077 auto *ChildRec = ChildDefInit->getDef(); 1078 1079 // Check for register classes. 1080 if (ChildRec->isSubClassOf("RegisterClass") || 1081 ChildRec->isSubClassOf("RegisterOperand")) { 1082 OM.addPredicate<RegisterBankOperandMatcher>( 1083 Target.getRegisterClass(getInitValueAsRegClass(ChildDefInit))); 1084 return Error::success(); 1085 } 1086 1087 if (ChildRec->isSubClassOf("Register")) { 1088 // This just be emitted as a copy to the specific register. 1089 ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode(); 1090 const CodeGenRegisterClass *RC = 1091 CGRegs.getMinimalPhysRegClass(ChildRec, &VT); 1092 if (!RC) { 1093 return failedImport( 1094 "Could not determine physical register class of pattern source"); 1095 } 1096 1097 OM.addPredicate<RegisterBankOperandMatcher>(*RC); 1098 return Error::success(); 1099 } 1100 1101 // Check for ValueType. 1102 if (ChildRec->isSubClassOf("ValueType")) { 1103 // We already added a type check as standard practice so this doesn't need 1104 // to do anything. 1105 return Error::success(); 1106 } 1107 1108 // Check for ComplexPattern's. 1109 if (ChildRec->isSubClassOf("ComplexPattern")) 1110 return importComplexPatternOperandMatcher(OM, ChildRec, TempOpIdx); 1111 1112 if (ChildRec->isSubClassOf("ImmLeaf")) { 1113 return failedImport( 1114 "Src pattern child def is an unsupported tablegen class (ImmLeaf)"); 1115 } 1116 1117 // Place holder for SRCVALUE nodes. Nothing to do here. 1118 if (ChildRec->getName() == "srcvalue") 1119 return Error::success(); 1120 1121 const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV"; 1122 if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") { 1123 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>( 1124 InsnMatcher.getRuleMatcher(), SrcChild->getName(), false); 1125 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand; 1126 1127 ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode(); 1128 1129 const CodeGenInstruction &BuildVector = 1130 Target.getInstruction(RK.getDef("G_BUILD_VECTOR")); 1131 const CodeGenInstruction &BuildVectorTrunc = 1132 Target.getInstruction(RK.getDef("G_BUILD_VECTOR_TRUNC")); 1133 1134 // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC 1135 // as an alternative. 1136 InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>( 1137 ArrayRef({&BuildVector, &BuildVectorTrunc})); 1138 1139 // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could 1140 // theoretically not emit any opcode check, but getOpcodeMatcher currently 1141 // has to succeed. 1142 OperandMatcher &OM = 1143 InsnOperand.getInsnMatcher().addOperand(0, "", TempOpIdx); 1144 if (auto Error = 1145 OM.addTypeCheckPredicate(VTy, false /* OperandIsAPointer */)) 1146 return failedImport(toString(std::move(Error)) + 1147 " for result of Src pattern operator"); 1148 1149 InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>( 1150 ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes 1151 : VectorSplatImmPredicateMatcher::AllZeros); 1152 return Error::success(); 1153 } 1154 1155 return failedImport( 1156 "Src pattern child def is an unsupported tablegen class"); 1157 } 1158 1159 return failedImport("Src pattern child is an unsupported kind"); 1160 } 1161 1162 Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer( 1163 action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder, 1164 const TreePatternNode *DstChild, const TreePatternNode *Src) { 1165 1166 const auto &SubOperand = Rule.getComplexSubOperand(DstChild->getName()); 1167 if (SubOperand) { 1168 DstMIBuilder.addRenderer<RenderComplexPatternOperand>( 1169 *std::get<0>(*SubOperand), DstChild->getName(), 1170 std::get<1>(*SubOperand), std::get<2>(*SubOperand)); 1171 return InsertPt; 1172 } 1173 1174 if (!DstChild->isLeaf()) { 1175 if (DstChild->getOperator()->isSubClassOf("SDNodeXForm")) { 1176 auto Child = DstChild->getChild(0); 1177 auto I = SDNodeXFormEquivs.find(DstChild->getOperator()); 1178 if (I != SDNodeXFormEquivs.end()) { 1179 Record *XFormOpc = DstChild->getOperator()->getValueAsDef("Opcode"); 1180 if (XFormOpc->getName() == "timm") { 1181 // If this is a TargetConstant, there won't be a corresponding 1182 // instruction to transform. Instead, this will refer directly to an 1183 // operand in an instruction's operand list. 1184 DstMIBuilder.addRenderer<CustomOperandRenderer>(*I->second, 1185 Child->getName()); 1186 } else { 1187 DstMIBuilder.addRenderer<CustomRenderer>(*I->second, 1188 Child->getName()); 1189 } 1190 1191 return InsertPt; 1192 } 1193 return failedImport("SDNodeXForm " + Child->getName() + 1194 " has no custom renderer"); 1195 } 1196 1197 // We accept 'bb' here. It's an operator because BasicBlockSDNode isn't 1198 // inline, but in MI it's just another operand. 1199 if (DstChild->getOperator()->isSubClassOf("SDNode")) { 1200 auto &ChildSDNI = CGP.getSDNodeInfo(DstChild->getOperator()); 1201 if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { 1202 DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName()); 1203 return InsertPt; 1204 } 1205 } 1206 1207 // Similarly, imm is an operator in TreePatternNode's view but must be 1208 // rendered as operands. 1209 // FIXME: The target should be able to choose sign-extended when appropriate 1210 // (e.g. on Mips). 1211 if (DstChild->getOperator()->getName() == "timm") { 1212 DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName()); 1213 return InsertPt; 1214 } else if (DstChild->getOperator()->getName() == "imm") { 1215 DstMIBuilder.addRenderer<CopyConstantAsImmRenderer>(DstChild->getName()); 1216 return InsertPt; 1217 } else if (DstChild->getOperator()->getName() == "fpimm") { 1218 DstMIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>( 1219 DstChild->getName()); 1220 return InsertPt; 1221 } 1222 1223 if (DstChild->getOperator()->isSubClassOf("Instruction")) { 1224 auto OpTy = getInstResultType(DstChild); 1225 if (!OpTy) 1226 return OpTy.takeError(); 1227 1228 unsigned TempRegID = Rule.allocateTempRegID(); 1229 InsertPt = 1230 Rule.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID); 1231 DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID); 1232 1233 auto InsertPtOrError = createAndImportSubInstructionRenderer( 1234 ++InsertPt, Rule, DstChild, Src, TempRegID); 1235 if (auto Error = InsertPtOrError.takeError()) 1236 return std::move(Error); 1237 return InsertPtOrError.get(); 1238 } 1239 1240 return failedImport("Dst pattern child isn't a leaf node or an MBB" + 1241 llvm::to_string(*DstChild)); 1242 } 1243 1244 // It could be a specific immediate in which case we should just check for 1245 // that immediate. 1246 if (const IntInit *ChildIntInit = 1247 dyn_cast<IntInit>(DstChild->getLeafValue())) { 1248 DstMIBuilder.addRenderer<ImmRenderer>(ChildIntInit->getValue()); 1249 return InsertPt; 1250 } 1251 1252 // Otherwise, we're looking for a bog-standard RegisterClass operand. 1253 if (auto *ChildDefInit = dyn_cast<DefInit>(DstChild->getLeafValue())) { 1254 auto *ChildRec = ChildDefInit->getDef(); 1255 1256 ArrayRef<TypeSetByHwMode> ChildTypes = DstChild->getExtTypes(); 1257 if (ChildTypes.size() != 1) 1258 return failedImport("Dst pattern child has multiple results"); 1259 1260 std::optional<LLTCodeGen> OpTyOrNone; 1261 if (ChildTypes.front().isMachineValueType()) 1262 OpTyOrNone = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy); 1263 if (!OpTyOrNone) 1264 return failedImport("Dst operand has an unsupported type"); 1265 1266 if (ChildRec->isSubClassOf("Register")) { 1267 DstMIBuilder.addRenderer<AddRegisterRenderer>(Target, ChildRec); 1268 return InsertPt; 1269 } 1270 1271 if (ChildRec->isSubClassOf("RegisterClass") || 1272 ChildRec->isSubClassOf("RegisterOperand") || 1273 ChildRec->isSubClassOf("ValueType")) { 1274 if (ChildRec->isSubClassOf("RegisterOperand") && 1275 !ChildRec->isValueUnset("GIZeroRegister")) { 1276 DstMIBuilder.addRenderer<CopyOrAddZeroRegRenderer>( 1277 DstChild->getName(), ChildRec->getValueAsDef("GIZeroRegister")); 1278 return InsertPt; 1279 } 1280 1281 DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName()); 1282 return InsertPt; 1283 } 1284 1285 if (ChildRec->isSubClassOf("SubRegIndex")) { 1286 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(ChildRec); 1287 DstMIBuilder.addRenderer<ImmRenderer>(SubIdx->EnumValue); 1288 return InsertPt; 1289 } 1290 1291 if (ChildRec->isSubClassOf("ComplexPattern")) { 1292 const auto &ComplexPattern = ComplexPatternEquivs.find(ChildRec); 1293 if (ComplexPattern == ComplexPatternEquivs.end()) 1294 return failedImport( 1295 "SelectionDAG ComplexPattern not mapped to GlobalISel"); 1296 1297 const OperandMatcher &OM = Rule.getOperandMatcher(DstChild->getName()); 1298 DstMIBuilder.addRenderer<RenderComplexPatternOperand>( 1299 *ComplexPattern->second, DstChild->getName(), 1300 OM.getAllocatedTemporariesBaseID()); 1301 return InsertPt; 1302 } 1303 1304 return failedImport( 1305 "Dst pattern child def is an unsupported tablegen class"); 1306 } 1307 1308 // Handle the case where the MVT/register class is omitted in the dest pattern 1309 // but MVT exists in the source pattern. 1310 if (isa<UnsetInit>(DstChild->getLeafValue())) { 1311 for (unsigned NumOp = 0; NumOp < Src->getNumChildren(); NumOp++) 1312 if (Src->getChild(NumOp)->getName() == DstChild->getName()) { 1313 DstMIBuilder.addRenderer<CopyRenderer>(Src->getChild(NumOp)->getName()); 1314 return InsertPt; 1315 } 1316 } 1317 return failedImport("Dst pattern child is an unsupported kind"); 1318 } 1319 1320 Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer( 1321 RuleMatcher &M, InstructionMatcher &InsnMatcher, const TreePatternNode *Src, 1322 const TreePatternNode *Dst) { 1323 auto InsertPtOrError = createInstructionRenderer(M.actions_end(), M, Dst); 1324 if (auto Error = InsertPtOrError.takeError()) 1325 return std::move(Error); 1326 1327 action_iterator InsertPt = InsertPtOrError.get(); 1328 BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get()); 1329 1330 for (auto PhysInput : InsnMatcher.getPhysRegInputs()) { 1331 InsertPt = M.insertAction<BuildMIAction>( 1332 InsertPt, M.allocateOutputInsnID(), 1333 &Target.getInstruction(RK.getDef("COPY"))); 1334 BuildMIAction &CopyToPhysRegMIBuilder = 1335 *static_cast<BuildMIAction *>(InsertPt->get()); 1336 CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>( 1337 Target, PhysInput.first, true); 1338 CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysInput.first); 1339 } 1340 1341 if (auto Error = 1342 importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Src, Dst) 1343 .takeError()) 1344 return std::move(Error); 1345 1346 if (auto Error = 1347 importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst, Src) 1348 .takeError()) 1349 return std::move(Error); 1350 1351 return DstMIBuilder; 1352 } 1353 1354 Expected<action_iterator> 1355 GlobalISelEmitter::createAndImportSubInstructionRenderer( 1356 const action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst, 1357 const TreePatternNode *Src, unsigned TempRegID) { 1358 auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst); 1359 1360 // TODO: Assert there's exactly one result. 1361 1362 if (auto Error = InsertPtOrError.takeError()) 1363 return std::move(Error); 1364 1365 BuildMIAction &DstMIBuilder = 1366 *static_cast<BuildMIAction *>(InsertPtOrError.get()->get()); 1367 1368 // Assign the result to TempReg. 1369 DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true); 1370 1371 InsertPtOrError = importExplicitUseRenderers(InsertPtOrError.get(), M, 1372 DstMIBuilder, Dst, Src); 1373 if (auto Error = InsertPtOrError.takeError()) 1374 return std::move(Error); 1375 1376 // We need to make sure that when we import an INSERT_SUBREG as a 1377 // subinstruction that it ends up being constrained to the correct super 1378 // register and subregister classes. 1379 auto OpName = Target.getInstruction(Dst->getOperator()).TheDef->getName(); 1380 if (OpName == "INSERT_SUBREG") { 1381 auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); 1382 if (!SubClass) 1383 return failedImport( 1384 "Cannot infer register class from INSERT_SUBREG operand #1"); 1385 std::optional<const CodeGenRegisterClass *> SuperClass = 1386 inferSuperRegisterClassForNode(Dst->getExtType(0), Dst->getChild(0), 1387 Dst->getChild(2)); 1388 if (!SuperClass) 1389 return failedImport( 1390 "Cannot infer register class for INSERT_SUBREG operand #0"); 1391 // The destination and the super register source of an INSERT_SUBREG must 1392 // be the same register class. 1393 M.insertAction<ConstrainOperandToRegClassAction>( 1394 InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); 1395 M.insertAction<ConstrainOperandToRegClassAction>( 1396 InsertPt, DstMIBuilder.getInsnID(), 1, **SuperClass); 1397 M.insertAction<ConstrainOperandToRegClassAction>( 1398 InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass); 1399 return InsertPtOrError.get(); 1400 } 1401 1402 if (OpName == "EXTRACT_SUBREG") { 1403 // EXTRACT_SUBREG selects into a subregister COPY but unlike most 1404 // instructions, the result register class is controlled by the 1405 // subregisters of the operand. As a result, we must constrain the result 1406 // class rather than check that it's already the right one. 1407 auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); 1408 if (!SuperClass) 1409 return failedImport( 1410 "Cannot infer register class from EXTRACT_SUBREG operand #0"); 1411 1412 auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1)); 1413 if (!SubIdx) 1414 return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); 1415 1416 const auto SrcRCDstRCPair = 1417 (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); 1418 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); 1419 M.insertAction<ConstrainOperandToRegClassAction>( 1420 InsertPt, DstMIBuilder.getInsnID(), 0, *SrcRCDstRCPair->second); 1421 M.insertAction<ConstrainOperandToRegClassAction>( 1422 InsertPt, DstMIBuilder.getInsnID(), 1, *SrcRCDstRCPair->first); 1423 1424 // We're done with this pattern! It's eligible for GISel emission; return 1425 // it. 1426 return InsertPtOrError.get(); 1427 } 1428 1429 // Similar to INSERT_SUBREG, we also have to handle SUBREG_TO_REG as a 1430 // subinstruction. 1431 if (OpName == "SUBREG_TO_REG") { 1432 auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); 1433 if (!SubClass) 1434 return failedImport( 1435 "Cannot infer register class from SUBREG_TO_REG child #1"); 1436 auto SuperClass = 1437 inferSuperRegisterClass(Dst->getExtType(0), Dst->getChild(2)); 1438 if (!SuperClass) 1439 return failedImport( 1440 "Cannot infer register class for SUBREG_TO_REG operand #0"); 1441 M.insertAction<ConstrainOperandToRegClassAction>( 1442 InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); 1443 M.insertAction<ConstrainOperandToRegClassAction>( 1444 InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass); 1445 return InsertPtOrError.get(); 1446 } 1447 1448 if (OpName == "REG_SEQUENCE") { 1449 auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); 1450 M.insertAction<ConstrainOperandToRegClassAction>( 1451 InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); 1452 1453 unsigned Num = Dst->getNumChildren(); 1454 for (unsigned I = 1; I != Num; I += 2) { 1455 const TreePatternNode *SubRegChild = Dst->getChild(I + 1); 1456 1457 auto SubIdx = inferSubRegIndexForNode(SubRegChild); 1458 if (!SubIdx) 1459 return failedImport("REG_SEQUENCE child is not a subreg index"); 1460 1461 const auto SrcRCDstRCPair = 1462 (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); 1463 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); 1464 M.insertAction<ConstrainOperandToRegClassAction>( 1465 InsertPt, DstMIBuilder.getInsnID(), I, *SrcRCDstRCPair->second); 1466 } 1467 1468 return InsertPtOrError.get(); 1469 } 1470 1471 M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, 1472 DstMIBuilder.getInsnID()); 1473 return InsertPtOrError.get(); 1474 } 1475 1476 Expected<action_iterator> GlobalISelEmitter::createInstructionRenderer( 1477 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst) { 1478 Record *DstOp = Dst->getOperator(); 1479 if (!DstOp->isSubClassOf("Instruction")) { 1480 if (DstOp->isSubClassOf("ValueType")) 1481 return failedImport( 1482 "Pattern operator isn't an instruction (it's a ValueType)"); 1483 return failedImport("Pattern operator isn't an instruction"); 1484 } 1485 CodeGenInstruction *DstI = &Target.getInstruction(DstOp); 1486 1487 // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction 1488 // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy. 1489 StringRef Name = DstI->TheDef->getName(); 1490 if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG") 1491 DstI = &Target.getInstruction(RK.getDef("COPY")); 1492 1493 return M.insertAction<BuildMIAction>(InsertPt, M.allocateOutputInsnID(), 1494 DstI); 1495 } 1496 1497 Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers( 1498 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, 1499 const TreePatternNode *Src, const TreePatternNode *Dst) { 1500 const CodeGenInstruction *DstI = DstMIBuilder.getCGI(); 1501 const unsigned SrcNumDefs = Src->getExtTypes().size(); 1502 const unsigned DstNumDefs = DstI->Operands.NumDefs; 1503 if (DstNumDefs == 0) 1504 return InsertPt; 1505 1506 for (unsigned I = 0; I < SrcNumDefs; ++I) { 1507 std::string OpName = getMangledRootDefName(DstI->Operands[I].Name); 1508 // CopyRenderer saves a StringRef, so cannot pass OpName itself - 1509 // let's use a string with an appropriate lifetime. 1510 StringRef PermanentRef = M.getOperandMatcher(OpName).getSymbolicName(); 1511 DstMIBuilder.addRenderer<CopyRenderer>(PermanentRef); 1512 } 1513 1514 // Some instructions have multiple defs, but are missing a type entry 1515 // (e.g. s_cc_out operands). 1516 if (Dst->getExtTypes().size() < DstNumDefs) 1517 return failedImport("unhandled discarded def"); 1518 1519 for (unsigned I = SrcNumDefs; I < DstNumDefs; ++I) { 1520 const TypeSetByHwMode &ExtTy = Dst->getExtType(I); 1521 if (!ExtTy.isMachineValueType()) 1522 return failedImport("unsupported typeset"); 1523 1524 auto OpTy = MVTToLLT(ExtTy.getMachineValueType().SimpleTy); 1525 if (!OpTy) 1526 return failedImport("unsupported type"); 1527 1528 unsigned TempRegID = M.allocateTempRegID(); 1529 InsertPt = 1530 M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID); 1531 DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true, nullptr, true); 1532 } 1533 1534 return InsertPt; 1535 } 1536 1537 Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers( 1538 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, 1539 const llvm::TreePatternNode *Dst, const llvm::TreePatternNode *Src) { 1540 const CodeGenInstruction *DstI = DstMIBuilder.getCGI(); 1541 CodeGenInstruction *OrigDstI = &Target.getInstruction(Dst->getOperator()); 1542 1543 StringRef Name = OrigDstI->TheDef->getName(); 1544 unsigned ExpectedDstINumUses = Dst->getNumChildren(); 1545 1546 // EXTRACT_SUBREG needs to use a subregister COPY. 1547 if (Name == "EXTRACT_SUBREG") { 1548 if (!Dst->getChild(1)->isLeaf()) 1549 return failedImport("EXTRACT_SUBREG child #1 is not a leaf"); 1550 DefInit *SubRegInit = dyn_cast<DefInit>(Dst->getChild(1)->getLeafValue()); 1551 if (!SubRegInit) 1552 return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); 1553 1554 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); 1555 const TreePatternNode *ValChild = Dst->getChild(0); 1556 if (!ValChild->isLeaf()) { 1557 // We really have to handle the source instruction, and then insert a 1558 // copy from the subregister. 1559 auto ExtractSrcTy = getInstResultType(ValChild); 1560 if (!ExtractSrcTy) 1561 return ExtractSrcTy.takeError(); 1562 1563 unsigned TempRegID = M.allocateTempRegID(); 1564 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, *ExtractSrcTy, 1565 TempRegID); 1566 1567 auto InsertPtOrError = createAndImportSubInstructionRenderer( 1568 ++InsertPt, M, ValChild, Src, TempRegID); 1569 if (auto Error = InsertPtOrError.takeError()) 1570 return std::move(Error); 1571 1572 DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, false, SubIdx); 1573 return InsertPt; 1574 } 1575 1576 // If this is a source operand, this is just a subregister copy. 1577 Record *RCDef = getInitValueAsRegClass(ValChild->getLeafValue()); 1578 if (!RCDef) 1579 return failedImport("EXTRACT_SUBREG child #0 could not " 1580 "be coerced to a register class"); 1581 1582 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef); 1583 1584 const auto SrcRCDstRCPair = 1585 RC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx); 1586 if (SrcRCDstRCPair) { 1587 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); 1588 if (SrcRCDstRCPair->first != RC) 1589 return failedImport("EXTRACT_SUBREG requires an additional COPY"); 1590 } 1591 1592 StringRef RegOperandName = Dst->getChild(0)->getName(); 1593 if (const auto &SubOperand = M.getComplexSubOperand(RegOperandName)) { 1594 DstMIBuilder.addRenderer<RenderComplexPatternOperand>( 1595 *std::get<0>(*SubOperand), RegOperandName, std::get<1>(*SubOperand), 1596 std::get<2>(*SubOperand), SubIdx); 1597 return InsertPt; 1598 } 1599 1600 DstMIBuilder.addRenderer<CopySubRegRenderer>(RegOperandName, SubIdx); 1601 return InsertPt; 1602 } 1603 1604 if (Name == "REG_SEQUENCE") { 1605 if (!Dst->getChild(0)->isLeaf()) 1606 return failedImport("REG_SEQUENCE child #0 is not a leaf"); 1607 1608 Record *RCDef = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); 1609 if (!RCDef) 1610 return failedImport("REG_SEQUENCE child #0 could not " 1611 "be coerced to a register class"); 1612 1613 if ((ExpectedDstINumUses - 1) % 2 != 0) 1614 return failedImport("Malformed REG_SEQUENCE"); 1615 1616 for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) { 1617 const TreePatternNode *ValChild = Dst->getChild(I); 1618 const TreePatternNode *SubRegChild = Dst->getChild(I + 1); 1619 1620 if (DefInit *SubRegInit = 1621 dyn_cast<DefInit>(SubRegChild->getLeafValue())) { 1622 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); 1623 1624 auto InsertPtOrError = 1625 importExplicitUseRenderer(InsertPt, M, DstMIBuilder, ValChild, Src); 1626 if (auto Error = InsertPtOrError.takeError()) 1627 return std::move(Error); 1628 InsertPt = InsertPtOrError.get(); 1629 DstMIBuilder.addRenderer<SubRegIndexRenderer>(SubIdx); 1630 } 1631 } 1632 1633 return InsertPt; 1634 } 1635 1636 // Render the explicit uses. 1637 unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs; 1638 if (Name == "COPY_TO_REGCLASS") { 1639 DstINumUses--; // Ignore the class constraint. 1640 ExpectedDstINumUses--; 1641 } 1642 1643 // NumResults - This is the number of results produced by the instruction in 1644 // the "outs" list. 1645 unsigned NumResults = OrigDstI->Operands.NumDefs; 1646 1647 // Number of operands we know the output instruction must have. If it is 1648 // variadic, we could have more operands. 1649 unsigned NumFixedOperands = DstI->Operands.size(); 1650 1651 // Loop over all of the fixed operands of the instruction pattern, emitting 1652 // code to fill them all in. The node 'N' usually has number children equal to 1653 // the number of input operands of the instruction. However, in cases where 1654 // there are predicate operands for an instruction, we need to fill in the 1655 // 'execute always' values. Match up the node operands to the instruction 1656 // operands to do this. 1657 unsigned Child = 0; 1658 1659 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the 1660 // number of operands at the end of the list which have default values. 1661 // Those can come from the pattern if it provides enough arguments, or be 1662 // filled in with the default if the pattern hasn't provided them. But any 1663 // operand with a default value _before_ the last mandatory one will be 1664 // filled in with their defaults unconditionally. 1665 unsigned NonOverridableOperands = NumFixedOperands; 1666 while (NonOverridableOperands > NumResults && 1667 CGP.operandHasDefault(DstI->Operands[NonOverridableOperands - 1].Rec)) 1668 --NonOverridableOperands; 1669 1670 unsigned NumDefaultOps = 0; 1671 for (unsigned I = 0; I != DstINumUses; ++I) { 1672 unsigned InstOpNo = DstI->Operands.NumDefs + I; 1673 1674 // Determine what to emit for this operand. 1675 Record *OperandNode = DstI->Operands[InstOpNo].Rec; 1676 1677 // If the operand has default values, introduce them now. 1678 if (CGP.operandHasDefault(OperandNode) && 1679 (InstOpNo < NonOverridableOperands || Child >= Dst->getNumChildren())) { 1680 // This is a predicate or optional def operand which the pattern has not 1681 // overridden, or which we aren't letting it override; emit the 'default 1682 // ops' operands. 1683 1684 Record *OperandNode = DstI->Operands[InstOpNo].Rec; 1685 if (auto Error = importDefaultOperandRenderers( 1686 InsertPt, M, DstMIBuilder, CGP.getDefaultOperand(OperandNode))) 1687 return std::move(Error); 1688 1689 ++NumDefaultOps; 1690 continue; 1691 } 1692 1693 auto InsertPtOrError = importExplicitUseRenderer(InsertPt, M, DstMIBuilder, 1694 Dst->getChild(Child), Src); 1695 if (auto Error = InsertPtOrError.takeError()) 1696 return std::move(Error); 1697 InsertPt = InsertPtOrError.get(); 1698 ++Child; 1699 } 1700 1701 if (NumDefaultOps + ExpectedDstINumUses != DstINumUses) 1702 return failedImport("Expected " + llvm::to_string(DstINumUses) + 1703 " used operands but found " + 1704 llvm::to_string(ExpectedDstINumUses) + 1705 " explicit ones and " + llvm::to_string(NumDefaultOps) + 1706 " default ones"); 1707 1708 return InsertPt; 1709 } 1710 1711 Error GlobalISelEmitter::importDefaultOperandRenderers( 1712 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, 1713 const DAGDefaultOperand &DefaultOp) const { 1714 for (const auto &Op : DefaultOp.DefaultOps) { 1715 const auto *N = Op.get(); 1716 if (!N->isLeaf()) 1717 return failedImport("Could not add default op"); 1718 1719 const auto *DefaultOp = N->getLeafValue(); 1720 1721 if (const DefInit *DefaultDefOp = dyn_cast<DefInit>(DefaultOp)) { 1722 std::optional<LLTCodeGen> OpTyOrNone = MVTToLLT(N->getSimpleType(0)); 1723 auto Def = DefaultDefOp->getDef(); 1724 if (Def->getName() == "undef_tied_input") { 1725 unsigned TempRegID = M.allocateTempRegID(); 1726 M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTyOrNone, 1727 TempRegID); 1728 InsertPt = M.insertAction<BuildMIAction>( 1729 InsertPt, M.allocateOutputInsnID(), 1730 &Target.getInstruction(RK.getDef("IMPLICIT_DEF"))); 1731 BuildMIAction &IDMIBuilder = 1732 *static_cast<BuildMIAction *>(InsertPt->get()); 1733 IDMIBuilder.addRenderer<TempRegRenderer>(TempRegID); 1734 DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID); 1735 } else { 1736 DstMIBuilder.addRenderer<AddRegisterRenderer>(Target, Def); 1737 } 1738 continue; 1739 } 1740 1741 if (const IntInit *DefaultIntOp = dyn_cast<IntInit>(DefaultOp)) { 1742 DstMIBuilder.addRenderer<ImmRenderer>(DefaultIntOp->getValue()); 1743 continue; 1744 } 1745 1746 return failedImport("Could not add default op"); 1747 } 1748 1749 return Error::success(); 1750 } 1751 1752 Error GlobalISelEmitter::importImplicitDefRenderers( 1753 BuildMIAction &DstMIBuilder, 1754 const std::vector<Record *> &ImplicitDefs) const { 1755 if (!ImplicitDefs.empty()) 1756 return failedImport("Pattern defines a physical register"); 1757 return Error::success(); 1758 } 1759 1760 std::optional<const CodeGenRegisterClass *> 1761 GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode *Leaf) { 1762 assert(Leaf && "Expected node?"); 1763 assert(Leaf->isLeaf() && "Expected leaf?"); 1764 Record *RCRec = getInitValueAsRegClass(Leaf->getLeafValue()); 1765 if (!RCRec) 1766 return std::nullopt; 1767 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCRec); 1768 if (!RC) 1769 return std::nullopt; 1770 return RC; 1771 } 1772 1773 std::optional<const CodeGenRegisterClass *> 1774 GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode *N) { 1775 if (!N) 1776 return std::nullopt; 1777 1778 if (N->isLeaf()) 1779 return getRegClassFromLeaf(N); 1780 1781 // We don't have a leaf node, so we have to try and infer something. Check 1782 // that we have an instruction that we an infer something from. 1783 1784 // Only handle things that produce a single type. 1785 if (N->getNumTypes() != 1) 1786 return std::nullopt; 1787 Record *OpRec = N->getOperator(); 1788 1789 // We only want instructions. 1790 if (!OpRec->isSubClassOf("Instruction")) 1791 return std::nullopt; 1792 1793 // Don't want to try and infer things when there could potentially be more 1794 // than one candidate register class. 1795 auto &Inst = Target.getInstruction(OpRec); 1796 if (Inst.Operands.NumDefs > 1) 1797 return std::nullopt; 1798 1799 // Handle any special-case instructions which we can safely infer register 1800 // classes from. 1801 StringRef InstName = Inst.TheDef->getName(); 1802 bool IsRegSequence = InstName == "REG_SEQUENCE"; 1803 if (IsRegSequence || InstName == "COPY_TO_REGCLASS") { 1804 // If we have a COPY_TO_REGCLASS, then we need to handle it specially. It 1805 // has the desired register class as the first child. 1806 const TreePatternNode *RCChild = N->getChild(IsRegSequence ? 0 : 1); 1807 if (!RCChild->isLeaf()) 1808 return std::nullopt; 1809 return getRegClassFromLeaf(RCChild); 1810 } 1811 if (InstName == "INSERT_SUBREG") { 1812 const TreePatternNode *Child0 = N->getChild(0); 1813 assert(Child0->getNumTypes() == 1 && "Unexpected number of types!"); 1814 const TypeSetByHwMode &VTy = Child0->getExtType(0); 1815 return inferSuperRegisterClassForNode(VTy, Child0, N->getChild(2)); 1816 } 1817 if (InstName == "EXTRACT_SUBREG") { 1818 assert(N->getNumTypes() == 1 && "Unexpected number of types!"); 1819 const TypeSetByHwMode &VTy = N->getExtType(0); 1820 return inferSuperRegisterClass(VTy, N->getChild(1)); 1821 } 1822 1823 // Handle destination record types that we can safely infer a register class 1824 // from. 1825 const auto &DstIOperand = Inst.Operands[0]; 1826 Record *DstIOpRec = DstIOperand.Rec; 1827 if (DstIOpRec->isSubClassOf("RegisterOperand")) { 1828 DstIOpRec = DstIOpRec->getValueAsDef("RegClass"); 1829 const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec); 1830 return &RC; 1831 } 1832 1833 if (DstIOpRec->isSubClassOf("RegisterClass")) { 1834 const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec); 1835 return &RC; 1836 } 1837 1838 return std::nullopt; 1839 } 1840 1841 std::optional<const CodeGenRegisterClass *> 1842 GlobalISelEmitter::inferSuperRegisterClass( 1843 const TypeSetByHwMode &Ty, const TreePatternNode *SubRegIdxNode) { 1844 assert(SubRegIdxNode && "Expected subregister index node!"); 1845 // We need a ValueTypeByHwMode for getSuperRegForSubReg. 1846 if (!Ty.isValueTypeByHwMode(false)) 1847 return std::nullopt; 1848 if (!SubRegIdxNode->isLeaf()) 1849 return std::nullopt; 1850 DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode->getLeafValue()); 1851 if (!SubRegInit) 1852 return std::nullopt; 1853 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); 1854 1855 // Use the information we found above to find a minimal register class which 1856 // supports the subregister and type we want. 1857 auto RC = 1858 Target.getSuperRegForSubReg(Ty.getValueTypeByHwMode(), CGRegs, SubIdx, 1859 /* MustBeAllocatable */ true); 1860 if (!RC) 1861 return std::nullopt; 1862 return *RC; 1863 } 1864 1865 std::optional<const CodeGenRegisterClass *> 1866 GlobalISelEmitter::inferSuperRegisterClassForNode( 1867 const TypeSetByHwMode &Ty, const TreePatternNode *SuperRegNode, 1868 const TreePatternNode *SubRegIdxNode) { 1869 assert(SuperRegNode && "Expected super register node!"); 1870 // Check if we already have a defined register class for the super register 1871 // node. If we do, then we should preserve that rather than inferring anything 1872 // from the subregister index node. We can assume that whoever wrote the 1873 // pattern in the first place made sure that the super register and 1874 // subregister are compatible. 1875 if (std::optional<const CodeGenRegisterClass *> SuperRegisterClass = 1876 inferRegClassFromPattern(SuperRegNode)) 1877 return *SuperRegisterClass; 1878 return inferSuperRegisterClass(Ty, SubRegIdxNode); 1879 } 1880 1881 std::optional<CodeGenSubRegIndex *> GlobalISelEmitter::inferSubRegIndexForNode( 1882 const TreePatternNode *SubRegIdxNode) { 1883 if (!SubRegIdxNode->isLeaf()) 1884 return std::nullopt; 1885 1886 DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode->getLeafValue()); 1887 if (!SubRegInit) 1888 return std::nullopt; 1889 return CGRegs.getSubRegIdx(SubRegInit->getDef()); 1890 } 1891 1892 Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { 1893 // Keep track of the matchers and actions to emit. 1894 int Score = P.getPatternComplexity(CGP); 1895 RuleMatcher M(P.getSrcRecord()->getLoc()); 1896 RuleMatcherScores[M.getRuleID()] = Score; 1897 M.addAction<DebugCommentAction>(llvm::to_string(*P.getSrcPattern()) + 1898 " => " + 1899 llvm::to_string(*P.getDstPattern())); 1900 1901 SmallVector<Record *, 4> Predicates; 1902 P.getPredicateRecords(Predicates); 1903 if (auto Error = importRulePredicates(M, Predicates)) 1904 return std::move(Error); 1905 1906 if (!P.getHwModeFeatures().empty()) 1907 M.addHwModeIdx(declareHwModeCheck(P.getHwModeFeatures())); 1908 1909 // Next, analyze the pattern operators. 1910 TreePatternNode *Src = P.getSrcPattern(); 1911 TreePatternNode *Dst = P.getDstPattern(); 1912 1913 // If the root of either pattern isn't a simple operator, ignore it. 1914 if (auto Err = isTrivialOperatorNode(Dst)) 1915 return failedImport("Dst pattern root isn't a trivial operator (" + 1916 toString(std::move(Err)) + ")"); 1917 if (auto Err = isTrivialOperatorNode(Src)) 1918 return failedImport("Src pattern root isn't a trivial operator (" + 1919 toString(std::move(Err)) + ")"); 1920 1921 // The different predicates and matchers created during 1922 // addInstructionMatcher use the RuleMatcher M to set up their 1923 // instruction ID (InsnVarID) that are going to be used when 1924 // M is going to be emitted. 1925 // However, the code doing the emission still relies on the IDs 1926 // returned during that process by the RuleMatcher when issuing 1927 // the recordInsn opcodes. 1928 // Because of that: 1929 // 1. The order in which we created the predicates 1930 // and such must be the same as the order in which we emit them, 1931 // and 1932 // 2. We need to reset the generation of the IDs in M somewhere between 1933 // addInstructionMatcher and emit 1934 // 1935 // FIXME: Long term, we don't want to have to rely on this implicit 1936 // naming being the same. One possible solution would be to have 1937 // explicit operator for operation capture and reference those. 1938 // The plus side is that it would expose opportunities to share 1939 // the capture accross rules. The downside is that it would 1940 // introduce a dependency between predicates (captures must happen 1941 // before their first use.) 1942 InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src->getName()); 1943 unsigned TempOpIdx = 0; 1944 1945 const auto SavedFlags = M.setGISelFlags(P.getSrcRecord()); 1946 1947 auto InsnMatcherOrError = 1948 createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx); 1949 if (auto Error = InsnMatcherOrError.takeError()) 1950 return std::move(Error); 1951 InstructionMatcher &InsnMatcher = InsnMatcherOrError.get(); 1952 1953 if (Dst->isLeaf()) { 1954 Record *RCDef = getInitValueAsRegClass(Dst->getLeafValue()); 1955 if (RCDef) { 1956 const CodeGenRegisterClass &RC = Target.getRegisterClass(RCDef); 1957 1958 // We need to replace the def and all its uses with the specified 1959 // operand. However, we must also insert COPY's wherever needed. 1960 // For now, emit a copy and let the register allocator clean up. 1961 auto &DstI = Target.getInstruction(RK.getDef("COPY")); 1962 const auto &DstIOperand = DstI.Operands[0]; 1963 1964 OperandMatcher &OM0 = InsnMatcher.getOperand(0); 1965 OM0.setSymbolicName(DstIOperand.Name); 1966 M.defineOperand(OM0.getSymbolicName(), OM0); 1967 OM0.addPredicate<RegisterBankOperandMatcher>(RC); 1968 1969 auto &DstMIBuilder = 1970 M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &DstI); 1971 DstMIBuilder.addRenderer<CopyRenderer>(DstIOperand.Name); 1972 DstMIBuilder.addRenderer<CopyRenderer>(Dst->getName()); 1973 M.addAction<ConstrainOperandToRegClassAction>(0, 0, RC); 1974 1975 // Erase the root. 1976 unsigned RootInsnID = M.getInsnVarID(InsnMatcher); 1977 M.addAction<EraseInstAction>(RootInsnID); 1978 1979 // We're done with this pattern! It's eligible for GISel emission; return 1980 // it. 1981 ++NumPatternImported; 1982 return std::move(M); 1983 } 1984 1985 return failedImport("Dst pattern root isn't a known leaf"); 1986 } 1987 1988 // Start with the defined operands (i.e., the results of the root operator). 1989 Record *DstOp = Dst->getOperator(); 1990 if (!DstOp->isSubClassOf("Instruction")) 1991 return failedImport("Pattern operator isn't an instruction"); 1992 1993 auto &DstI = Target.getInstruction(DstOp); 1994 StringRef DstIName = DstI.TheDef->getName(); 1995 1996 unsigned DstNumDefs = DstI.Operands.NumDefs, 1997 SrcNumDefs = Src->getExtTypes().size(); 1998 if (DstNumDefs < SrcNumDefs) { 1999 if (DstNumDefs != 0) 2000 return failedImport("Src pattern result has more defs than dst MI (" + 2001 to_string(SrcNumDefs) + " def(s) vs " + 2002 to_string(DstNumDefs) + " def(s))"); 2003 2004 bool FoundNoUsePred = false; 2005 for (const auto &Pred : InsnMatcher.predicates()) { 2006 if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Pred.get()))) 2007 break; 2008 } 2009 if (!FoundNoUsePred) 2010 return failedImport("Src pattern result has " + to_string(SrcNumDefs) + 2011 " def(s) without the HasNoUse predicate set to true " 2012 "but Dst MI has no def"); 2013 } 2014 2015 // The root of the match also has constraints on the register bank so that it 2016 // matches the result instruction. 2017 unsigned OpIdx = 0; 2018 unsigned N = std::min(DstNumDefs, SrcNumDefs); 2019 for (unsigned I = 0; I < N; ++I) { 2020 const TypeSetByHwMode &VTy = Src->getExtType(I); 2021 2022 const auto &DstIOperand = DstI.Operands[OpIdx]; 2023 PointerUnion<Record *, const CodeGenRegisterClass *> MatchedRC = 2024 DstIOperand.Rec; 2025 if (DstIName == "COPY_TO_REGCLASS") { 2026 MatchedRC = getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); 2027 2028 if (MatchedRC.isNull()) 2029 return failedImport( 2030 "COPY_TO_REGCLASS operand #1 isn't a register class"); 2031 } else if (DstIName == "REG_SEQUENCE") { 2032 MatchedRC = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); 2033 if (MatchedRC.isNull()) 2034 return failedImport("REG_SEQUENCE operand #0 isn't a register class"); 2035 } else if (DstIName == "EXTRACT_SUBREG") { 2036 auto InferredClass = inferRegClassFromPattern(Dst->getChild(0)); 2037 if (!InferredClass) 2038 return failedImport( 2039 "Could not infer class for EXTRACT_SUBREG operand #0"); 2040 2041 // We can assume that a subregister is in the same bank as it's super 2042 // register. 2043 MatchedRC = (*InferredClass)->getDef(); 2044 } else if (DstIName == "INSERT_SUBREG") { 2045 auto MaybeSuperClass = inferSuperRegisterClassForNode( 2046 VTy, Dst->getChild(0), Dst->getChild(2)); 2047 if (!MaybeSuperClass) 2048 return failedImport( 2049 "Cannot infer register class for INSERT_SUBREG operand #0"); 2050 // Move to the next pattern here, because the register class we found 2051 // doesn't necessarily have a record associated with it. So, we can't 2052 // set DstIOpRec using this. 2053 MatchedRC = *MaybeSuperClass; 2054 } else if (DstIName == "SUBREG_TO_REG") { 2055 auto MaybeRegClass = inferSuperRegisterClass(VTy, Dst->getChild(2)); 2056 if (!MaybeRegClass) 2057 return failedImport( 2058 "Cannot infer register class for SUBREG_TO_REG operand #0"); 2059 MatchedRC = *MaybeRegClass; 2060 } else if (MatchedRC.get<Record *>()->isSubClassOf("RegisterOperand")) 2061 MatchedRC = MatchedRC.get<Record *>()->getValueAsDef("RegClass"); 2062 else if (!MatchedRC.get<Record *>()->isSubClassOf("RegisterClass")) 2063 return failedImport("Dst MI def isn't a register class" + 2064 to_string(*Dst)); 2065 2066 OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); 2067 // The operand names declared in the DstI instruction are unrelated to 2068 // those used in pattern's source and destination DAGs, so mangle the 2069 // former to prevent implicitly adding unexpected 2070 // GIM_CheckIsSameOperand predicates by the defineOperand method. 2071 OM.setSymbolicName(getMangledRootDefName(DstIOperand.Name)); 2072 M.defineOperand(OM.getSymbolicName(), OM); 2073 if (MatchedRC.is<Record *>()) 2074 MatchedRC = &Target.getRegisterClass(MatchedRC.get<Record *>()); 2075 OM.addPredicate<RegisterBankOperandMatcher>( 2076 *MatchedRC.get<const CodeGenRegisterClass *>()); 2077 ++OpIdx; 2078 } 2079 2080 auto DstMIBuilderOrError = 2081 createAndImportInstructionRenderer(M, InsnMatcher, Src, Dst); 2082 if (auto Error = DstMIBuilderOrError.takeError()) 2083 return std::move(Error); 2084 BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get(); 2085 2086 // Render the implicit defs. 2087 // These are only added to the root of the result. 2088 if (auto Error = importImplicitDefRenderers(DstMIBuilder, P.getDstRegs())) 2089 return std::move(Error); 2090 2091 DstMIBuilder.chooseInsnToMutate(M); 2092 2093 // Constrain the registers to classes. This is normally derived from the 2094 // emitted instruction but a few instructions require special handling. 2095 if (DstIName == "COPY_TO_REGCLASS") { 2096 // COPY_TO_REGCLASS does not provide operand constraints itself but the 2097 // result is constrained to the class given by the second child. 2098 Record *DstIOpRec = 2099 getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); 2100 2101 if (DstIOpRec == nullptr) 2102 return failedImport("COPY_TO_REGCLASS operand #1 isn't a register class"); 2103 2104 M.addAction<ConstrainOperandToRegClassAction>( 2105 0, 0, Target.getRegisterClass(DstIOpRec)); 2106 } else if (DstIName == "EXTRACT_SUBREG") { 2107 auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); 2108 if (!SuperClass) 2109 return failedImport( 2110 "Cannot infer register class from EXTRACT_SUBREG operand #0"); 2111 2112 auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1)); 2113 if (!SubIdx) 2114 return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); 2115 2116 // It would be nice to leave this constraint implicit but we're required 2117 // to pick a register class so constrain the result to a register class 2118 // that can hold the correct MVT. 2119 // 2120 // FIXME: This may introduce an extra copy if the chosen class doesn't 2121 // actually contain the subregisters. 2122 assert(Src->getExtTypes().size() == 1 && 2123 "Expected Src of EXTRACT_SUBREG to have one result type"); 2124 2125 const auto SrcRCDstRCPair = 2126 (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); 2127 if (!SrcRCDstRCPair) { 2128 return failedImport("subreg index is incompatible " 2129 "with inferred reg class"); 2130 } 2131 2132 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); 2133 M.addAction<ConstrainOperandToRegClassAction>(0, 0, 2134 *SrcRCDstRCPair->second); 2135 M.addAction<ConstrainOperandToRegClassAction>(0, 1, *SrcRCDstRCPair->first); 2136 } else if (DstIName == "INSERT_SUBREG") { 2137 assert(Src->getExtTypes().size() == 1 && 2138 "Expected Src of INSERT_SUBREG to have one result type"); 2139 // We need to constrain the destination, a super regsister source, and a 2140 // subregister source. 2141 auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); 2142 if (!SubClass) 2143 return failedImport( 2144 "Cannot infer register class from INSERT_SUBREG operand #1"); 2145 auto SuperClass = inferSuperRegisterClassForNode( 2146 Src->getExtType(0), Dst->getChild(0), Dst->getChild(2)); 2147 if (!SuperClass) 2148 return failedImport( 2149 "Cannot infer register class for INSERT_SUBREG operand #0"); 2150 M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); 2151 M.addAction<ConstrainOperandToRegClassAction>(0, 1, **SuperClass); 2152 M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass); 2153 } else if (DstIName == "SUBREG_TO_REG") { 2154 // We need to constrain the destination and subregister source. 2155 assert(Src->getExtTypes().size() == 1 && 2156 "Expected Src of SUBREG_TO_REG to have one result type"); 2157 2158 // Attempt to infer the subregister source from the first child. If it has 2159 // an explicitly given register class, we'll use that. Otherwise, we will 2160 // fail. 2161 auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); 2162 if (!SubClass) 2163 return failedImport( 2164 "Cannot infer register class from SUBREG_TO_REG child #1"); 2165 // We don't have a child to look at that might have a super register node. 2166 auto SuperClass = 2167 inferSuperRegisterClass(Src->getExtType(0), Dst->getChild(2)); 2168 if (!SuperClass) 2169 return failedImport( 2170 "Cannot infer register class for SUBREG_TO_REG operand #0"); 2171 M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); 2172 M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass); 2173 } else if (DstIName == "REG_SEQUENCE") { 2174 auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); 2175 2176 M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); 2177 2178 unsigned Num = Dst->getNumChildren(); 2179 for (unsigned I = 1; I != Num; I += 2) { 2180 TreePatternNode *SubRegChild = Dst->getChild(I + 1); 2181 2182 auto SubIdx = inferSubRegIndexForNode(SubRegChild); 2183 if (!SubIdx) 2184 return failedImport("REG_SEQUENCE child is not a subreg index"); 2185 2186 const auto SrcRCDstRCPair = 2187 (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); 2188 2189 M.addAction<ConstrainOperandToRegClassAction>(0, I, 2190 *SrcRCDstRCPair->second); 2191 } 2192 } else { 2193 M.addAction<ConstrainOperandsToDefinitionAction>(0); 2194 } 2195 2196 // Erase the root. 2197 unsigned RootInsnID = M.getInsnVarID(InsnMatcher); 2198 M.addAction<EraseInstAction>(RootInsnID); 2199 2200 // We're done with this pattern! It's eligible for GISel emission; return it. 2201 ++NumPatternImported; 2202 return std::move(M); 2203 } 2204 2205 MatchTable 2206 GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules, 2207 bool Optimize, bool WithCoverage) { 2208 std::vector<Matcher *> InputRules; 2209 for (Matcher &Rule : Rules) 2210 InputRules.push_back(&Rule); 2211 2212 if (!Optimize) 2213 return MatchTable::buildTable(InputRules, WithCoverage); 2214 2215 unsigned CurrentOrdering = 0; 2216 StringMap<unsigned> OpcodeOrder; 2217 for (RuleMatcher &Rule : Rules) { 2218 const StringRef Opcode = Rule.getOpcode(); 2219 assert(!Opcode.empty() && "Didn't expect an undefined opcode"); 2220 if (OpcodeOrder.count(Opcode) == 0) 2221 OpcodeOrder[Opcode] = CurrentOrdering++; 2222 } 2223 2224 llvm::stable_sort(InputRules, [&OpcodeOrder](const Matcher *A, 2225 const Matcher *B) { 2226 auto *L = static_cast<const RuleMatcher *>(A); 2227 auto *R = static_cast<const RuleMatcher *>(B); 2228 return std::make_tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) < 2229 std::make_tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands()); 2230 }); 2231 2232 for (Matcher *Rule : InputRules) 2233 Rule->optimize(); 2234 2235 std::vector<std::unique_ptr<Matcher>> MatcherStorage; 2236 std::vector<Matcher *> OptRules = 2237 optimizeRules<GroupMatcher>(InputRules, MatcherStorage); 2238 2239 for (Matcher *Rule : OptRules) 2240 Rule->optimize(); 2241 2242 OptRules = optimizeRules<SwitchMatcher>(OptRules, MatcherStorage); 2243 2244 return MatchTable::buildTable(OptRules, WithCoverage); 2245 } 2246 2247 void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) { 2248 OS << "bool " << getClassName() 2249 << "::selectImpl(MachineInstr &I, CodeGenCoverage " 2250 "&CoverageInfo) const {\n" 2251 << " const PredicateBitset AvailableFeatures = " 2252 "getAvailableFeatures();\n" 2253 << " MachineIRBuilder B(I);\n" 2254 << " State.MIs.clear();\n" 2255 << " State.MIs.push_back(&I);\n\n" 2256 << " if (executeMatchTable(*this, State, ExecInfo, B" 2257 << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures" 2258 << ", &CoverageInfo)) {\n" 2259 << " return true;\n" 2260 << " }\n\n" 2261 << " return false;\n" 2262 << "}\n\n"; 2263 } 2264 2265 void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) { 2266 std::vector<Record *> MatchedRecords; 2267 std::copy_if(AllPatFrags.begin(), AllPatFrags.end(), 2268 std::back_inserter(MatchedRecords), [&](Record *R) { 2269 return !R->getValueAsString("GISelPredicateCode").empty(); 2270 }); 2271 emitMIPredicateFnsImpl<Record *>( 2272 OS, 2273 " const MachineFunction &MF = *MI.getParent()->getParent();\n" 2274 " const MachineRegisterInfo &MRI = MF.getRegInfo();\n" 2275 " const auto &Operands = State.RecordedOperands;\n" 2276 " (void)Operands;\n" 2277 " (void)MRI;", 2278 ArrayRef<Record *>(MatchedRecords), &getPatFragPredicateEnumName, 2279 [&](Record *R) { return R->getValueAsString("GISelPredicateCode"); }, 2280 "PatFrag predicates."); 2281 } 2282 2283 void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) { 2284 std::vector<Record *> MatchedRecords; 2285 std::copy_if(AllPatFrags.begin(), AllPatFrags.end(), 2286 std::back_inserter(MatchedRecords), [&](Record *R) { 2287 bool Unset; 2288 return !R->getValueAsString("ImmediateCode").empty() && 2289 !R->getValueAsBitOrUnset("IsAPFloat", Unset) && 2290 !R->getValueAsBit("IsAPInt"); 2291 }); 2292 emitImmPredicateFnsImpl<Record *>( 2293 OS, "I64", "int64_t", ArrayRef<Record *>(MatchedRecords), 2294 &getPatFragPredicateEnumName, 2295 [&](Record *R) { return R->getValueAsString("ImmediateCode"); }, 2296 "PatFrag predicates."); 2297 } 2298 2299 void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) { 2300 std::vector<Record *> MatchedRecords; 2301 std::copy_if(AllPatFrags.begin(), AllPatFrags.end(), 2302 std::back_inserter(MatchedRecords), [&](Record *R) { 2303 bool Unset; 2304 return !R->getValueAsString("ImmediateCode").empty() && 2305 R->getValueAsBitOrUnset("IsAPFloat", Unset); 2306 }); 2307 emitImmPredicateFnsImpl<Record *>( 2308 OS, "APFloat", "const APFloat &", ArrayRef<Record *>(MatchedRecords), 2309 &getPatFragPredicateEnumName, 2310 [&](Record *R) { return R->getValueAsString("ImmediateCode"); }, 2311 "PatFrag predicates."); 2312 } 2313 2314 void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) { 2315 std::vector<Record *> MatchedRecords; 2316 std::copy_if(AllPatFrags.begin(), AllPatFrags.end(), 2317 std::back_inserter(MatchedRecords), [&](Record *R) { 2318 return !R->getValueAsString("ImmediateCode").empty() && 2319 R->getValueAsBit("IsAPInt"); 2320 }); 2321 emitImmPredicateFnsImpl<Record *>( 2322 OS, "APInt", "const APInt &", ArrayRef<Record *>(MatchedRecords), 2323 &getPatFragPredicateEnumName, 2324 [&](Record *R) { return R->getValueAsString("ImmediateCode"); }, 2325 "PatFrag predicates."); 2326 } 2327 2328 void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) { 2329 OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n" 2330 << " llvm_unreachable(\"" + getClassName() + 2331 " does not support simple predicates!\");\n" 2332 << " return false;\n" 2333 << "}\n"; 2334 } 2335 2336 void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) { 2337 OS << "void " << getClassName() 2338 << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const " 2339 "{\n" 2340 << " llvm_unreachable(\"" + getClassName() + 2341 " does not support custom C++ actions!\");\n" 2342 << "}\n"; 2343 } 2344 2345 void GlobalISelEmitter::postProcessRule(RuleMatcher &M) { 2346 SmallPtrSet<Record *, 16> UsedRegs; 2347 2348 // TODO: deal with subregs? 2349 for (auto &A : M.actions()) { 2350 auto *MI = dyn_cast<BuildMIAction>(A.get()); 2351 if (!MI) 2352 continue; 2353 2354 for (auto *Use : MI->getCGI()->ImplicitUses) 2355 UsedRegs.insert(Use); 2356 } 2357 2358 for (auto &A : M.actions()) { 2359 auto *MI = dyn_cast<BuildMIAction>(A.get()); 2360 if (!MI) 2361 continue; 2362 2363 for (auto *Def : MI->getCGI()->ImplicitDefs) { 2364 if (!UsedRegs.contains(Def)) 2365 MI->setDeadImplicitDef(Def); 2366 } 2367 } 2368 } 2369 2370 void GlobalISelEmitter::run(raw_ostream &OS) { 2371 if (!UseCoverageFile.empty()) { 2372 RuleCoverage = CodeGenCoverage(); 2373 auto RuleCoverageBufOrErr = MemoryBuffer::getFile(UseCoverageFile); 2374 if (!RuleCoverageBufOrErr) { 2375 PrintWarning(SMLoc(), "Missing rule coverage data"); 2376 RuleCoverage = std::nullopt; 2377 } else { 2378 if (!RuleCoverage->parse(*RuleCoverageBufOrErr.get(), Target.getName())) { 2379 PrintWarning(SMLoc(), "Ignoring invalid or missing rule coverage data"); 2380 RuleCoverage = std::nullopt; 2381 } 2382 } 2383 } 2384 2385 // Track the run-time opcode values 2386 gatherOpcodeValues(); 2387 // Track the run-time LLT ID values 2388 gatherTypeIDValues(); 2389 2390 // Track the GINodeEquiv definitions. 2391 gatherNodeEquivs(); 2392 2393 AllPatFrags = RK.getAllDerivedDefinitions("PatFrags"); 2394 2395 emitSourceFileHeader( 2396 ("Global Instruction Selector for the " + Target.getName() + " target") 2397 .str(), 2398 OS); 2399 std::vector<RuleMatcher> Rules; 2400 // Look through the SelectionDAG patterns we found, possibly emitting some. 2401 for (const PatternToMatch &Pat : CGP.ptms()) { 2402 ++NumPatternTotal; 2403 2404 auto MatcherOrErr = runOnPattern(Pat); 2405 2406 // The pattern analysis can fail, indicating an unsupported pattern. 2407 // Report that if we've been asked to do so. 2408 if (auto Err = MatcherOrErr.takeError()) { 2409 if (WarnOnSkippedPatterns) { 2410 PrintWarning(Pat.getSrcRecord()->getLoc(), 2411 "Skipped pattern: " + toString(std::move(Err))); 2412 } else { 2413 consumeError(std::move(Err)); 2414 } 2415 ++NumPatternImportsSkipped; 2416 continue; 2417 } 2418 2419 if (RuleCoverage) { 2420 if (RuleCoverage->isCovered(MatcherOrErr->getRuleID())) 2421 ++NumPatternsTested; 2422 else 2423 PrintWarning(Pat.getSrcRecord()->getLoc(), 2424 "Pattern is not covered by a test"); 2425 } 2426 Rules.push_back(std::move(MatcherOrErr.get())); 2427 postProcessRule(Rules.back()); 2428 } 2429 2430 // Comparison function to order records by name. 2431 auto orderByName = [](const Record *A, const Record *B) { 2432 return A->getName() < B->getName(); 2433 }; 2434 2435 std::vector<Record *> ComplexPredicates = 2436 RK.getAllDerivedDefinitions("GIComplexOperandMatcher"); 2437 llvm::sort(ComplexPredicates, orderByName); 2438 2439 std::vector<StringRef> CustomRendererFns; 2440 transform(RK.getAllDerivedDefinitions("GICustomOperandRenderer"), 2441 std::back_inserter(CustomRendererFns), [](const auto &Record) { 2442 return Record->getValueAsString("RendererFn"); 2443 }); 2444 // Sort and remove duplicates to get a list of unique renderer functions, in 2445 // case some were mentioned more than once. 2446 llvm::sort(CustomRendererFns); 2447 CustomRendererFns.erase( 2448 std::unique(CustomRendererFns.begin(), CustomRendererFns.end()), 2449 CustomRendererFns.end()); 2450 2451 // Create a table containing the LLT objects needed by the matcher and an enum 2452 // for the matcher to reference them with. 2453 std::vector<LLTCodeGen> TypeObjects; 2454 append_range(TypeObjects, KnownTypes); 2455 llvm::sort(TypeObjects); 2456 2457 // Sort rules. 2458 llvm::stable_sort(Rules, [&](const RuleMatcher &A, const RuleMatcher &B) { 2459 int ScoreA = RuleMatcherScores[A.getRuleID()]; 2460 int ScoreB = RuleMatcherScores[B.getRuleID()]; 2461 if (ScoreA > ScoreB) 2462 return true; 2463 if (ScoreB > ScoreA) 2464 return false; 2465 if (A.isHigherPriorityThan(B)) { 2466 assert(!B.isHigherPriorityThan(A) && "Cannot be more important " 2467 "and less important at " 2468 "the same time"); 2469 return true; 2470 } 2471 return false; 2472 }); 2473 2474 unsigned MaxTemporaries = 0; 2475 for (const auto &Rule : Rules) 2476 MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns()); 2477 2478 // Build match table 2479 const MatchTable Table = 2480 buildMatchTable(Rules, OptimizeMatchTable, GenerateCoverage); 2481 2482 emitPredicateBitset(OS, "GET_GLOBALISEL_PREDICATE_BITSET"); 2483 emitTemporariesDecl(OS, "GET_GLOBALISEL_TEMPORARIES_DECL"); 2484 emitTemporariesInit(OS, MaxTemporaries, "GET_GLOBALISEL_TEMPORARIES_INIT"); 2485 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexPredicates, 2486 CustomRendererFns, "GET_GLOBALISEL_IMPL"); 2487 emitPredicatesDecl(OS, "GET_GLOBALISEL_PREDICATES_DECL"); 2488 emitPredicatesInit(OS, "GET_GLOBALISEL_PREDICATES_INIT"); 2489 } 2490 2491 void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) { 2492 SubtargetFeatures.try_emplace(Predicate, Predicate, SubtargetFeatures.size()); 2493 } 2494 2495 unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) { 2496 return HwModes.emplace(HwModeFeatures.str(), HwModes.size()).first->second; 2497 } 2498 2499 } // end anonymous namespace 2500 2501 //===----------------------------------------------------------------------===// 2502 2503 static TableGen::Emitter::OptClass<GlobalISelEmitter> 2504 X("gen-global-isel", "Generate GlobalISel selector"); 2505