1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10 /// function. This is implemented in a separate file because the function is
11 /// quite large.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterBankInfo.h"
27 #include "llvm/CodeGen/TargetInstrInfo.h"
28 #include "llvm/CodeGen/TargetOpcodes.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/IR/Constants.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/Support/CodeGenCoverage.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/LEB128.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include <cassert>
39 #include <cstddef>
40 #include <cstdint>
41
42 namespace llvm {
43
44 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45 class CustomRendererFn>
executeMatchTable(TgtExecutor & Exec,MatcherState & State,const ExecInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ExecInfo,MachineIRBuilder & Builder,const uint8_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage * CoverageInfo)46 bool GIMatchTableExecutor::executeMatchTable(
47 TgtExecutor &Exec, MatcherState &State,
48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
49 &ExecInfo,
50 MachineIRBuilder &Builder, const uint8_t *MatchTable,
51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53 const PredicateBitset &AvailableFeatures,
54 CodeGenCoverage *CoverageInfo) const {
55
56 uint64_t CurrentIdx = 0;
57 SmallVector<uint64_t, 4> OnFailResumeAt;
58 NewMIVector OutMIs;
59
60 GISelChangeObserver *Observer = Builder.getObserver();
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint16_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtExecutor::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtExecutor::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 const auto propagateFlags = [&]() {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint16_t MIBFlags = Flags;
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 if (Observer)
87 Observer->changingInstr(*MIB);
88 MIB.setMIFlags(MIBFlags);
89 if (Observer)
90 Observer->changedInstr(*MIB);
91 }
92 };
93
94 // If the index is >= 0, it's an index in the type objects generated by
95 // TableGen. If the index is <0, it's an index in the recorded types object.
96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97 if (Idx >= 0)
98 return ExecInfo.TypeObjects[Idx];
99 return State.RecordedTypes[1 - Idx];
100 };
101
102 const auto readULEB = [&]() {
103 return fastDecodeULEB128(MatchTable, CurrentIdx);
104 };
105
106 // Convenience function to return a signed value. This avoids
107 // us forgetting to first cast to int8_t before casting to a
108 // wider signed int type.
109 // if we casted uint8 directly to a wider type we'd lose
110 // negative values.
111 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
112
113 const auto readU16 = [&]() {
114 auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx);
115 CurrentIdx += 2;
116 return V;
117 };
118
119 const auto readU32 = [&]() {
120 auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx);
121 CurrentIdx += 4;
122 return V;
123 };
124
125 const auto readU64 = [&]() {
126 auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx);
127 CurrentIdx += 8;
128 return V;
129 };
130
131 const auto eraseImpl = [&](MachineInstr *MI) {
132 // If we're erasing the insertion point, ensure we don't leave a dangling
133 // pointer in the builder.
134 if (Builder.getInsertPt() == MI)
135 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());
136 if (Observer)
137 Observer->erasingInstr(*MI);
138 MI->eraseFromParent();
139 };
140
141 while (true) {
142 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
143 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
144 switch (MatcherOpcode) {
145 case GIM_Try: {
146 DEBUG_WITH_TYPE(TgtExecutor::getName(),
147 dbgs() << CurrentIdx << ": Begin try-block\n");
148 OnFailResumeAt.push_back(readU32());
149 break;
150 }
151
152 case GIM_RecordInsn:
153 case GIM_RecordInsnIgnoreCopies: {
154 uint64_t NewInsnID = readULEB();
155 uint64_t InsnID = readULEB();
156 uint64_t OpIdx = readULEB();
157
158 // As an optimisation we require that MIs[0] is always the root. Refuse
159 // any attempt to modify it.
160 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
161
162 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
163 if (!MO.isReg()) {
164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
165 dbgs() << CurrentIdx << ": Not a register\n");
166 if (handleReject() == RejectAndGiveUp)
167 return false;
168 break;
169 }
170 if (MO.getReg().isPhysical()) {
171 DEBUG_WITH_TYPE(TgtExecutor::getName(),
172 dbgs() << CurrentIdx << ": Is a physical register\n");
173 if (handleReject() == RejectAndGiveUp)
174 return false;
175 break;
176 }
177
178 MachineInstr *NewMI;
179 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
180 NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
181 else
182 NewMI = MRI.getVRegDef(MO.getReg());
183
184 if ((size_t)NewInsnID < State.MIs.size())
185 State.MIs[NewInsnID] = NewMI;
186 else {
187 assert((size_t)NewInsnID == State.MIs.size() &&
188 "Expected to store MIs in order");
189 State.MIs.push_back(NewMI);
190 }
191 DEBUG_WITH_TYPE(TgtExecutor::getName(),
192 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
193 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
194 << ")\n");
195 break;
196 }
197
198 case GIM_CheckFeatures: {
199 uint16_t ExpectedBitsetID = readU16();
200 DEBUG_WITH_TYPE(TgtExecutor::getName(),
201 dbgs() << CurrentIdx
202 << ": GIM_CheckFeatures(ExpectedBitsetID="
203 << ExpectedBitsetID << ")\n");
204 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
205 ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
206 if (handleReject() == RejectAndGiveUp)
207 return false;
208 }
209 break;
210 }
211 case GIM_CheckOpcode:
212 case GIM_CheckOpcodeIsEither: {
213 uint64_t InsnID = readULEB();
214 uint16_t Expected0 = readU16();
215 uint16_t Expected1 = -1;
216 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
217 Expected1 = readU16();
218
219 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
220 unsigned Opcode = State.MIs[InsnID]->getOpcode();
221
222 DEBUG_WITH_TYPE(TgtExecutor::getName(),
223 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
224 << "], ExpectedOpcode=" << Expected0;
225 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs()
226 << " || " << Expected1;
227 dbgs() << ") // Got=" << Opcode << "\n";);
228
229 if (Opcode != Expected0 && Opcode != Expected1) {
230 if (handleReject() == RejectAndGiveUp)
231 return false;
232 }
233 break;
234 }
235 case GIM_SwitchOpcode: {
236 uint64_t InsnID = readULEB();
237 uint16_t LowerBound = readU16();
238 uint16_t UpperBound = readU16();
239 uint32_t Default = readU32();
240
241 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
242 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
243
244 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
245 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
246 << LowerBound << ", " << UpperBound << "), Default=" << Default
247 << ", JumpTable...) // Got=" << Opcode << "\n";
248 });
249 if (Opcode < LowerBound || UpperBound <= Opcode) {
250 CurrentIdx = Default;
251 break;
252 }
253 const auto EntryIdx = (Opcode - LowerBound);
254 // Each entry is 4 bytes
255 CurrentIdx =
256 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4));
257 if (!CurrentIdx) {
258 CurrentIdx = Default;
259 break;
260 }
261 OnFailResumeAt.push_back(Default);
262 break;
263 }
264
265 case GIM_SwitchType: {
266 uint64_t InsnID = readULEB();
267 uint64_t OpIdx = readULEB();
268 uint16_t LowerBound = readU16();
269 uint16_t UpperBound = readU16();
270 int64_t Default = readU32();
271
272 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
274
275 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
276 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
277 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
278 << UpperBound << "), Default=" << Default
279 << ", JumpTable...) // Got=";
280 if (!MO.isReg())
281 dbgs() << "Not a VReg\n";
282 else
283 dbgs() << MRI.getType(MO.getReg()) << "\n";
284 });
285 if (!MO.isReg()) {
286 CurrentIdx = Default;
287 break;
288 }
289 const LLT Ty = MRI.getType(MO.getReg());
290 const auto TyI = ExecInfo.TypeIDMap.find(Ty);
291 if (TyI == ExecInfo.TypeIDMap.end()) {
292 CurrentIdx = Default;
293 break;
294 }
295 const int64_t TypeID = TyI->second;
296 if (TypeID < LowerBound || UpperBound <= TypeID) {
297 CurrentIdx = Default;
298 break;
299 }
300 const auto NumEntry = (TypeID - LowerBound);
301 // Each entry is 4 bytes
302 CurrentIdx =
303 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4));
304 if (!CurrentIdx) {
305 CurrentIdx = Default;
306 break;
307 }
308 OnFailResumeAt.push_back(Default);
309 break;
310 }
311
312 case GIM_CheckNumOperands: {
313 uint64_t InsnID = readULEB();
314 uint64_t Expected = readULEB();
315 DEBUG_WITH_TYPE(TgtExecutor::getName(),
316 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
317 << InsnID << "], Expected=" << Expected << ")\n");
318 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
319 if (State.MIs[InsnID]->getNumOperands() != Expected) {
320 if (handleReject() == RejectAndGiveUp)
321 return false;
322 }
323 break;
324 }
325 case GIM_CheckI64ImmPredicate:
326 case GIM_CheckImmOperandPredicate: {
327 uint64_t InsnID = readULEB();
328 unsigned OpIdx =
329 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
330 uint16_t Predicate = readU16();
331 DEBUG_WITH_TYPE(TgtExecutor::getName(),
332 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
333 << InsnID << "]->getOperand(" << OpIdx
334 << "), Predicate=" << Predicate << ")\n");
335 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
336 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
337 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
338 "Expected immediate operand");
339 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
340 int64_t Value = 0;
341 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
342 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
343 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
344 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
345 else
346 llvm_unreachable("Expected Imm or CImm operand");
347
348 if (!testImmPredicate_I64(Predicate, Value))
349 if (handleReject() == RejectAndGiveUp)
350 return false;
351 break;
352 }
353 case GIM_CheckAPIntImmPredicate: {
354 uint64_t InsnID = readULEB();
355 uint16_t Predicate = readU16();
356 DEBUG_WITH_TYPE(TgtExecutor::getName(),
357 dbgs()
358 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
359 << InsnID << "], Predicate=" << Predicate << ")\n");
360 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
361 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
362 "Expected G_CONSTANT");
363 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
364 if (!State.MIs[InsnID]->getOperand(1).isCImm())
365 llvm_unreachable("Expected Imm or CImm operand");
366
367 const APInt &Value =
368 State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
369 if (!testImmPredicate_APInt(Predicate, Value))
370 if (handleReject() == RejectAndGiveUp)
371 return false;
372 break;
373 }
374 case GIM_CheckAPFloatImmPredicate: {
375 uint64_t InsnID = readULEB();
376 uint16_t Predicate = readU16();
377 DEBUG_WITH_TYPE(TgtExecutor::getName(),
378 dbgs()
379 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
380 << InsnID << "], Predicate=" << Predicate << ")\n");
381 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
382 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
383 "Expected G_FCONSTANT");
384 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
385 "Expected FPImm operand");
386 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
387 const APFloat &Value =
388 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
389
390 if (!testImmPredicate_APFloat(Predicate, Value))
391 if (handleReject() == RejectAndGiveUp)
392 return false;
393 break;
394 }
395 case GIM_CheckIsBuildVectorAllOnes:
396 case GIM_CheckIsBuildVectorAllZeros: {
397 uint64_t InsnID = readULEB();
398
399 DEBUG_WITH_TYPE(TgtExecutor::getName(),
400 dbgs() << CurrentIdx
401 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
402 << InsnID << "])\n");
403 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
404
405 const MachineInstr *MI = State.MIs[InsnID];
406 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
407 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
408 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
409
410 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
411 if (!isBuildVectorAllOnes(*MI, MRI)) {
412 if (handleReject() == RejectAndGiveUp)
413 return false;
414 }
415 } else {
416 if (!isBuildVectorAllZeros(*MI, MRI)) {
417 if (handleReject() == RejectAndGiveUp)
418 return false;
419 }
420 }
421
422 break;
423 }
424 case GIM_CheckSimplePredicate: {
425 // Note: we don't check for invalid here because this is purely a hook to
426 // allow some executors (such as the combiner) to check arbitrary,
427 // contextless predicates, such as whether a rule is enabled or not.
428 uint16_t Predicate = readU16();
429 DEBUG_WITH_TYPE(TgtExecutor::getName(),
430 dbgs() << CurrentIdx
431 << ": GIM_CheckSimplePredicate(Predicate="
432 << Predicate << ")\n");
433 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
434 if (!testSimplePredicate(Predicate)) {
435 if (handleReject() == RejectAndGiveUp)
436 return false;
437 }
438 break;
439 }
440 case GIM_CheckCxxInsnPredicate: {
441 uint64_t InsnID = readULEB();
442 uint16_t Predicate = readU16();
443 DEBUG_WITH_TYPE(TgtExecutor::getName(),
444 dbgs()
445 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
446 << InsnID << "], Predicate=" << Predicate << ")\n");
447 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
448 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
449
450 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
451 if (handleReject() == RejectAndGiveUp)
452 return false;
453 break;
454 }
455 case GIM_CheckHasNoUse: {
456 uint64_t InsnID = readULEB();
457
458 DEBUG_WITH_TYPE(TgtExecutor::getName(),
459 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
460 << InsnID << "]\n");
461
462 const MachineInstr *MI = State.MIs[InsnID];
463 assert(MI && "Used insn before defined");
464 assert(MI->getNumDefs() > 0 && "No defs");
465 const Register Res = MI->getOperand(0).getReg();
466
467 if (!MRI.use_nodbg_empty(Res)) {
468 if (handleReject() == RejectAndGiveUp)
469 return false;
470 }
471 break;
472 }
473 case GIM_CheckHasOneUse: {
474 uint64_t InsnID = readULEB();
475
476 DEBUG_WITH_TYPE(TgtExecutor::getName(),
477 dbgs() << CurrentIdx << ": GIM_CheckHasOneUse(MIs["
478 << InsnID << "]\n");
479
480 const MachineInstr *MI = State.MIs[InsnID];
481 assert(MI && "Used insn before defined");
482 assert(MI->getNumDefs() > 0 && "No defs");
483 const Register Res = MI->getOperand(0).getReg();
484
485 if (!MRI.hasOneNonDBGUse(Res)) {
486 if (handleReject() == RejectAndGiveUp)
487 return false;
488 }
489 break;
490 }
491 case GIM_CheckAtomicOrdering: {
492 uint64_t InsnID = readULEB();
493 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
494 DEBUG_WITH_TYPE(TgtExecutor::getName(),
495 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
496 << InsnID << "], " << (uint64_t)Ordering << ")\n");
497 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
498 if (!State.MIs[InsnID]->hasOneMemOperand())
499 if (handleReject() == RejectAndGiveUp)
500 return false;
501
502 for (const auto &MMO : State.MIs[InsnID]->memoperands())
503 if (MMO->getMergedOrdering() != Ordering)
504 if (handleReject() == RejectAndGiveUp)
505 return false;
506 break;
507 }
508 case GIM_CheckAtomicOrderingOrStrongerThan: {
509 uint64_t InsnID = readULEB();
510 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
511 DEBUG_WITH_TYPE(TgtExecutor::getName(),
512 dbgs() << CurrentIdx
513 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
514 << InsnID << "], " << (uint64_t)Ordering << ")\n");
515 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
516 if (!State.MIs[InsnID]->hasOneMemOperand())
517 if (handleReject() == RejectAndGiveUp)
518 return false;
519
520 for (const auto &MMO : State.MIs[InsnID]->memoperands())
521 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
522 if (handleReject() == RejectAndGiveUp)
523 return false;
524 break;
525 }
526 case GIM_CheckAtomicOrderingWeakerThan: {
527 uint64_t InsnID = readULEB();
528 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
529 DEBUG_WITH_TYPE(TgtExecutor::getName(),
530 dbgs() << CurrentIdx
531 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
532 << InsnID << "], " << (uint64_t)Ordering << ")\n");
533 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
534 if (!State.MIs[InsnID]->hasOneMemOperand())
535 if (handleReject() == RejectAndGiveUp)
536 return false;
537
538 for (const auto &MMO : State.MIs[InsnID]->memoperands())
539 if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
540 if (handleReject() == RejectAndGiveUp)
541 return false;
542 break;
543 }
544 case GIM_CheckMemoryAddressSpace: {
545 uint64_t InsnID = readULEB();
546 uint64_t MMOIdx = readULEB();
547 // This accepts a list of possible address spaces.
548 const uint64_t NumAddrSpace = MatchTable[CurrentIdx++];
549
550 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
551 if (handleReject() == RejectAndGiveUp)
552 return false;
553 break;
554 }
555
556 // Need to still jump to the end of the list of address spaces if we find
557 // a match earlier.
558 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
559
560 const MachineMemOperand *MMO =
561 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
562 const unsigned MMOAddrSpace = MMO->getAddrSpace();
563
564 bool Success = false;
565 for (unsigned I = 0; I != NumAddrSpace; ++I) {
566 uint64_t AddrSpace = readULEB();
567 DEBUG_WITH_TYPE(TgtExecutor::getName(),
568 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
569 << AddrSpace << '\n');
570
571 if (AddrSpace == MMOAddrSpace) {
572 Success = true;
573 break;
574 }
575 }
576
577 CurrentIdx = LastIdx;
578 if (!Success && handleReject() == RejectAndGiveUp)
579 return false;
580 break;
581 }
582 case GIM_CheckMemoryAlignment: {
583 uint64_t InsnID = readULEB();
584 uint64_t MMOIdx = readULEB();
585 uint64_t MinAlign = MatchTable[CurrentIdx++];
586
587 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
588
589 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
590 if (handleReject() == RejectAndGiveUp)
591 return false;
592 break;
593 }
594
595 MachineMemOperand *MMO =
596 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
597 DEBUG_WITH_TYPE(TgtExecutor::getName(),
598 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
599 << "(MIs[" << InsnID << "]->memoperands() + "
600 << MMOIdx << ")->getAlignment() >= " << MinAlign
601 << ")\n");
602 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
603 return false;
604
605 break;
606 }
607 case GIM_CheckMemorySizeEqualTo: {
608 uint64_t InsnID = readULEB();
609 uint64_t MMOIdx = readULEB();
610 uint32_t Size = readU32();
611
612 DEBUG_WITH_TYPE(TgtExecutor::getName(),
613 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
614 << InsnID << "]->memoperands() + " << MMOIdx
615 << ", Size=" << Size << ")\n");
616 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
617
618 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
619 if (handleReject() == RejectAndGiveUp)
620 return false;
621 break;
622 }
623
624 MachineMemOperand *MMO =
625 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
626
627 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
628 << " bytes vs " << Size
629 << " bytes\n");
630 if (MMO->getSize() != Size)
631 if (handleReject() == RejectAndGiveUp)
632 return false;
633
634 break;
635 }
636 case GIM_CheckMemorySizeEqualToLLT:
637 case GIM_CheckMemorySizeLessThanLLT:
638 case GIM_CheckMemorySizeGreaterThanLLT: {
639 uint64_t InsnID = readULEB();
640 uint64_t MMOIdx = readULEB();
641 uint64_t OpIdx = readULEB();
642
643 DEBUG_WITH_TYPE(
644 TgtExecutor::getName(),
645 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
646 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
647 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
648 ? "GreaterThan"
649 : "LessThan")
650 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
651 << ", OpIdx=" << OpIdx << ")\n");
652 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
653
654 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
655 if (!MO.isReg()) {
656 DEBUG_WITH_TYPE(TgtExecutor::getName(),
657 dbgs() << CurrentIdx << ": Not a register\n");
658 if (handleReject() == RejectAndGiveUp)
659 return false;
660 break;
661 }
662
663 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
664 if (handleReject() == RejectAndGiveUp)
665 return false;
666 break;
667 }
668
669 MachineMemOperand *MMO =
670 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
671
672 const TypeSize Size = MRI.getType(MO.getReg()).getSizeInBits();
673 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
674 MMO->getSizeInBits() != Size) {
675 if (handleReject() == RejectAndGiveUp)
676 return false;
677 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
678 TypeSize::isKnownGE(MMO->getSizeInBits().getValue(), Size)) {
679 if (handleReject() == RejectAndGiveUp)
680 return false;
681 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
682 TypeSize::isKnownLE(MMO->getSizeInBits().getValue(), Size))
683 if (handleReject() == RejectAndGiveUp)
684 return false;
685
686 break;
687 }
688 case GIM_RootCheckType:
689 case GIM_CheckType: {
690 uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB();
691 uint64_t OpIdx = readULEB();
692 int TypeID = readS8();
693 DEBUG_WITH_TYPE(TgtExecutor::getName(),
694 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
695 << "]->getOperand(" << OpIdx
696 << "), TypeID=" << TypeID << ")\n");
697 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
698 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
699 if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) {
700 if (handleReject() == RejectAndGiveUp)
701 return false;
702 }
703 break;
704 }
705 case GIM_CheckPointerToAny: {
706 uint64_t InsnID = readULEB();
707 uint64_t OpIdx = readULEB();
708 uint64_t SizeInBits = readULEB();
709
710 DEBUG_WITH_TYPE(TgtExecutor::getName(),
711 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
712 << InsnID << "]->getOperand(" << OpIdx
713 << "), SizeInBits=" << SizeInBits << ")\n");
714 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
715 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
716 const LLT Ty = MRI.getType(MO.getReg());
717
718 // iPTR must be looked up in the target.
719 if (SizeInBits == 0) {
720 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
721 const unsigned AddrSpace = Ty.getAddressSpace();
722 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
723 }
724
725 assert(SizeInBits != 0 && "Pointer size must be known");
726
727 if (MO.isReg()) {
728 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
729 if (handleReject() == RejectAndGiveUp)
730 return false;
731 } else if (handleReject() == RejectAndGiveUp)
732 return false;
733
734 break;
735 }
736 case GIM_RecordNamedOperand: {
737 uint64_t InsnID = readULEB();
738 uint64_t OpIdx = readULEB();
739 uint64_t StoreIdx = readULEB();
740
741 DEBUG_WITH_TYPE(TgtExecutor::getName(),
742 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
743 << InsnID << "]->getOperand(" << OpIdx
744 << "), StoreIdx=" << StoreIdx << ")\n");
745 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
746 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
747 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
748 break;
749 }
750 case GIM_RecordRegType: {
751 uint64_t InsnID = readULEB();
752 uint64_t OpIdx = readULEB();
753 int TypeIdx = readS8();
754
755 DEBUG_WITH_TYPE(TgtExecutor::getName(),
756 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
757 << InsnID << "]->getOperand(" << OpIdx
758 << "), TypeIdx=" << TypeIdx << ")\n");
759 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
760 assert(TypeIdx < 0 && "Temp types always have negative indexes!");
761 // Indexes start at -1.
762 TypeIdx = 1 - TypeIdx;
763 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);
764 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
765 State.RecordedTypes.resize(TypeIdx + 1, LLT());
766 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg());
767 break;
768 }
769
770 case GIM_RootCheckRegBankForClass:
771 case GIM_CheckRegBankForClass: {
772 uint64_t InsnID =
773 (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB();
774 uint64_t OpIdx = readULEB();
775 uint16_t RCEnum = readU16();
776 DEBUG_WITH_TYPE(TgtExecutor::getName(),
777 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
778 << InsnID << "]->getOperand(" << OpIdx
779 << "), RCEnum=" << RCEnum << ")\n");
780 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
781 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
782 if (!MO.isReg() ||
783 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
784 MRI.getType(MO.getReg())) !=
785 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
786 if (handleReject() == RejectAndGiveUp)
787 return false;
788 }
789 break;
790 }
791
792 case GIM_CheckComplexPattern: {
793 uint64_t InsnID = readULEB();
794 uint64_t OpIdx = readULEB();
795 uint16_t RendererID = readU16();
796 uint16_t ComplexPredicateID = readU16();
797 DEBUG_WITH_TYPE(TgtExecutor::getName(),
798 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
799 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
800 << "]->getOperand(" << OpIdx
801 << "), ComplexPredicateID=" << ComplexPredicateID
802 << ")\n");
803 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
804 // FIXME: Use std::invoke() when it's available.
805 ComplexRendererFns Renderer =
806 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
807 State.MIs[InsnID]->getOperand(OpIdx));
808 if (Renderer)
809 State.Renderers[RendererID] = *Renderer;
810 else if (handleReject() == RejectAndGiveUp)
811 return false;
812 break;
813 }
814
815 case GIM_CheckConstantInt:
816 case GIM_CheckConstantInt8: {
817 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
818
819 uint64_t InsnID = readULEB();
820 uint64_t OpIdx = readULEB();
821 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
822 DEBUG_WITH_TYPE(TgtExecutor::getName(),
823 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
824 << InsnID << "]->getOperand(" << OpIdx
825 << "), Value=" << Value << ")\n");
826 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
827 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
828 if (MO.isReg()) {
829 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
830 LLT Ty = MRI.getType(MO.getReg());
831 // If the type is > 64 bits, it can't be a constant int, so we bail
832 // early because SignExtend64 will assert otherwise.
833 if (Ty.getScalarSizeInBits() > 64) {
834 if (handleReject() == RejectAndGiveUp)
835 return false;
836 break;
837 }
838
839 Value = SignExtend64(Value, Ty.getScalarSizeInBits());
840 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) {
841 if (handleReject() == RejectAndGiveUp)
842 return false;
843 }
844 } else if (handleReject() == RejectAndGiveUp)
845 return false;
846
847 break;
848 }
849
850 case GIM_CheckLiteralInt: {
851 uint64_t InsnID = readULEB();
852 uint64_t OpIdx = readULEB();
853 int64_t Value = readU64();
854 DEBUG_WITH_TYPE(TgtExecutor::getName(),
855 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
856 << InsnID << "]->getOperand(" << OpIdx
857 << "), Value=" << Value << ")\n");
858 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
859 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
860 if (MO.isImm() && MO.getImm() == Value)
861 break;
862
863 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
864 break;
865
866 if (handleReject() == RejectAndGiveUp)
867 return false;
868
869 break;
870 }
871
872 case GIM_CheckIntrinsicID: {
873 uint64_t InsnID = readULEB();
874 uint64_t OpIdx = readULEB();
875 uint16_t Value = readU16();
876 DEBUG_WITH_TYPE(TgtExecutor::getName(),
877 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
878 << InsnID << "]->getOperand(" << OpIdx
879 << "), Value=" << Value << ")\n");
880 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
881 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
882 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
883 if (handleReject() == RejectAndGiveUp)
884 return false;
885 break;
886 }
887 case GIM_CheckCmpPredicate: {
888 uint64_t InsnID = readULEB();
889 uint64_t OpIdx = readULEB();
890 uint16_t Value = readU16();
891 DEBUG_WITH_TYPE(TgtExecutor::getName(),
892 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
893 << InsnID << "]->getOperand(" << OpIdx
894 << "), Value=" << Value << ")\n");
895 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
896 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
897 if (!MO.isPredicate() || MO.getPredicate() != Value)
898 if (handleReject() == RejectAndGiveUp)
899 return false;
900 break;
901 }
902 case GIM_CheckIsMBB: {
903 uint64_t InsnID = readULEB();
904 uint64_t OpIdx = readULEB();
905 DEBUG_WITH_TYPE(TgtExecutor::getName(),
906 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
907 << "]->getOperand(" << OpIdx << "))\n");
908 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
909 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
910 if (handleReject() == RejectAndGiveUp)
911 return false;
912 }
913 break;
914 }
915 case GIM_CheckIsImm: {
916 uint64_t InsnID = readULEB();
917 uint64_t OpIdx = readULEB();
918 DEBUG_WITH_TYPE(TgtExecutor::getName(),
919 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
920 << "]->getOperand(" << OpIdx << "))\n");
921 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
922 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
923 if (handleReject() == RejectAndGiveUp)
924 return false;
925 }
926 break;
927 }
928 case GIM_CheckIsSafeToFold: {
929 uint64_t NumInsn = MatchTable[CurrentIdx++];
930 DEBUG_WITH_TYPE(TgtExecutor::getName(),
931 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = "
932 << NumInsn << ")\n");
933 MachineInstr &Root = *State.MIs[0];
934 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) {
935 if (!isObviouslySafeToFold(*State.MIs[K], Root)) {
936 if (handleReject() == RejectAndGiveUp)
937 return false;
938 }
939 }
940 break;
941 }
942 case GIM_CheckIsSameOperand:
943 case GIM_CheckIsSameOperandIgnoreCopies: {
944 uint64_t InsnID = readULEB();
945 uint64_t OpIdx = readULEB();
946 uint64_t OtherInsnID = readULEB();
947 uint64_t OtherOpIdx = readULEB();
948 DEBUG_WITH_TYPE(TgtExecutor::getName(),
949 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
950 << InsnID << "][" << OpIdx << "], MIs["
951 << OtherInsnID << "][" << OtherOpIdx << "])\n");
952 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
953 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
954
955 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
956 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
957
958 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
959 if (Op.isReg() && OtherOp.isReg()) {
960 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) ==
961 getSrcRegIgnoringCopies(OtherOp.getReg(), MRI))
962 break;
963 }
964 }
965
966 if (!Op.isIdenticalTo(OtherOp)) {
967 if (handleReject() == RejectAndGiveUp)
968 return false;
969 }
970 break;
971 }
972 case GIM_CheckCanReplaceReg: {
973 uint64_t OldInsnID = readULEB();
974 uint64_t OldOpIdx = readULEB();
975 uint64_t NewInsnID = readULEB();
976 uint64_t NewOpIdx = readULEB();
977
978 DEBUG_WITH_TYPE(TgtExecutor::getName(),
979 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
980 << OldInsnID << "][" << OldOpIdx << "] = MIs["
981 << NewInsnID << "][" << NewOpIdx << "])\n");
982
983 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
984 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
985 if (!canReplaceReg(Old, New, MRI)) {
986 if (handleReject() == RejectAndGiveUp)
987 return false;
988 }
989 break;
990 }
991 case GIM_MIFlags: {
992 uint64_t InsnID = readULEB();
993 uint32_t Flags = readU32();
994
995 DEBUG_WITH_TYPE(TgtExecutor::getName(),
996 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
997 << "], " << Flags << ")\n");
998 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
999 if (handleReject() == RejectAndGiveUp)
1000 return false;
1001 }
1002 break;
1003 }
1004 case GIM_MIFlagsNot: {
1005 uint64_t InsnID = readULEB();
1006 uint32_t Flags = readU32();
1007
1008 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1009 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
1010 << "], " << Flags << ")\n");
1011 if ((State.MIs[InsnID]->getFlags() & Flags)) {
1012 if (handleReject() == RejectAndGiveUp)
1013 return false;
1014 }
1015 break;
1016 }
1017 case GIM_Reject:
1018 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1019 dbgs() << CurrentIdx << ": GIM_Reject\n");
1020 if (handleReject() == RejectAndGiveUp)
1021 return false;
1022 break;
1023 case GIR_MutateOpcode: {
1024 uint64_t OldInsnID = readULEB();
1025 uint64_t NewInsnID = readULEB();
1026 uint16_t NewOpcode = readU16();
1027 if (NewInsnID >= OutMIs.size())
1028 OutMIs.resize(NewInsnID + 1);
1029
1030 MachineInstr *OldMI = State.MIs[OldInsnID];
1031 if (Observer)
1032 Observer->changingInstr(*OldMI);
1033 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1034 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
1035 if (Observer)
1036 Observer->changedInstr(*OldMI);
1037 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1038 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1039 << NewInsnID << "], MIs[" << OldInsnID << "], "
1040 << NewOpcode << ")\n");
1041 break;
1042 }
1043
1044 case GIR_BuildRootMI:
1045 case GIR_BuildMI: {
1046 uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB();
1047 uint16_t Opcode = readU16();
1048 if (NewInsnID >= OutMIs.size())
1049 OutMIs.resize(NewInsnID + 1);
1050
1051 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1052 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1053 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1054 << NewInsnID << "], " << Opcode << ")\n");
1055 break;
1056 }
1057
1058 case GIR_BuildConstant: {
1059 uint64_t TempRegID = readULEB();
1060 uint64_t Imm = readU64();
1061 Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
1062 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1063 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1064 << TempRegID << "], Imm=" << Imm << ")\n");
1065 break;
1066 }
1067
1068 case GIR_RootToRootCopy:
1069 case GIR_Copy: {
1070 uint64_t NewInsnID =
1071 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1072 uint64_t OldInsnID =
1073 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1074 uint64_t OpIdx = readULEB();
1075 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1076 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
1077 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1078 dbgs()
1079 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1080 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1081 break;
1082 }
1083
1084 case GIR_CopyOrAddZeroReg: {
1085 uint64_t NewInsnID = readULEB();
1086 uint64_t OldInsnID = readULEB();
1087 uint64_t OpIdx = readULEB();
1088 uint16_t ZeroReg = readU16();
1089 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1090 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
1091 if (isOperandImmEqual(MO, 0, MRI))
1092 OutMIs[NewInsnID].addReg(ZeroReg);
1093 else
1094 OutMIs[NewInsnID].add(MO);
1095 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1096 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1097 << NewInsnID << "], MIs[" << OldInsnID << "], "
1098 << OpIdx << ", " << ZeroReg << ")\n");
1099 break;
1100 }
1101
1102 case GIR_CopySubReg: {
1103 uint64_t NewInsnID = readULEB();
1104 uint64_t OldInsnID = readULEB();
1105 uint64_t OpIdx = readULEB();
1106 uint16_t SubRegIdx = readU16();
1107 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1108 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
1109 0, SubRegIdx);
1110 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1111 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1112 << NewInsnID << "], MIs[" << OldInsnID << "], "
1113 << OpIdx << ", " << SubRegIdx << ")\n");
1114 break;
1115 }
1116
1117 case GIR_AddImplicitDef: {
1118 uint64_t InsnID = readULEB();
1119 uint16_t RegNum = readU16();
1120 uint16_t Flags = readU16();
1121 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1122 Flags |= RegState::Implicit;
1123 OutMIs[InsnID].addDef(RegNum, Flags);
1124 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1125 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1126 << InsnID << "], " << RegNum << ")\n");
1127 break;
1128 }
1129
1130 case GIR_AddImplicitUse: {
1131 uint64_t InsnID = readULEB();
1132 uint16_t RegNum = readU16();
1133 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1134 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
1135 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1136 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1137 << InsnID << "], " << RegNum << ")\n");
1138 break;
1139 }
1140
1141 case GIR_AddRegister: {
1142 uint64_t InsnID = readULEB();
1143 uint16_t RegNum = readU16();
1144 uint16_t RegFlags = readU16();
1145 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1146 OutMIs[InsnID].addReg(RegNum, RegFlags);
1147 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1148 dbgs()
1149 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1150 << "], " << RegNum << ", " << RegFlags << ")\n");
1151 break;
1152 }
1153 case GIR_AddIntrinsicID: {
1154 uint64_t InsnID = readULEB();
1155 uint16_t Value = readU16();
1156 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1157 OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
1158 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1159 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1160 << InsnID << "], " << Value << ")\n");
1161 break;
1162 }
1163 case GIR_SetImplicitDefDead: {
1164 uint64_t InsnID = readULEB();
1165 uint64_t OpIdx = readULEB();
1166 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1167 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1168 << InsnID << "], OpIdx=" << OpIdx << ")\n");
1169 MachineInstr *MI = OutMIs[InsnID];
1170 assert(MI && "Modifying undefined instruction");
1171 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead();
1172 break;
1173 }
1174 case GIR_SetMIFlags: {
1175 uint64_t InsnID = readULEB();
1176 uint32_t Flags = readU32();
1177
1178 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1179 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1180 << InsnID << "], " << Flags << ")\n");
1181 MachineInstr *MI = OutMIs[InsnID];
1182 MI->setFlags(MI->getFlags() | Flags);
1183 break;
1184 }
1185 case GIR_UnsetMIFlags: {
1186 uint64_t InsnID = readULEB();
1187 uint32_t Flags = readU32();
1188
1189 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1190 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1191 << InsnID << "], " << Flags << ")\n");
1192 MachineInstr *MI = OutMIs[InsnID];
1193 MI->setFlags(MI->getFlags() & ~Flags);
1194 break;
1195 }
1196 case GIR_CopyMIFlags: {
1197 uint64_t InsnID = readULEB();
1198 uint64_t OldInsnID = readULEB();
1199
1200 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1201 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1202 << InsnID << "], MIs[" << OldInsnID << "])\n");
1203 MachineInstr *MI = OutMIs[InsnID];
1204 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1205 break;
1206 }
1207 case GIR_AddSimpleTempRegister:
1208 case GIR_AddTempRegister:
1209 case GIR_AddTempSubRegister: {
1210 uint64_t InsnID = readULEB();
1211 uint64_t TempRegID = readULEB();
1212 uint16_t TempRegFlags = 0;
1213 if (MatcherOpcode != GIR_AddSimpleTempRegister)
1214 TempRegFlags = readU16();
1215 uint16_t SubReg = 0;
1216 if (MatcherOpcode == GIR_AddTempSubRegister)
1217 SubReg = readU16();
1218
1219 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1220
1221 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,
1222 SubReg);
1223 DEBUG_WITH_TYPE(
1224 TgtExecutor::getName(),
1225 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1226 << "], TempRegisters[" << TempRegID << "]";
1227 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1228 dbgs() << ", " << TempRegFlags << ")\n");
1229 break;
1230 }
1231
1232 case GIR_AddImm8:
1233 case GIR_AddImm: {
1234 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1235 uint64_t InsnID = readULEB();
1236 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1237 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1238 OutMIs[InsnID].addImm(Imm);
1239 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1240 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1241 << "], " << Imm << ")\n");
1242 break;
1243 }
1244
1245 case GIR_AddCImm: {
1246 uint64_t InsnID = readULEB();
1247 int TypeID = readS8();
1248 uint64_t Imm = readU64();
1249 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1250
1251 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1252 LLVMContext &Ctx = MF->getFunction().getContext();
1253 OutMIs[InsnID].addCImm(
1254 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true));
1255 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1256 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1257 << "], TypeID=" << TypeID << ", Imm=" << Imm
1258 << ")\n");
1259 break;
1260 }
1261
1262 case GIR_ComplexRenderer: {
1263 uint64_t InsnID = readULEB();
1264 uint16_t RendererID = readU16();
1265 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1266 for (const auto &RenderOpFn : State.Renderers[RendererID])
1267 RenderOpFn(OutMIs[InsnID]);
1268 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1269 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1270 << InsnID << "], " << RendererID << ")\n");
1271 break;
1272 }
1273 case GIR_ComplexSubOperandRenderer: {
1274 uint64_t InsnID = readULEB();
1275 uint16_t RendererID = readU16();
1276 uint64_t RenderOpID = readULEB();
1277 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1278 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1279 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1280 dbgs() << CurrentIdx
1281 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1282 << InsnID << "], " << RendererID << ", "
1283 << RenderOpID << ")\n");
1284 break;
1285 }
1286 case GIR_ComplexSubOperandSubRegRenderer: {
1287 uint64_t InsnID = readULEB();
1288 uint16_t RendererID = readU16();
1289 uint64_t RenderOpID = readULEB();
1290 uint16_t SubRegIdx = readU16();
1291 MachineInstrBuilder &MI = OutMIs[InsnID];
1292 assert(MI && "Attempted to add to undefined instruction");
1293 State.Renderers[RendererID][RenderOpID](MI);
1294 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1295 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1296 dbgs() << CurrentIdx
1297 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1298 << InsnID << "], " << RendererID << ", "
1299 << RenderOpID << ", " << SubRegIdx << ")\n");
1300 break;
1301 }
1302
1303 case GIR_CopyConstantAsSImm: {
1304 uint64_t NewInsnID = readULEB();
1305 uint64_t OldInsnID = readULEB();
1306 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1307 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1308 "Expected G_CONSTANT");
1309 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1310 OutMIs[NewInsnID].addImm(
1311 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1312 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1313 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1314 else
1315 llvm_unreachable("Expected Imm or CImm operand");
1316 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1317 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1318 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1319 break;
1320 }
1321
1322 // TODO: Needs a test case once we have a pattern that uses this.
1323 case GIR_CopyFConstantAsFPImm: {
1324 uint64_t NewInsnID = readULEB();
1325 uint64_t OldInsnID = readULEB();
1326 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1327 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1328 "Expected G_FCONSTANT");
1329 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1330 OutMIs[NewInsnID].addFPImm(
1331 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1332 else
1333 llvm_unreachable("Expected FPImm operand");
1334 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1335 dbgs()
1336 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1337 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1338 break;
1339 }
1340
1341 case GIR_CustomRenderer: {
1342 uint64_t InsnID = readULEB();
1343 uint64_t OldInsnID = readULEB();
1344 uint16_t RendererFnID = readU16();
1345 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1346 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1347 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1348 << InsnID << "], MIs[" << OldInsnID << "], "
1349 << RendererFnID << ")\n");
1350 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1351 OutMIs[InsnID], *State.MIs[OldInsnID],
1352 -1); // Not a source operand of the old instruction.
1353 break;
1354 }
1355 case GIR_DoneWithCustomAction: {
1356 uint16_t FnID = readU16();
1357 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1358 dbgs() << CurrentIdx << ": GIR_DoneWithCustomAction(FnID="
1359 << FnID << ")\n");
1360 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1361 if (runCustomAction(FnID, State, OutMIs)) {
1362 propagateFlags();
1363 return true;
1364 }
1365
1366 if (handleReject() == RejectAndGiveUp)
1367 return false;
1368 break;
1369 }
1370 case GIR_CustomOperandRenderer: {
1371 uint64_t InsnID = readULEB();
1372 uint64_t OldInsnID = readULEB();
1373 uint64_t OpIdx = readULEB();
1374 uint16_t RendererFnID = readU16();
1375 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1376
1377 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1378 dbgs() << CurrentIdx
1379 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1380 << "], MIs[" << OldInsnID << "]->getOperand("
1381 << OpIdx << "), " << RendererFnID << ")\n");
1382 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1383 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1384 break;
1385 }
1386 case GIR_ConstrainOperandRC: {
1387 uint64_t InsnID = readULEB();
1388 uint64_t OpIdx = readULEB();
1389 uint16_t RCEnum = readU16();
1390 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1391 MachineInstr &I = *OutMIs[InsnID].getInstr();
1392 MachineFunction &MF = *I.getParent()->getParent();
1393 MachineRegisterInfo &MRI = MF.getRegInfo();
1394 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1395 MachineOperand &MO = I.getOperand(OpIdx);
1396 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1397 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1398 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1399 << InsnID << "], " << OpIdx << ", " << RCEnum
1400 << ")\n");
1401 break;
1402 }
1403
1404 case GIR_RootConstrainSelectedInstOperands:
1405 case GIR_ConstrainSelectedInstOperands: {
1406 uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands)
1407 ? 0
1408 : readULEB();
1409 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1410 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1411 RBI);
1412 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1413 dbgs() << CurrentIdx
1414 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1415 << InsnID << "])\n");
1416 break;
1417 }
1418 case GIR_MergeMemOperands: {
1419 uint64_t InsnID = readULEB();
1420 uint64_t NumInsn = MatchTable[CurrentIdx++];
1421 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1422
1423 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1424 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1425 << InsnID << "]");
1426 for (unsigned K = 0; K < NumInsn; ++K) {
1427 uint64_t NextID = readULEB();
1428 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1429 dbgs() << ", MIs[" << NextID << "]");
1430 for (const auto &MMO : State.MIs[NextID]->memoperands())
1431 OutMIs[InsnID].addMemOperand(MMO);
1432 }
1433 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1434 break;
1435 }
1436 case GIR_EraseFromParent: {
1437 uint64_t InsnID = readULEB();
1438 MachineInstr *MI = State.MIs[InsnID];
1439 assert(MI && "Attempted to erase an undefined instruction");
1440 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1441 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1442 << InsnID << "])\n");
1443 eraseImpl(MI);
1444 break;
1445 }
1446 case GIR_EraseRootFromParent_Done: {
1447 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1448 dbgs()
1449 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n");
1450 eraseImpl(State.MIs[0]);
1451 propagateFlags();
1452 return true;
1453 }
1454 case GIR_MakeTempReg: {
1455 uint64_t TempRegID = readULEB();
1456 int TypeID = readS8();
1457
1458 State.TempRegisters[TempRegID] =
1459 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));
1460 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1461 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1462 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1463 break;
1464 }
1465 case GIR_ReplaceReg: {
1466 uint64_t OldInsnID = readULEB();
1467 uint64_t OldOpIdx = readULEB();
1468 uint64_t NewInsnID = readULEB();
1469 uint64_t NewOpIdx = readULEB();
1470
1471 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1472 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1473 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1474 << NewInsnID << "][" << NewOpIdx << "])\n");
1475
1476 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1477 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1478 if (Observer)
1479 Observer->changingAllUsesOfReg(MRI, Old);
1480 MRI.replaceRegWith(Old, New);
1481 if (Observer)
1482 Observer->finishedChangingAllUsesOfReg();
1483 break;
1484 }
1485 case GIR_ReplaceRegWithTempReg: {
1486 uint64_t OldInsnID = readULEB();
1487 uint64_t OldOpIdx = readULEB();
1488 uint64_t TempRegID = readULEB();
1489
1490 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1491 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1492 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1493 << TempRegID << "])\n");
1494
1495 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1496 Register New = State.TempRegisters[TempRegID];
1497 if (Observer)
1498 Observer->changingAllUsesOfReg(MRI, Old);
1499 MRI.replaceRegWith(Old, New);
1500 if (Observer)
1501 Observer->finishedChangingAllUsesOfReg();
1502 break;
1503 }
1504 case GIR_Coverage: {
1505 uint32_t RuleID = readU32();
1506 assert(CoverageInfo);
1507 CoverageInfo->setCovered(RuleID);
1508
1509 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1510 << ": GIR_Coverage("
1511 << RuleID << ")");
1512 break;
1513 }
1514
1515 case GIR_Done:
1516 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1517 dbgs() << CurrentIdx << ": GIR_Done\n");
1518 propagateFlags();
1519 return true;
1520 default:
1521 llvm_unreachable("Unexpected command");
1522 }
1523 }
1524 }
1525
1526 } // end namespace llvm
1527
1528 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1529