1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10 /// function. This is implemented in a separate file because the function is
11 /// quite large.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterBankInfo.h"
27 #include "llvm/CodeGen/TargetInstrInfo.h"
28 #include "llvm/CodeGen/TargetOpcodes.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/IR/Constants.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/Support/CodeGenCoverage.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/LEB128.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include <cassert>
39 #include <cstddef>
40 #include <cstdint>
41
42 namespace llvm {
43
44 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45 class CustomRendererFn>
executeMatchTable(TgtExecutor & Exec,MatcherState & State,const ExecInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ExecInfo,MachineIRBuilder & Builder,const uint8_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage * CoverageInfo)46 bool GIMatchTableExecutor::executeMatchTable(
47 TgtExecutor &Exec, MatcherState &State,
48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
49 &ExecInfo,
50 MachineIRBuilder &Builder, const uint8_t *MatchTable,
51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53 const PredicateBitset &AvailableFeatures,
54 CodeGenCoverage *CoverageInfo) const {
55
56 uint64_t CurrentIdx = 0;
57 SmallVector<uint64_t, 4> OnFailResumeAt;
58 NewMIVector OutMIs;
59
60 GISelChangeObserver *Observer = Builder.getObserver();
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint32_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtExecutor::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtExecutor::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 const auto propagateFlags = [&]() {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint32_t MIBFlags = Flags | MIB.getInstr()->getFlags();
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 if (Observer)
87 Observer->changingInstr(*MIB);
88 MIB.setMIFlags(MIBFlags);
89 if (Observer)
90 Observer->changedInstr(*MIB);
91 }
92 };
93
94 // If the index is >= 0, it's an index in the type objects generated by
95 // TableGen. If the index is <0, it's an index in the recorded types object.
96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97 if (Idx >= 0)
98 return ExecInfo.TypeObjects[Idx];
99 return State.RecordedTypes[1 - Idx];
100 };
101
102 const auto readULEB = [&]() {
103 return fastDecodeULEB128(MatchTable, CurrentIdx);
104 };
105
106 // Convenience function to return a signed value. This avoids
107 // us forgetting to first cast to int8_t before casting to a
108 // wider signed int type.
109 // if we casted uint8 directly to a wider type we'd lose
110 // negative values.
111 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
112
113 const auto readU16 = [&]() {
114 auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx);
115 CurrentIdx += 2;
116 return V;
117 };
118
119 const auto readU32 = [&]() {
120 auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx);
121 CurrentIdx += 4;
122 return V;
123 };
124
125 const auto readU64 = [&]() {
126 auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx);
127 CurrentIdx += 8;
128 return V;
129 };
130
131 const auto eraseImpl = [&](MachineInstr *MI) {
132 // If we're erasing the insertion point, ensure we don't leave a dangling
133 // pointer in the builder.
134 if (Builder.getInsertPt() == MI)
135 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());
136 if (Observer)
137 Observer->erasingInstr(*MI);
138 MI->eraseFromParent();
139 };
140
141 while (true) {
142 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
143 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
144 switch (MatcherOpcode) {
145 case GIM_Try: {
146 DEBUG_WITH_TYPE(TgtExecutor::getName(),
147 dbgs() << CurrentIdx << ": Begin try-block\n");
148 OnFailResumeAt.push_back(readU32());
149 break;
150 }
151
152 case GIM_RecordInsn:
153 case GIM_RecordInsnIgnoreCopies: {
154 uint64_t NewInsnID = readULEB();
155 uint64_t InsnID = readULEB();
156 uint64_t OpIdx = readULEB();
157
158 // As an optimisation we require that MIs[0] is always the root. Refuse
159 // any attempt to modify it.
160 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
161
162 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
163 if (!MO.isReg()) {
164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
165 dbgs() << CurrentIdx << ": Not a register\n");
166 if (handleReject() == RejectAndGiveUp)
167 return false;
168 break;
169 }
170 if (MO.getReg().isPhysical()) {
171 DEBUG_WITH_TYPE(TgtExecutor::getName(),
172 dbgs() << CurrentIdx << ": Is a physical register\n");
173 if (handleReject() == RejectAndGiveUp)
174 return false;
175 break;
176 }
177
178 MachineInstr *NewMI;
179 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
180 NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
181 else
182 NewMI = MRI.getVRegDef(MO.getReg());
183
184 if ((size_t)NewInsnID < State.MIs.size())
185 State.MIs[NewInsnID] = NewMI;
186 else {
187 assert((size_t)NewInsnID == State.MIs.size() &&
188 "Expected to store MIs in order");
189 State.MIs.push_back(NewMI);
190 }
191 DEBUG_WITH_TYPE(TgtExecutor::getName(),
192 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
193 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
194 << ")\n");
195 break;
196 }
197
198 case GIM_CheckFeatures: {
199 uint16_t ExpectedBitsetID = readU16();
200 DEBUG_WITH_TYPE(TgtExecutor::getName(),
201 dbgs() << CurrentIdx
202 << ": GIM_CheckFeatures(ExpectedBitsetID="
203 << ExpectedBitsetID << ")\n");
204 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
205 ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
206 if (handleReject() == RejectAndGiveUp)
207 return false;
208 }
209 break;
210 }
211 case GIM_CheckOpcode:
212 case GIM_CheckOpcodeIsEither: {
213 uint64_t InsnID = readULEB();
214 uint16_t Expected0 = readU16();
215 uint16_t Expected1 = -1;
216 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
217 Expected1 = readU16();
218
219 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
220 unsigned Opcode = State.MIs[InsnID]->getOpcode();
221
222 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
223 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
224 << "], ExpectedOpcode=" << Expected0;
225 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
226 dbgs() << " || " << Expected1;
227 dbgs() << ") // Got=" << Opcode << "\n";
228 });
229
230 if (Opcode != Expected0 && Opcode != Expected1) {
231 if (handleReject() == RejectAndGiveUp)
232 return false;
233 }
234 break;
235 }
236 case GIM_SwitchOpcode: {
237 uint64_t InsnID = readULEB();
238 uint16_t LowerBound = readU16();
239 uint16_t UpperBound = readU16();
240 uint32_t Default = readU32();
241
242 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
243 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
244
245 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
246 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
247 << LowerBound << ", " << UpperBound << "), Default=" << Default
248 << ", JumpTable...) // Got=" << Opcode << "\n";
249 });
250 if (Opcode < LowerBound || UpperBound <= Opcode) {
251 CurrentIdx = Default;
252 break;
253 }
254 const auto EntryIdx = (Opcode - LowerBound);
255 // Each entry is 4 bytes
256 CurrentIdx =
257 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4));
258 if (!CurrentIdx) {
259 CurrentIdx = Default;
260 break;
261 }
262 OnFailResumeAt.push_back(Default);
263 break;
264 }
265
266 case GIM_SwitchType: {
267 uint64_t InsnID = readULEB();
268 uint64_t OpIdx = readULEB();
269 uint16_t LowerBound = readU16();
270 uint16_t UpperBound = readU16();
271 int64_t Default = readU32();
272
273 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
274 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
275
276 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
277 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
278 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
279 << UpperBound << "), Default=" << Default
280 << ", JumpTable...) // Got=";
281 if (!MO.isReg())
282 dbgs() << "Not a VReg\n";
283 else
284 dbgs() << MRI.getType(MO.getReg()) << "\n";
285 });
286 if (!MO.isReg()) {
287 CurrentIdx = Default;
288 break;
289 }
290 const LLT Ty = MRI.getType(MO.getReg());
291 const auto TyI = ExecInfo.TypeIDMap.find(Ty);
292 if (TyI == ExecInfo.TypeIDMap.end()) {
293 CurrentIdx = Default;
294 break;
295 }
296 const int64_t TypeID = TyI->second;
297 if (TypeID < LowerBound || UpperBound <= TypeID) {
298 CurrentIdx = Default;
299 break;
300 }
301 const auto NumEntry = (TypeID - LowerBound);
302 // Each entry is 4 bytes
303 CurrentIdx =
304 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4));
305 if (!CurrentIdx) {
306 CurrentIdx = Default;
307 break;
308 }
309 OnFailResumeAt.push_back(Default);
310 break;
311 }
312
313 case GIM_CheckNumOperandsGE:
314 case GIM_CheckNumOperandsLE: {
315 uint64_t InsnID = readULEB();
316 uint64_t Expected = readULEB();
317 const bool IsLE = (MatcherOpcode == GIM_CheckNumOperandsLE);
318 DEBUG_WITH_TYPE(TgtExecutor::getName(),
319 dbgs() << CurrentIdx << ": GIM_CheckNumOperands"
320 << (IsLE ? "LE" : "GE") << "(MIs[" << InsnID
321 << "], Expected=" << Expected << ")\n");
322 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
323 const unsigned NumOps = State.MIs[InsnID]->getNumOperands();
324 if (IsLE ? (NumOps > Expected) : (NumOps < Expected)) {
325 if (handleReject() == RejectAndGiveUp)
326 return false;
327 }
328 break;
329 }
330 case GIM_CheckNumOperands: {
331 uint64_t InsnID = readULEB();
332 uint64_t Expected = readULEB();
333 DEBUG_WITH_TYPE(TgtExecutor::getName(),
334 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
335 << InsnID << "], Expected=" << Expected << ")\n");
336 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
337 if (State.MIs[InsnID]->getNumOperands() != Expected) {
338 if (handleReject() == RejectAndGiveUp)
339 return false;
340 }
341 break;
342 }
343 case GIM_CheckI64ImmPredicate:
344 case GIM_CheckImmOperandPredicate: {
345 uint64_t InsnID = readULEB();
346 unsigned OpIdx =
347 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
348 uint16_t Predicate = readU16();
349 DEBUG_WITH_TYPE(TgtExecutor::getName(),
350 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
351 << InsnID << "]->getOperand(" << OpIdx
352 << "), Predicate=" << Predicate << ")\n");
353 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
354 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
355 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
356 "Expected immediate operand");
357 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
358 int64_t Value = 0;
359 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
360 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
361 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
362 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
363 else
364 llvm_unreachable("Expected Imm or CImm operand");
365
366 if (!testImmPredicate_I64(Predicate, Value))
367 if (handleReject() == RejectAndGiveUp)
368 return false;
369 break;
370 }
371 case GIM_CheckAPIntImmPredicate: {
372 uint64_t InsnID = readULEB();
373 uint16_t Predicate = readU16();
374 DEBUG_WITH_TYPE(TgtExecutor::getName(),
375 dbgs()
376 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
377 << InsnID << "], Predicate=" << Predicate << ")\n");
378 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
379 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
380 "Expected G_CONSTANT");
381 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
382 if (!State.MIs[InsnID]->getOperand(1).isCImm())
383 llvm_unreachable("Expected Imm or CImm operand");
384
385 const APInt &Value =
386 State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
387 if (!testImmPredicate_APInt(Predicate, Value))
388 if (handleReject() == RejectAndGiveUp)
389 return false;
390 break;
391 }
392 case GIM_CheckAPFloatImmPredicate: {
393 uint64_t InsnID = readULEB();
394 uint16_t Predicate = readU16();
395 DEBUG_WITH_TYPE(TgtExecutor::getName(),
396 dbgs()
397 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
398 << InsnID << "], Predicate=" << Predicate << ")\n");
399 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
400 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
401 "Expected G_FCONSTANT");
402 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
403 "Expected FPImm operand");
404 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
405 const APFloat &Value =
406 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
407
408 if (!testImmPredicate_APFloat(Predicate, Value))
409 if (handleReject() == RejectAndGiveUp)
410 return false;
411 break;
412 }
413 case GIM_CheckLeafOperandPredicate: {
414 uint64_t InsnID = readULEB();
415 uint64_t OpIdx = readULEB();
416 uint16_t Predicate = readU16();
417 DEBUG_WITH_TYPE(TgtExecutor::getName(),
418 dbgs() << CurrentIdx
419 << ": GIM_CheckLeafOperandPredicate(MIs[" << InsnID
420 << "]->getOperand(" << OpIdx
421 << "), Predicate=" << Predicate << ")\n");
422 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423 assert(State.MIs[InsnID]->getOperand(OpIdx).isReg() &&
424 "Expected register operand");
425 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
426 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
427
428 if (!testMOPredicate_MO(Predicate, MO, State))
429 if (handleReject() == RejectAndGiveUp)
430 return false;
431 break;
432 }
433 case GIM_CheckIsBuildVectorAllOnes:
434 case GIM_CheckIsBuildVectorAllZeros: {
435 uint64_t InsnID = readULEB();
436
437 DEBUG_WITH_TYPE(TgtExecutor::getName(),
438 dbgs() << CurrentIdx
439 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
440 << InsnID << "])\n");
441 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
442
443 const MachineInstr *MI = State.MIs[InsnID];
444 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
445 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
446 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
447
448 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
449 if (!isBuildVectorAllOnes(*MI, MRI)) {
450 if (handleReject() == RejectAndGiveUp)
451 return false;
452 }
453 } else {
454 if (!isBuildVectorAllZeros(*MI, MRI)) {
455 if (handleReject() == RejectAndGiveUp)
456 return false;
457 }
458 }
459
460 break;
461 }
462 case GIM_CheckSimplePredicate: {
463 // Note: we don't check for invalid here because this is purely a hook to
464 // allow some executors (such as the combiner) to check arbitrary,
465 // contextless predicates, such as whether a rule is enabled or not.
466 uint16_t Predicate = readU16();
467 DEBUG_WITH_TYPE(TgtExecutor::getName(),
468 dbgs() << CurrentIdx
469 << ": GIM_CheckSimplePredicate(Predicate="
470 << Predicate << ")\n");
471 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
472 if (!testSimplePredicate(Predicate)) {
473 if (handleReject() == RejectAndGiveUp)
474 return false;
475 }
476 break;
477 }
478 case GIM_CheckCxxInsnPredicate: {
479 uint64_t InsnID = readULEB();
480 uint16_t Predicate = readU16();
481 DEBUG_WITH_TYPE(TgtExecutor::getName(),
482 dbgs()
483 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
484 << InsnID << "], Predicate=" << Predicate << ")\n");
485 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
486 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
487
488 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
489 if (handleReject() == RejectAndGiveUp)
490 return false;
491 break;
492 }
493 case GIM_CheckHasNoUse: {
494 uint64_t InsnID = readULEB();
495
496 DEBUG_WITH_TYPE(TgtExecutor::getName(),
497 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
498 << InsnID << "]\n");
499
500 const MachineInstr *MI = State.MIs[InsnID];
501 assert(MI && "Used insn before defined");
502 assert(MI->getNumDefs() > 0 && "No defs");
503 const Register Res = MI->getOperand(0).getReg();
504
505 if (!MRI.use_nodbg_empty(Res)) {
506 if (handleReject() == RejectAndGiveUp)
507 return false;
508 }
509 break;
510 }
511 case GIM_CheckHasOneUse: {
512 uint64_t InsnID = readULEB();
513
514 DEBUG_WITH_TYPE(TgtExecutor::getName(),
515 dbgs() << CurrentIdx << ": GIM_CheckHasOneUse(MIs["
516 << InsnID << "]\n");
517
518 const MachineInstr *MI = State.MIs[InsnID];
519 assert(MI && "Used insn before defined");
520 assert(MI->getNumDefs() > 0 && "No defs");
521 const Register Res = MI->getOperand(0).getReg();
522
523 if (!MRI.hasOneNonDBGUse(Res)) {
524 if (handleReject() == RejectAndGiveUp)
525 return false;
526 }
527 break;
528 }
529 case GIM_CheckAtomicOrdering: {
530 uint64_t InsnID = readULEB();
531 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
532 DEBUG_WITH_TYPE(TgtExecutor::getName(),
533 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
534 << InsnID << "], " << (uint64_t)Ordering << ")\n");
535 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
536 if (!State.MIs[InsnID]->hasOneMemOperand())
537 if (handleReject() == RejectAndGiveUp)
538 return false;
539
540 for (const auto &MMO : State.MIs[InsnID]->memoperands())
541 if (MMO->getMergedOrdering() != Ordering)
542 if (handleReject() == RejectAndGiveUp)
543 return false;
544 break;
545 }
546 case GIM_CheckAtomicOrderingOrStrongerThan: {
547 uint64_t InsnID = readULEB();
548 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
549 DEBUG_WITH_TYPE(TgtExecutor::getName(),
550 dbgs() << CurrentIdx
551 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
552 << InsnID << "], " << (uint64_t)Ordering << ")\n");
553 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
554 if (!State.MIs[InsnID]->hasOneMemOperand())
555 if (handleReject() == RejectAndGiveUp)
556 return false;
557
558 for (const auto &MMO : State.MIs[InsnID]->memoperands())
559 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
560 if (handleReject() == RejectAndGiveUp)
561 return false;
562 break;
563 }
564 case GIM_CheckAtomicOrderingWeakerThan: {
565 uint64_t InsnID = readULEB();
566 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
567 DEBUG_WITH_TYPE(TgtExecutor::getName(),
568 dbgs() << CurrentIdx
569 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
570 << InsnID << "], " << (uint64_t)Ordering << ")\n");
571 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572 if (!State.MIs[InsnID]->hasOneMemOperand())
573 if (handleReject() == RejectAndGiveUp)
574 return false;
575
576 for (const auto &MMO : State.MIs[InsnID]->memoperands())
577 if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
578 if (handleReject() == RejectAndGiveUp)
579 return false;
580 break;
581 }
582 case GIM_CheckMemoryAddressSpace: {
583 uint64_t InsnID = readULEB();
584 uint64_t MMOIdx = readULEB();
585 // This accepts a list of possible address spaces.
586 const uint64_t NumAddrSpace = MatchTable[CurrentIdx++];
587
588 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
589 if (handleReject() == RejectAndGiveUp)
590 return false;
591 break;
592 }
593
594 // Need to still jump to the end of the list of address spaces if we find
595 // a match earlier.
596 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
597
598 const MachineMemOperand *MMO =
599 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
600 const unsigned MMOAddrSpace = MMO->getAddrSpace();
601
602 bool Success = false;
603 for (unsigned I = 0; I != NumAddrSpace; ++I) {
604 uint64_t AddrSpace = readULEB();
605 DEBUG_WITH_TYPE(TgtExecutor::getName(),
606 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
607 << AddrSpace << '\n');
608
609 if (AddrSpace == MMOAddrSpace) {
610 Success = true;
611 break;
612 }
613 }
614
615 CurrentIdx = LastIdx;
616 if (!Success && handleReject() == RejectAndGiveUp)
617 return false;
618 break;
619 }
620 case GIM_CheckMemoryAlignment: {
621 uint64_t InsnID = readULEB();
622 uint64_t MMOIdx = readULEB();
623 uint64_t MinAlign = MatchTable[CurrentIdx++];
624
625 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
626
627 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
628 if (handleReject() == RejectAndGiveUp)
629 return false;
630 break;
631 }
632
633 MachineMemOperand *MMO =
634 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
635 DEBUG_WITH_TYPE(TgtExecutor::getName(),
636 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
637 << "(MIs[" << InsnID << "]->memoperands() + "
638 << MMOIdx << ")->getAlignment() >= " << MinAlign
639 << ")\n");
640 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
641 return false;
642
643 break;
644 }
645 case GIM_CheckMemorySizeEqualTo: {
646 uint64_t InsnID = readULEB();
647 uint64_t MMOIdx = readULEB();
648 uint32_t Size = readU32();
649
650 DEBUG_WITH_TYPE(TgtExecutor::getName(),
651 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
652 << InsnID << "]->memoperands() + " << MMOIdx
653 << ", Size=" << Size << ")\n");
654 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
655
656 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
657 if (handleReject() == RejectAndGiveUp)
658 return false;
659 break;
660 }
661
662 MachineMemOperand *MMO =
663 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
664
665 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
666 << " bytes vs " << Size
667 << " bytes\n");
668 if (MMO->getSize() != Size)
669 if (handleReject() == RejectAndGiveUp)
670 return false;
671
672 break;
673 }
674 case GIM_CheckMemorySizeEqualToLLT:
675 case GIM_CheckMemorySizeLessThanLLT:
676 case GIM_CheckMemorySizeGreaterThanLLT: {
677 uint64_t InsnID = readULEB();
678 uint64_t MMOIdx = readULEB();
679 uint64_t OpIdx = readULEB();
680
681 DEBUG_WITH_TYPE(
682 TgtExecutor::getName(),
683 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
684 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
685 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
686 ? "GreaterThan"
687 : "LessThan")
688 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
689 << ", OpIdx=" << OpIdx << ")\n");
690 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
691
692 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
693 if (!MO.isReg()) {
694 DEBUG_WITH_TYPE(TgtExecutor::getName(),
695 dbgs() << CurrentIdx << ": Not a register\n");
696 if (handleReject() == RejectAndGiveUp)
697 return false;
698 break;
699 }
700
701 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
702 if (handleReject() == RejectAndGiveUp)
703 return false;
704 break;
705 }
706
707 MachineMemOperand *MMO =
708 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
709
710 const TypeSize Size = MRI.getType(MO.getReg()).getSizeInBits();
711 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
712 MMO->getSizeInBits() != Size) {
713 if (handleReject() == RejectAndGiveUp)
714 return false;
715 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
716 TypeSize::isKnownGE(MMO->getSizeInBits().getValue(), Size)) {
717 if (handleReject() == RejectAndGiveUp)
718 return false;
719 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
720 TypeSize::isKnownLE(MMO->getSizeInBits().getValue(), Size))
721 if (handleReject() == RejectAndGiveUp)
722 return false;
723
724 break;
725 }
726 case GIM_RootCheckType:
727 case GIM_CheckType: {
728 uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB();
729 uint64_t OpIdx = readULEB();
730 int TypeID = readS8();
731 DEBUG_WITH_TYPE(TgtExecutor::getName(),
732 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
733 << "]->getOperand(" << OpIdx
734 << "), TypeID=" << TypeID << ")\n");
735 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
736 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
737 if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) {
738 if (handleReject() == RejectAndGiveUp)
739 return false;
740 }
741 break;
742 }
743 case GIM_CheckPointerToAny: {
744 uint64_t InsnID = readULEB();
745 uint64_t OpIdx = readULEB();
746 uint64_t SizeInBits = readULEB();
747
748 DEBUG_WITH_TYPE(TgtExecutor::getName(),
749 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
750 << InsnID << "]->getOperand(" << OpIdx
751 << "), SizeInBits=" << SizeInBits << ")\n");
752 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
753 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
754 const LLT Ty = MRI.getType(MO.getReg());
755
756 // iPTR must be looked up in the target.
757 if (SizeInBits == 0) {
758 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
759 const unsigned AddrSpace = Ty.getAddressSpace();
760 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
761 }
762
763 assert(SizeInBits != 0 && "Pointer size must be known");
764
765 if (MO.isReg()) {
766 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
767 if (handleReject() == RejectAndGiveUp)
768 return false;
769 } else if (handleReject() == RejectAndGiveUp)
770 return false;
771
772 break;
773 }
774 case GIM_RecordNamedOperand: {
775 uint64_t InsnID = readULEB();
776 uint64_t OpIdx = readULEB();
777 uint64_t StoreIdx = readULEB();
778
779 DEBUG_WITH_TYPE(TgtExecutor::getName(),
780 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
781 << InsnID << "]->getOperand(" << OpIdx
782 << "), StoreIdx=" << StoreIdx << ")\n");
783 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
784 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
785 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
786 break;
787 }
788 case GIM_RecordRegType: {
789 uint64_t InsnID = readULEB();
790 uint64_t OpIdx = readULEB();
791 int TypeIdx = readS8();
792
793 DEBUG_WITH_TYPE(TgtExecutor::getName(),
794 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
795 << InsnID << "]->getOperand(" << OpIdx
796 << "), TypeIdx=" << TypeIdx << ")\n");
797 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
798 assert(TypeIdx < 0 && "Temp types always have negative indexes!");
799 // Indexes start at -1.
800 TypeIdx = 1 - TypeIdx;
801 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);
802 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
803 State.RecordedTypes.resize(TypeIdx + 1, LLT());
804 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg());
805 break;
806 }
807
808 case GIM_RootCheckRegBankForClass:
809 case GIM_CheckRegBankForClass: {
810 uint64_t InsnID =
811 (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB();
812 uint64_t OpIdx = readULEB();
813 uint16_t RCEnum = readU16();
814 DEBUG_WITH_TYPE(TgtExecutor::getName(),
815 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
816 << InsnID << "]->getOperand(" << OpIdx
817 << "), RCEnum=" << RCEnum << ")\n");
818 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
819 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
820 if (!MO.isReg() ||
821 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
822 MRI.getType(MO.getReg())) !=
823 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
824 if (handleReject() == RejectAndGiveUp)
825 return false;
826 }
827 break;
828 }
829
830 case GIM_CheckComplexPattern: {
831 uint64_t InsnID = readULEB();
832 uint64_t OpIdx = readULEB();
833 uint16_t RendererID = readU16();
834 uint16_t ComplexPredicateID = readU16();
835 DEBUG_WITH_TYPE(TgtExecutor::getName(),
836 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
837 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
838 << "]->getOperand(" << OpIdx
839 << "), ComplexPredicateID=" << ComplexPredicateID
840 << ")\n");
841 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
842 // FIXME: Use std::invoke() when it's available.
843 ComplexRendererFns Renderer =
844 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
845 State.MIs[InsnID]->getOperand(OpIdx));
846 if (Renderer)
847 State.Renderers[RendererID] = *Renderer;
848 else if (handleReject() == RejectAndGiveUp)
849 return false;
850 break;
851 }
852
853 case GIM_CheckConstantInt:
854 case GIM_CheckConstantInt8: {
855 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
856
857 uint64_t InsnID = readULEB();
858 uint64_t OpIdx = readULEB();
859 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
860 DEBUG_WITH_TYPE(TgtExecutor::getName(),
861 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
862 << InsnID << "]->getOperand(" << OpIdx
863 << "), Value=" << Value << ")\n");
864 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
865 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
866 if (MO.isReg()) {
867 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
868 LLT Ty = MRI.getType(MO.getReg());
869 // If the type is > 64 bits, it can't be a constant int, so we bail
870 // early because SignExtend64 will assert otherwise.
871 if (Ty.getScalarSizeInBits() > 64) {
872 if (handleReject() == RejectAndGiveUp)
873 return false;
874 break;
875 }
876
877 Value = SignExtend64(Value, Ty.getScalarSizeInBits());
878 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) {
879 if (handleReject() == RejectAndGiveUp)
880 return false;
881 }
882 } else if (handleReject() == RejectAndGiveUp)
883 return false;
884
885 break;
886 }
887
888 case GIM_CheckLiteralInt: {
889 uint64_t InsnID = readULEB();
890 uint64_t OpIdx = readULEB();
891 int64_t Value = readU64();
892 DEBUG_WITH_TYPE(TgtExecutor::getName(),
893 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
894 << InsnID << "]->getOperand(" << OpIdx
895 << "), Value=" << Value << ")\n");
896 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
897 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
898 if (MO.isImm() && MO.getImm() == Value)
899 break;
900
901 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
902 break;
903
904 if (handleReject() == RejectAndGiveUp)
905 return false;
906
907 break;
908 }
909
910 case GIM_CheckIntrinsicID: {
911 uint64_t InsnID = readULEB();
912 uint64_t OpIdx = readULEB();
913 uint16_t Value = readU16();
914 DEBUG_WITH_TYPE(TgtExecutor::getName(),
915 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
916 << InsnID << "]->getOperand(" << OpIdx
917 << "), Value=" << Value << ")\n");
918 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
919 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
920 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
921 if (handleReject() == RejectAndGiveUp)
922 return false;
923 break;
924 }
925 case GIM_CheckCmpPredicate: {
926 uint64_t InsnID = readULEB();
927 uint64_t OpIdx = readULEB();
928 uint16_t Value = readU16();
929 DEBUG_WITH_TYPE(TgtExecutor::getName(),
930 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
931 << InsnID << "]->getOperand(" << OpIdx
932 << "), Value=" << Value << ")\n");
933 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
934 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
935 if (!MO.isPredicate() || MO.getPredicate() != Value)
936 if (handleReject() == RejectAndGiveUp)
937 return false;
938 break;
939 }
940 case GIM_CheckIsMBB: {
941 uint64_t InsnID = readULEB();
942 uint64_t OpIdx = readULEB();
943 DEBUG_WITH_TYPE(TgtExecutor::getName(),
944 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
945 << "]->getOperand(" << OpIdx << "))\n");
946 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
947 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
948 if (handleReject() == RejectAndGiveUp)
949 return false;
950 }
951 break;
952 }
953 case GIM_CheckIsImm: {
954 uint64_t InsnID = readULEB();
955 uint64_t OpIdx = readULEB();
956 DEBUG_WITH_TYPE(TgtExecutor::getName(),
957 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
958 << "]->getOperand(" << OpIdx << "))\n");
959 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
960 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
961 if (handleReject() == RejectAndGiveUp)
962 return false;
963 }
964 break;
965 }
966 case GIM_CheckIsSafeToFold: {
967 uint64_t NumInsn = MatchTable[CurrentIdx++];
968 DEBUG_WITH_TYPE(TgtExecutor::getName(),
969 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = "
970 << NumInsn << ")\n");
971 MachineInstr &Root = *State.MIs[0];
972 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) {
973 if (!isObviouslySafeToFold(*State.MIs[K], Root)) {
974 if (handleReject() == RejectAndGiveUp)
975 return false;
976 }
977 }
978 break;
979 }
980 case GIM_CheckIsSameOperand:
981 case GIM_CheckIsSameOperandIgnoreCopies: {
982 uint64_t InsnID = readULEB();
983 uint64_t OpIdx = readULEB();
984 uint64_t OtherInsnID = readULEB();
985 uint64_t OtherOpIdx = readULEB();
986 DEBUG_WITH_TYPE(TgtExecutor::getName(),
987 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
988 << InsnID << "][" << OpIdx << "], MIs["
989 << OtherInsnID << "][" << OtherOpIdx << "])\n");
990 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
991 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
992
993 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
994 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
995
996 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
997 if (Op.isReg() && OtherOp.isReg()) {
998 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) ==
999 getSrcRegIgnoringCopies(OtherOp.getReg(), MRI))
1000 break;
1001 }
1002 }
1003
1004 if (!Op.isIdenticalTo(OtherOp)) {
1005 if (handleReject() == RejectAndGiveUp)
1006 return false;
1007 }
1008 break;
1009 }
1010 case GIM_CheckCanReplaceReg: {
1011 uint64_t OldInsnID = readULEB();
1012 uint64_t OldOpIdx = readULEB();
1013 uint64_t NewInsnID = readULEB();
1014 uint64_t NewOpIdx = readULEB();
1015
1016 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1017 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
1018 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1019 << NewInsnID << "][" << NewOpIdx << "])\n");
1020
1021 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1022 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1023 if (!canReplaceReg(Old, New, MRI)) {
1024 if (handleReject() == RejectAndGiveUp)
1025 return false;
1026 }
1027 break;
1028 }
1029 case GIM_MIFlags: {
1030 uint64_t InsnID = readULEB();
1031 uint32_t Flags = readU32();
1032
1033 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1034 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
1035 << "], " << Flags << ")\n");
1036 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
1037 if (handleReject() == RejectAndGiveUp)
1038 return false;
1039 }
1040 break;
1041 }
1042 case GIM_MIFlagsNot: {
1043 uint64_t InsnID = readULEB();
1044 uint32_t Flags = readU32();
1045
1046 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1047 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
1048 << "], " << Flags << ")\n");
1049 if ((State.MIs[InsnID]->getFlags() & Flags)) {
1050 if (handleReject() == RejectAndGiveUp)
1051 return false;
1052 }
1053 break;
1054 }
1055 case GIM_Reject:
1056 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1057 dbgs() << CurrentIdx << ": GIM_Reject\n");
1058 if (handleReject() == RejectAndGiveUp)
1059 return false;
1060 break;
1061 case GIR_MutateOpcode: {
1062 uint64_t OldInsnID = readULEB();
1063 uint64_t NewInsnID = readULEB();
1064 uint16_t NewOpcode = readU16();
1065 if (NewInsnID >= OutMIs.size())
1066 OutMIs.resize(NewInsnID + 1);
1067
1068 MachineInstr *OldMI = State.MIs[OldInsnID];
1069 if (Observer)
1070 Observer->changingInstr(*OldMI);
1071 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1072 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
1073 if (Observer)
1074 Observer->changedInstr(*OldMI);
1075 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1076 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1077 << NewInsnID << "], MIs[" << OldInsnID << "], "
1078 << NewOpcode << ")\n");
1079 break;
1080 }
1081
1082 case GIR_BuildRootMI:
1083 case GIR_BuildMI: {
1084 uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB();
1085 uint16_t Opcode = readU16();
1086 if (NewInsnID >= OutMIs.size())
1087 OutMIs.resize(NewInsnID + 1);
1088
1089 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1090 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1091 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1092 << NewInsnID << "], " << Opcode << ")\n");
1093 break;
1094 }
1095
1096 case GIR_BuildConstant: {
1097 uint64_t TempRegID = readULEB();
1098 uint64_t Imm = readU64();
1099 Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
1100 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1101 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1102 << TempRegID << "], Imm=" << Imm << ")\n");
1103 break;
1104 }
1105
1106 case GIR_RootToRootCopy:
1107 case GIR_Copy: {
1108 uint64_t NewInsnID =
1109 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1110 uint64_t OldInsnID =
1111 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1112 uint64_t OpIdx = readULEB();
1113 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1114 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
1115 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1116 dbgs()
1117 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1118 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1119 break;
1120 }
1121
1122 case GIR_CopyRemaining: {
1123 uint64_t NewInsnID = readULEB();
1124 uint64_t OldInsnID = readULEB();
1125 uint64_t OpIdx = readULEB();
1126 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1127 MachineInstr &OldMI = *State.MIs[OldInsnID];
1128 MachineInstrBuilder &NewMI = OutMIs[NewInsnID];
1129 for (const auto &Op : drop_begin(OldMI.operands(), OpIdx))
1130 NewMI.add(Op);
1131 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1132 dbgs() << CurrentIdx << ": GIR_CopyRemaining(OutMIs["
1133 << NewInsnID << "], MIs[" << OldInsnID
1134 << "], /*start=*/" << OpIdx << ")\n");
1135 break;
1136 }
1137
1138 case GIR_CopyOrAddZeroReg: {
1139 uint64_t NewInsnID = readULEB();
1140 uint64_t OldInsnID = readULEB();
1141 uint64_t OpIdx = readULEB();
1142 uint16_t ZeroReg = readU16();
1143 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1144 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
1145 if (isOperandImmEqual(MO, 0, MRI))
1146 OutMIs[NewInsnID].addReg(ZeroReg);
1147 else
1148 OutMIs[NewInsnID].add(MO);
1149 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1150 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1151 << NewInsnID << "], MIs[" << OldInsnID << "], "
1152 << OpIdx << ", " << ZeroReg << ")\n");
1153 break;
1154 }
1155
1156 case GIR_CopySubReg: {
1157 uint64_t NewInsnID = readULEB();
1158 uint64_t OldInsnID = readULEB();
1159 uint64_t OpIdx = readULEB();
1160 uint16_t SubRegIdx = readU16();
1161 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1162 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
1163 0, SubRegIdx);
1164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1165 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1166 << NewInsnID << "], MIs[" << OldInsnID << "], "
1167 << OpIdx << ", " << SubRegIdx << ")\n");
1168 break;
1169 }
1170
1171 case GIR_AddImplicitDef: {
1172 uint64_t InsnID = readULEB();
1173 uint16_t RegNum = readU16();
1174 uint16_t Flags = readU16();
1175 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1176 Flags |= RegState::Implicit;
1177 OutMIs[InsnID].addDef(RegNum, Flags);
1178 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1179 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1180 << InsnID << "], " << RegNum << ")\n");
1181 break;
1182 }
1183
1184 case GIR_AddImplicitUse: {
1185 uint64_t InsnID = readULEB();
1186 uint16_t RegNum = readU16();
1187 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1188 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
1189 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1190 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1191 << InsnID << "], " << RegNum << ")\n");
1192 break;
1193 }
1194
1195 case GIR_AddRegister: {
1196 uint64_t InsnID = readULEB();
1197 uint16_t RegNum = readU16();
1198 uint16_t RegFlags = readU16();
1199 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1200 OutMIs[InsnID].addReg(RegNum, RegFlags);
1201 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1202 dbgs()
1203 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1204 << "], " << RegNum << ", " << RegFlags << ")\n");
1205 break;
1206 }
1207 case GIR_AddIntrinsicID: {
1208 uint64_t InsnID = readULEB();
1209 uint16_t Value = readU16();
1210 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1211 OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
1212 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1213 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1214 << InsnID << "], " << Value << ")\n");
1215 break;
1216 }
1217 case GIR_SetImplicitDefDead: {
1218 uint64_t InsnID = readULEB();
1219 uint64_t OpIdx = readULEB();
1220 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1221 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1222 << InsnID << "], OpIdx=" << OpIdx << ")\n");
1223 MachineInstr *MI = OutMIs[InsnID];
1224 assert(MI && "Modifying undefined instruction");
1225 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead();
1226 break;
1227 }
1228 case GIR_SetMIFlags: {
1229 uint64_t InsnID = readULEB();
1230 uint32_t Flags = readU32();
1231
1232 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1233 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1234 << InsnID << "], " << Flags << ")\n");
1235 MachineInstr *MI = OutMIs[InsnID];
1236 MI->setFlags(MI->getFlags() | Flags);
1237 break;
1238 }
1239 case GIR_UnsetMIFlags: {
1240 uint64_t InsnID = readULEB();
1241 uint32_t Flags = readU32();
1242
1243 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1244 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1245 << InsnID << "], " << Flags << ")\n");
1246 MachineInstr *MI = OutMIs[InsnID];
1247 MI->setFlags(MI->getFlags() & ~Flags);
1248 break;
1249 }
1250 case GIR_CopyMIFlags: {
1251 uint64_t InsnID = readULEB();
1252 uint64_t OldInsnID = readULEB();
1253
1254 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1255 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1256 << InsnID << "], MIs[" << OldInsnID << "])\n");
1257 MachineInstr *MI = OutMIs[InsnID];
1258 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1259 break;
1260 }
1261 case GIR_AddSimpleTempRegister:
1262 case GIR_AddTempRegister:
1263 case GIR_AddTempSubRegister: {
1264 uint64_t InsnID = readULEB();
1265 uint64_t TempRegID = readULEB();
1266 uint16_t TempRegFlags = 0;
1267 if (MatcherOpcode != GIR_AddSimpleTempRegister)
1268 TempRegFlags = readU16();
1269 uint16_t SubReg = 0;
1270 if (MatcherOpcode == GIR_AddTempSubRegister)
1271 SubReg = readU16();
1272
1273 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1274
1275 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,
1276 SubReg);
1277 DEBUG_WITH_TYPE(
1278 TgtExecutor::getName(),
1279 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1280 << "], TempRegisters[" << TempRegID << "]";
1281 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1282 dbgs() << ", " << TempRegFlags << ")\n");
1283 break;
1284 }
1285
1286 case GIR_AddImm8:
1287 case GIR_AddImm: {
1288 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1289 uint64_t InsnID = readULEB();
1290 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1291 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1292 OutMIs[InsnID].addImm(Imm);
1293 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1294 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1295 << "], " << Imm << ")\n");
1296 break;
1297 }
1298
1299 case GIR_AddCImm: {
1300 uint64_t InsnID = readULEB();
1301 int TypeID = readS8();
1302 uint64_t Imm = readU64();
1303 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1304
1305 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1306 LLVMContext &Ctx = MF->getFunction().getContext();
1307 OutMIs[InsnID].addCImm(
1308 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true));
1309 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1310 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1311 << "], TypeID=" << TypeID << ", Imm=" << Imm
1312 << ")\n");
1313 break;
1314 }
1315
1316 case GIR_ComplexRenderer: {
1317 uint64_t InsnID = readULEB();
1318 uint16_t RendererID = readU16();
1319 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1320 for (const auto &RenderOpFn : State.Renderers[RendererID])
1321 RenderOpFn(OutMIs[InsnID]);
1322 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1323 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1324 << InsnID << "], " << RendererID << ")\n");
1325 break;
1326 }
1327 case GIR_ComplexSubOperandRenderer: {
1328 uint64_t InsnID = readULEB();
1329 uint16_t RendererID = readU16();
1330 uint64_t RenderOpID = readULEB();
1331 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1332 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1333 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1334 dbgs() << CurrentIdx
1335 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1336 << InsnID << "], " << RendererID << ", "
1337 << RenderOpID << ")\n");
1338 break;
1339 }
1340 case GIR_ComplexSubOperandSubRegRenderer: {
1341 uint64_t InsnID = readULEB();
1342 uint16_t RendererID = readU16();
1343 uint64_t RenderOpID = readULEB();
1344 uint16_t SubRegIdx = readU16();
1345 MachineInstrBuilder &MI = OutMIs[InsnID];
1346 assert(MI && "Attempted to add to undefined instruction");
1347 State.Renderers[RendererID][RenderOpID](MI);
1348 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1349 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1350 dbgs() << CurrentIdx
1351 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1352 << InsnID << "], " << RendererID << ", "
1353 << RenderOpID << ", " << SubRegIdx << ")\n");
1354 break;
1355 }
1356
1357 case GIR_CopyConstantAsSImm: {
1358 uint64_t NewInsnID = readULEB();
1359 uint64_t OldInsnID = readULEB();
1360 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1361 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1362 "Expected G_CONSTANT");
1363 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1364 OutMIs[NewInsnID].addImm(
1365 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1366 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1367 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1368 else
1369 llvm_unreachable("Expected Imm or CImm operand");
1370 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1371 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1372 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1373 break;
1374 }
1375
1376 // TODO: Needs a test case once we have a pattern that uses this.
1377 case GIR_CopyFConstantAsFPImm: {
1378 uint64_t NewInsnID = readULEB();
1379 uint64_t OldInsnID = readULEB();
1380 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1381 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1382 "Expected G_FCONSTANT");
1383 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1384 OutMIs[NewInsnID].addFPImm(
1385 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1386 else
1387 llvm_unreachable("Expected FPImm operand");
1388 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1389 dbgs()
1390 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1391 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1392 break;
1393 }
1394
1395 case GIR_CustomRenderer: {
1396 uint64_t InsnID = readULEB();
1397 uint64_t OldInsnID = readULEB();
1398 uint16_t RendererFnID = readU16();
1399 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1400 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1401 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1402 << InsnID << "], MIs[" << OldInsnID << "], "
1403 << RendererFnID << ")\n");
1404 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1405 OutMIs[InsnID], *State.MIs[OldInsnID],
1406 -1); // Not a source operand of the old instruction.
1407 break;
1408 }
1409 case GIR_DoneWithCustomAction: {
1410 uint16_t FnID = readU16();
1411 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1412 dbgs() << CurrentIdx << ": GIR_DoneWithCustomAction(FnID="
1413 << FnID << ")\n");
1414 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1415 if (runCustomAction(FnID, State, OutMIs)) {
1416 propagateFlags();
1417 return true;
1418 }
1419
1420 if (handleReject() == RejectAndGiveUp)
1421 return false;
1422 break;
1423 }
1424 case GIR_CustomOperandRenderer: {
1425 uint64_t InsnID = readULEB();
1426 uint64_t OldInsnID = readULEB();
1427 uint64_t OpIdx = readULEB();
1428 uint16_t RendererFnID = readU16();
1429 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1430
1431 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1432 dbgs() << CurrentIdx
1433 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1434 << "], MIs[" << OldInsnID << "]->getOperand("
1435 << OpIdx << "), " << RendererFnID << ")\n");
1436 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1437 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1438 break;
1439 }
1440 case GIR_ConstrainOperandRC: {
1441 uint64_t InsnID = readULEB();
1442 uint64_t OpIdx = readULEB();
1443 uint16_t RCEnum = readU16();
1444 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1445 MachineInstr &I = *OutMIs[InsnID].getInstr();
1446 MachineFunction &MF = *I.getParent()->getParent();
1447 MachineRegisterInfo &MRI = MF.getRegInfo();
1448 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1449 MachineOperand &MO = I.getOperand(OpIdx);
1450 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1451 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1452 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1453 << InsnID << "], " << OpIdx << ", " << RCEnum
1454 << ")\n");
1455 break;
1456 }
1457
1458 case GIR_RootConstrainSelectedInstOperands:
1459 case GIR_ConstrainSelectedInstOperands: {
1460 uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands)
1461 ? 0
1462 : readULEB();
1463 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1464 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1465 RBI);
1466 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1467 dbgs() << CurrentIdx
1468 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1469 << InsnID << "])\n");
1470 break;
1471 }
1472 case GIR_MergeMemOperands: {
1473 uint64_t InsnID = readULEB();
1474 uint64_t NumInsn = MatchTable[CurrentIdx++];
1475 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1476
1477 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1478 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1479 << InsnID << "]");
1480 for (unsigned K = 0; K < NumInsn; ++K) {
1481 uint64_t NextID = readULEB();
1482 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1483 dbgs() << ", MIs[" << NextID << "]");
1484 for (const auto &MMO : State.MIs[NextID]->memoperands())
1485 OutMIs[InsnID].addMemOperand(MMO);
1486 }
1487 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1488 break;
1489 }
1490 case GIR_EraseFromParent: {
1491 uint64_t InsnID = readULEB();
1492 MachineInstr *MI = State.MIs[InsnID];
1493 assert(MI && "Attempted to erase an undefined instruction");
1494 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1495 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1496 << InsnID << "])\n");
1497 eraseImpl(MI);
1498 break;
1499 }
1500 case GIR_EraseRootFromParent_Done: {
1501 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1502 dbgs()
1503 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n");
1504 eraseImpl(State.MIs[0]);
1505 propagateFlags();
1506 return true;
1507 }
1508 case GIR_MakeTempReg: {
1509 uint64_t TempRegID = readULEB();
1510 int TypeID = readS8();
1511
1512 State.TempRegisters[TempRegID] =
1513 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));
1514 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1515 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1516 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1517 break;
1518 }
1519 case GIR_ReplaceReg: {
1520 uint64_t OldInsnID = readULEB();
1521 uint64_t OldOpIdx = readULEB();
1522 uint64_t NewInsnID = readULEB();
1523 uint64_t NewOpIdx = readULEB();
1524
1525 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1526 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1527 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1528 << NewInsnID << "][" << NewOpIdx << "])\n");
1529
1530 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1531 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1532 if (Observer)
1533 Observer->changingAllUsesOfReg(MRI, Old);
1534 MRI.replaceRegWith(Old, New);
1535 if (Observer)
1536 Observer->finishedChangingAllUsesOfReg();
1537 break;
1538 }
1539 case GIR_ReplaceRegWithTempReg: {
1540 uint64_t OldInsnID = readULEB();
1541 uint64_t OldOpIdx = readULEB();
1542 uint64_t TempRegID = readULEB();
1543
1544 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1545 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1546 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1547 << TempRegID << "])\n");
1548
1549 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1550 Register New = State.TempRegisters[TempRegID];
1551 if (Observer)
1552 Observer->changingAllUsesOfReg(MRI, Old);
1553 MRI.replaceRegWith(Old, New);
1554 if (Observer)
1555 Observer->finishedChangingAllUsesOfReg();
1556 break;
1557 }
1558 case GIR_Coverage: {
1559 uint32_t RuleID = readU32();
1560 assert(CoverageInfo);
1561 CoverageInfo->setCovered(RuleID);
1562
1563 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1564 << ": GIR_Coverage("
1565 << RuleID << ")");
1566 break;
1567 }
1568
1569 case GIR_Done:
1570 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1571 dbgs() << CurrentIdx << ": GIR_Done\n");
1572 propagateFlags();
1573 return true;
1574 default:
1575 llvm_unreachable("Unexpected command");
1576 }
1577 }
1578 }
1579
1580 } // end namespace llvm
1581
1582 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1583