xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp (revision 43e29d03f416d7dda52112a29600a7c82ee1a91e) 
1 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the MachineIRBuidler class.
10 //===----------------------------------------------------------------------===//
11 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12 #include "llvm/CodeGen/MachineFunction.h"
13 #include "llvm/CodeGen/MachineInstr.h"
14 #include "llvm/CodeGen/MachineInstrBuilder.h"
15 #include "llvm/CodeGen/MachineRegisterInfo.h"
16 #include "llvm/CodeGen/TargetInstrInfo.h"
17 #include "llvm/CodeGen/TargetLowering.h"
18 #include "llvm/CodeGen/TargetOpcodes.h"
19 #include "llvm/CodeGen/TargetSubtargetInfo.h"
20 #include "llvm/IR/DebugInfoMetadata.h"
21 
22 using namespace llvm;
23 
24 void MachineIRBuilder::setMF(MachineFunction &MF) {
25   State.MF = &MF;
26   State.MBB = nullptr;
27   State.MRI = &MF.getRegInfo();
28   State.TII = MF.getSubtarget().getInstrInfo();
29   State.DL = DebugLoc();
30   State.PCSections = nullptr;
31   State.II = MachineBasicBlock::iterator();
32   State.Observer = nullptr;
33 }
34 
35 //------------------------------------------------------------------------------
36 // Build instruction variants.
37 //------------------------------------------------------------------------------
38 
39 MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
40   return BuildMI(getMF(), {getDL(), getPCSections()}, getTII().get(Opcode));
41 }
42 
43 MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
44   getMBB().insert(getInsertPt(), MIB);
45   recordInsertion(MIB);
46   return MIB;
47 }
48 
49 MachineInstrBuilder
50 MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
51                                       const MDNode *Expr) {
52   assert(isa<DILocalVariable>(Variable) && "not a variable");
53   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
54   assert(
55       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
56       "Expected inlined-at fields to agree");
57   return insertInstr(BuildMI(getMF(), getDL(),
58                              getTII().get(TargetOpcode::DBG_VALUE),
59                              /*IsIndirect*/ false, Reg, Variable, Expr));
60 }
61 
62 MachineInstrBuilder
63 MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
64                                         const MDNode *Expr) {
65   assert(isa<DILocalVariable>(Variable) && "not a variable");
66   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
67   assert(
68       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
69       "Expected inlined-at fields to agree");
70   return insertInstr(BuildMI(getMF(), getDL(),
71                              getTII().get(TargetOpcode::DBG_VALUE),
72                              /*IsIndirect*/ true, Reg, Variable, Expr));
73 }
74 
75 MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
76                                                       const MDNode *Variable,
77                                                       const MDNode *Expr) {
78   assert(isa<DILocalVariable>(Variable) && "not a variable");
79   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
80   assert(
81       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
82       "Expected inlined-at fields to agree");
83   return buildInstr(TargetOpcode::DBG_VALUE)
84       .addFrameIndex(FI)
85       .addImm(0)
86       .addMetadata(Variable)
87       .addMetadata(Expr);
88 }
89 
90 MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
91                                                          const MDNode *Variable,
92                                                          const MDNode *Expr) {
93   assert(isa<DILocalVariable>(Variable) && "not a variable");
94   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
95   assert(
96       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
97       "Expected inlined-at fields to agree");
98   auto MIB = buildInstrNoInsert(TargetOpcode::DBG_VALUE);
99 
100   auto *NumericConstant = [&] () -> const Constant* {
101     if (const auto *CE = dyn_cast<ConstantExpr>(&C))
102       if (CE->getOpcode() == Instruction::IntToPtr)
103         return CE->getOperand(0);
104     return &C;
105   }();
106 
107   if (auto *CI = dyn_cast<ConstantInt>(NumericConstant)) {
108     if (CI->getBitWidth() > 64)
109       MIB.addCImm(CI);
110     else
111       MIB.addImm(CI->getZExtValue());
112   } else if (auto *CFP = dyn_cast<ConstantFP>(NumericConstant)) {
113     MIB.addFPImm(CFP);
114   } else if (isa<ConstantPointerNull>(NumericConstant)) {
115     MIB.addImm(0);
116   } else {
117     // Insert $noreg if we didn't find a usable constant and had to drop it.
118     MIB.addReg(Register());
119   }
120 
121   MIB.addImm(0).addMetadata(Variable).addMetadata(Expr);
122   return insertInstr(MIB);
123 }
124 
125 MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
126   assert(isa<DILabel>(Label) && "not a label");
127   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
128          "Expected inlined-at fields to agree");
129   auto MIB = buildInstr(TargetOpcode::DBG_LABEL);
130 
131   return MIB.addMetadata(Label);
132 }
133 
134 MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
135                                                          const SrcOp &Size,
136                                                          Align Alignment) {
137   assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
138   auto MIB = buildInstr(TargetOpcode::G_DYN_STACKALLOC);
139   Res.addDefToMIB(*getMRI(), MIB);
140   Size.addSrcToMIB(MIB);
141   MIB.addImm(Alignment.value());
142   return MIB;
143 }
144 
145 MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
146                                                       int Idx) {
147   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
148   auto MIB = buildInstr(TargetOpcode::G_FRAME_INDEX);
149   Res.addDefToMIB(*getMRI(), MIB);
150   MIB.addFrameIndex(Idx);
151   return MIB;
152 }
153 
154 MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
155                                                        const GlobalValue *GV) {
156   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
157   assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
158              GV->getType()->getAddressSpace() &&
159          "address space mismatch");
160 
161   auto MIB = buildInstr(TargetOpcode::G_GLOBAL_VALUE);
162   Res.addDefToMIB(*getMRI(), MIB);
163   MIB.addGlobalAddress(GV);
164   return MIB;
165 }
166 
167 MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
168                                                      unsigned JTI) {
169   return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {})
170       .addJumpTableIndex(JTI);
171 }
172 
173 void MachineIRBuilder::validateUnaryOp(const LLT Res, const LLT Op0) {
174   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
175   assert((Res == Op0) && "type mismatch");
176 }
177 
178 void MachineIRBuilder::validateBinaryOp(const LLT Res, const LLT Op0,
179                                         const LLT Op1) {
180   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
181   assert((Res == Op0 && Res == Op1) && "type mismatch");
182 }
183 
184 void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0,
185                                        const LLT Op1) {
186   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
187   assert((Res == Op0) && "type mismatch");
188 }
189 
190 MachineInstrBuilder MachineIRBuilder::buildPtrAdd(const DstOp &Res,
191                                                   const SrcOp &Op0,
192                                                   const SrcOp &Op1) {
193   assert(Res.getLLTTy(*getMRI()).getScalarType().isPointer() &&
194          Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
195   assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
196 
197   return buildInstr(TargetOpcode::G_PTR_ADD, {Res}, {Op0, Op1});
198 }
199 
200 std::optional<MachineInstrBuilder>
201 MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
202                                     const LLT ValueTy, uint64_t Value) {
203   assert(Res == 0 && "Res is a result argument");
204   assert(ValueTy.isScalar()  && "invalid offset type");
205 
206   if (Value == 0) {
207     Res = Op0;
208     return std::nullopt;
209   }
210 
211   Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0));
212   auto Cst = buildConstant(ValueTy, Value);
213   return buildPtrAdd(Res, Op0, Cst.getReg(0));
214 }
215 
216 MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
217                                                           const SrcOp &Op0,
218                                                           uint32_t NumBits) {
219   LLT PtrTy = Res.getLLTTy(*getMRI());
220   LLT MaskTy = LLT::scalar(PtrTy.getSizeInBits());
221   Register MaskReg = getMRI()->createGenericVirtualRegister(MaskTy);
222   buildConstant(MaskReg, maskTrailingZeros<uint64_t>(NumBits));
223   return buildPtrMask(Res, Op0, MaskReg);
224 }
225 
226 MachineInstrBuilder
227 MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
228                                                   const SrcOp &Op0) {
229   LLT ResTy = Res.getLLTTy(*getMRI());
230   LLT Op0Ty = Op0.getLLTTy(*getMRI());
231 
232   assert((ResTy.isVector() && Op0Ty.isVector()) && "Non vector type");
233   assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
234          "Different vector element types");
235   assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
236          "Op0 has more elements");
237 
238   auto Unmerge = buildUnmerge(Op0Ty.getElementType(), Op0);
239   SmallVector<Register, 8> Regs;
240   for (auto Op : Unmerge.getInstr()->defs())
241     Regs.push_back(Op.getReg());
242   Register Undef = buildUndef(Op0Ty.getElementType()).getReg(0);
243   unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
244   for (unsigned i = 0; i < NumberOfPadElts; ++i)
245     Regs.push_back(Undef);
246   return buildMergeLikeInstr(Res, Regs);
247 }
248 
249 MachineInstrBuilder
250 MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
251                                                     const SrcOp &Op0) {
252   LLT ResTy = Res.getLLTTy(*getMRI());
253   LLT Op0Ty = Op0.getLLTTy(*getMRI());
254 
255   assert((ResTy.isVector() && Op0Ty.isVector()) && "Non vector type");
256   assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
257          "Different vector element types");
258   assert((ResTy.getNumElements() < Op0Ty.getNumElements()) &&
259          "Op0 has fewer elements");
260 
261   SmallVector<Register, 8> Regs;
262   auto Unmerge = buildUnmerge(Op0Ty.getElementType(), Op0);
263   for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
264     Regs.push_back(Unmerge.getReg(i));
265   return buildMergeLikeInstr(Res, Regs);
266 }
267 
268 MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
269   return buildInstr(TargetOpcode::G_BR).addMBB(&Dest);
270 }
271 
272 MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
273   assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
274   return buildInstr(TargetOpcode::G_BRINDIRECT).addUse(Tgt);
275 }
276 
277 MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
278                                                 unsigned JTI,
279                                                 Register IndexReg) {
280   assert(getMRI()->getType(TablePtr).isPointer() &&
281          "Table reg must be a pointer");
282   return buildInstr(TargetOpcode::G_BRJT)
283       .addUse(TablePtr)
284       .addJumpTableIndex(JTI)
285       .addUse(IndexReg);
286 }
287 
288 MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
289                                                 const SrcOp &Op) {
290   return buildInstr(TargetOpcode::COPY, Res, Op);
291 }
292 
293 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
294                                                     const ConstantInt &Val) {
295   LLT Ty = Res.getLLTTy(*getMRI());
296   LLT EltTy = Ty.getScalarType();
297   assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
298          "creating constant with the wrong size");
299 
300   if (Ty.isVector()) {
301     auto Const = buildInstr(TargetOpcode::G_CONSTANT)
302     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
303     .addCImm(&Val);
304     return buildSplatVector(Res, Const);
305   }
306 
307   auto Const = buildInstr(TargetOpcode::G_CONSTANT);
308   Const->setDebugLoc(DebugLoc());
309   Res.addDefToMIB(*getMRI(), Const);
310   Const.addCImm(&Val);
311   return Const;
312 }
313 
314 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
315                                                     int64_t Val) {
316   auto IntN = IntegerType::get(getMF().getFunction().getContext(),
317                                Res.getLLTTy(*getMRI()).getScalarSizeInBits());
318   ConstantInt *CI = ConstantInt::get(IntN, Val, true);
319   return buildConstant(Res, *CI);
320 }
321 
322 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
323                                                      const ConstantFP &Val) {
324   LLT Ty = Res.getLLTTy(*getMRI());
325   LLT EltTy = Ty.getScalarType();
326 
327   assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
328          == EltTy.getSizeInBits() &&
329          "creating fconstant with the wrong size");
330 
331   assert(!Ty.isPointer() && "invalid operand type");
332 
333   if (Ty.isVector()) {
334     auto Const = buildInstr(TargetOpcode::G_FCONSTANT)
335     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
336     .addFPImm(&Val);
337 
338     return buildSplatVector(Res, Const);
339   }
340 
341   auto Const = buildInstr(TargetOpcode::G_FCONSTANT);
342   Const->setDebugLoc(DebugLoc());
343   Res.addDefToMIB(*getMRI(), Const);
344   Const.addFPImm(&Val);
345   return Const;
346 }
347 
348 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
349                                                     const APInt &Val) {
350   ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val);
351   return buildConstant(Res, *CI);
352 }
353 
354 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
355                                                      double Val) {
356   LLT DstTy = Res.getLLTTy(*getMRI());
357   auto &Ctx = getMF().getFunction().getContext();
358   auto *CFP =
359       ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits()));
360   return buildFConstant(Res, *CFP);
361 }
362 
363 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
364                                                      const APFloat &Val) {
365   auto &Ctx = getMF().getFunction().getContext();
366   auto *CFP = ConstantFP::get(Ctx, Val);
367   return buildFConstant(Res, *CFP);
368 }
369 
370 MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
371                                                   MachineBasicBlock &Dest) {
372   assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
373 
374   auto MIB = buildInstr(TargetOpcode::G_BRCOND);
375   Tst.addSrcToMIB(MIB);
376   MIB.addMBB(&Dest);
377   return MIB;
378 }
379 
380 MachineInstrBuilder
381 MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
382                             MachinePointerInfo PtrInfo, Align Alignment,
383                             MachineMemOperand::Flags MMOFlags,
384                             const AAMDNodes &AAInfo) {
385   MMOFlags |= MachineMemOperand::MOLoad;
386   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
387 
388   LLT Ty = Dst.getLLTTy(*getMRI());
389   MachineMemOperand *MMO =
390       getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo);
391   return buildLoad(Dst, Addr, *MMO);
392 }
393 
394 MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
395                                                      const DstOp &Res,
396                                                      const SrcOp &Addr,
397                                                      MachineMemOperand &MMO) {
398   assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
399   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
400 
401   auto MIB = buildInstr(Opcode);
402   Res.addDefToMIB(*getMRI(), MIB);
403   Addr.addSrcToMIB(MIB);
404   MIB.addMemOperand(&MMO);
405   return MIB;
406 }
407 
408 MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
409   const DstOp &Dst, const SrcOp &BasePtr,
410   MachineMemOperand &BaseMMO, int64_t Offset) {
411   LLT LoadTy = Dst.getLLTTy(*getMRI());
412   MachineMemOperand *OffsetMMO =
413       getMF().getMachineMemOperand(&BaseMMO, Offset, LoadTy);
414 
415   if (Offset == 0) // This may be a size or type changing load.
416     return buildLoad(Dst, BasePtr, *OffsetMMO);
417 
418   LLT PtrTy = BasePtr.getLLTTy(*getMRI());
419   LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits());
420   auto ConstOffset = buildConstant(OffsetTy, Offset);
421   auto Ptr = buildPtrAdd(PtrTy, BasePtr, ConstOffset);
422   return buildLoad(Dst, Ptr, *OffsetMMO);
423 }
424 
425 MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
426                                                  const SrcOp &Addr,
427                                                  MachineMemOperand &MMO) {
428   assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
429   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
430 
431   auto MIB = buildInstr(TargetOpcode::G_STORE);
432   Val.addSrcToMIB(MIB);
433   Addr.addSrcToMIB(MIB);
434   MIB.addMemOperand(&MMO);
435   return MIB;
436 }
437 
438 MachineInstrBuilder
439 MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
440                              MachinePointerInfo PtrInfo, Align Alignment,
441                              MachineMemOperand::Flags MMOFlags,
442                              const AAMDNodes &AAInfo) {
443   MMOFlags |= MachineMemOperand::MOStore;
444   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
445 
446   LLT Ty = Val.getLLTTy(*getMRI());
447   MachineMemOperand *MMO =
448       getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo);
449   return buildStore(Val, Addr, *MMO);
450 }
451 
452 MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
453                                                   const SrcOp &Op) {
454   return buildInstr(TargetOpcode::G_ANYEXT, Res, Op);
455 }
456 
457 MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
458                                                 const SrcOp &Op) {
459   return buildInstr(TargetOpcode::G_SEXT, Res, Op);
460 }
461 
462 MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
463                                                 const SrcOp &Op) {
464   return buildInstr(TargetOpcode::G_ZEXT, Res, Op);
465 }
466 
467 unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
468   const auto *TLI = getMF().getSubtarget().getTargetLowering();
469   switch (TLI->getBooleanContents(IsVec, IsFP)) {
470   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
471     return TargetOpcode::G_SEXT;
472   case TargetLoweringBase::ZeroOrOneBooleanContent:
473     return TargetOpcode::G_ZEXT;
474   default:
475     return TargetOpcode::G_ANYEXT;
476   }
477 }
478 
479 MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
480                                                    const SrcOp &Op,
481                                                    bool IsFP) {
482   unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP);
483   return buildInstr(ExtOp, Res, Op);
484 }
485 
486 MachineInstrBuilder MachineIRBuilder::buildBoolExtInReg(const DstOp &Res,
487                                                         const SrcOp &Op,
488                                                         bool IsVector,
489                                                         bool IsFP) {
490   const auto *TLI = getMF().getSubtarget().getTargetLowering();
491   switch (TLI->getBooleanContents(IsVector, IsFP)) {
492   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
493     return buildSExtInReg(Res, Op, 1);
494   case TargetLoweringBase::ZeroOrOneBooleanContent:
495     return buildZExtInReg(Res, Op, 1);
496   case TargetLoweringBase::UndefinedBooleanContent:
497     return buildCopy(Res, Op);
498   }
499 
500   llvm_unreachable("unexpected BooleanContent");
501 }
502 
503 MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
504                                                       const DstOp &Res,
505                                                       const SrcOp &Op) {
506   assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
507           TargetOpcode::G_SEXT == ExtOpc) &&
508          "Expecting Extending Opc");
509   assert(Res.getLLTTy(*getMRI()).isScalar() ||
510          Res.getLLTTy(*getMRI()).isVector());
511   assert(Res.getLLTTy(*getMRI()).isScalar() ==
512          Op.getLLTTy(*getMRI()).isScalar());
513 
514   unsigned Opcode = TargetOpcode::COPY;
515   if (Res.getLLTTy(*getMRI()).getSizeInBits() >
516       Op.getLLTTy(*getMRI()).getSizeInBits())
517     Opcode = ExtOpc;
518   else if (Res.getLLTTy(*getMRI()).getSizeInBits() <
519            Op.getLLTTy(*getMRI()).getSizeInBits())
520     Opcode = TargetOpcode::G_TRUNC;
521   else
522     assert(Res.getLLTTy(*getMRI()) == Op.getLLTTy(*getMRI()));
523 
524   return buildInstr(Opcode, Res, Op);
525 }
526 
527 MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
528                                                        const SrcOp &Op) {
529   return buildExtOrTrunc(TargetOpcode::G_SEXT, Res, Op);
530 }
531 
532 MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
533                                                        const SrcOp &Op) {
534   return buildExtOrTrunc(TargetOpcode::G_ZEXT, Res, Op);
535 }
536 
537 MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
538                                                          const SrcOp &Op) {
539   return buildExtOrTrunc(TargetOpcode::G_ANYEXT, Res, Op);
540 }
541 
542 MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
543                                                      const SrcOp &Op,
544                                                      int64_t ImmOp) {
545   LLT ResTy = Res.getLLTTy(*getMRI());
546   auto Mask = buildConstant(
547       ResTy, APInt::getLowBitsSet(ResTy.getScalarSizeInBits(), ImmOp));
548   return buildAnd(Res, Op, Mask);
549 }
550 
551 MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
552                                                 const SrcOp &Src) {
553   LLT SrcTy = Src.getLLTTy(*getMRI());
554   LLT DstTy = Dst.getLLTTy(*getMRI());
555   if (SrcTy == DstTy)
556     return buildCopy(Dst, Src);
557 
558   unsigned Opcode;
559   if (SrcTy.isPointer() && DstTy.isScalar())
560     Opcode = TargetOpcode::G_PTRTOINT;
561   else if (DstTy.isPointer() && SrcTy.isScalar())
562     Opcode = TargetOpcode::G_INTTOPTR;
563   else {
564     assert(!SrcTy.isPointer() && !DstTy.isPointer() && "n G_ADDRCAST yet");
565     Opcode = TargetOpcode::G_BITCAST;
566   }
567 
568   return buildInstr(Opcode, Dst, Src);
569 }
570 
571 MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
572                                                    const SrcOp &Src,
573                                                    uint64_t Index) {
574   LLT SrcTy = Src.getLLTTy(*getMRI());
575   LLT DstTy = Dst.getLLTTy(*getMRI());
576 
577 #ifndef NDEBUG
578   assert(SrcTy.isValid() && "invalid operand type");
579   assert(DstTy.isValid() && "invalid operand type");
580   assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
581          "extracting off end of register");
582 #endif
583 
584   if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
585     assert(Index == 0 && "insertion past the end of a register");
586     return buildCast(Dst, Src);
587   }
588 
589   auto Extract = buildInstr(TargetOpcode::G_EXTRACT);
590   Dst.addDefToMIB(*getMRI(), Extract);
591   Src.addSrcToMIB(Extract);
592   Extract.addImm(Index);
593   return Extract;
594 }
595 
596 MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
597   return buildInstr(TargetOpcode::G_IMPLICIT_DEF, {Res}, {});
598 }
599 
600 MachineInstrBuilder MachineIRBuilder::buildMergeValues(const DstOp &Res,
601                                                        ArrayRef<Register> Ops) {
602   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
603   // we need some temporary storage for the DstOp objects. Here we use a
604   // sufficiently large SmallVector to not go through the heap.
605   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
606   assert(TmpVec.size() > 1);
607   return buildInstr(TargetOpcode::G_MERGE_VALUES, Res, TmpVec);
608 }
609 
610 MachineInstrBuilder
611 MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
612                                       ArrayRef<Register> Ops) {
613   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
614   // we need some temporary storage for the DstOp objects. Here we use a
615   // sufficiently large SmallVector to not go through the heap.
616   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
617   assert(TmpVec.size() > 1);
618   return buildInstr(getOpcodeForMerge(Res, TmpVec), Res, TmpVec);
619 }
620 
621 MachineInstrBuilder
622 MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
623                                       std::initializer_list<SrcOp> Ops) {
624   assert(Ops.size() > 1);
625   return buildInstr(getOpcodeForMerge(Res, Ops), Res, Ops);
626 }
627 
628 unsigned MachineIRBuilder::getOpcodeForMerge(const DstOp &DstOp,
629                                              ArrayRef<SrcOp> SrcOps) const {
630   if (DstOp.getLLTTy(*getMRI()).isVector()) {
631     if (SrcOps[0].getLLTTy(*getMRI()).isVector())
632       return TargetOpcode::G_CONCAT_VECTORS;
633     return TargetOpcode::G_BUILD_VECTOR;
634   }
635 
636   return TargetOpcode::G_MERGE_VALUES;
637 }
638 
639 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
640                                                    const SrcOp &Op) {
641   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
642   // we need some temporary storage for the DstOp objects. Here we use a
643   // sufficiently large SmallVector to not go through the heap.
644   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
645   assert(TmpVec.size() > 1);
646   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
647 }
648 
649 MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
650                                                    const SrcOp &Op) {
651   unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits();
652   SmallVector<DstOp, 8> TmpVec(NumReg, Res);
653   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
654 }
655 
656 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
657                                                    const SrcOp &Op) {
658   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
659   // we need some temporary storage for the DstOp objects. Here we use a
660   // sufficiently large SmallVector to not go through the heap.
661   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
662   assert(TmpVec.size() > 1);
663   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
664 }
665 
666 MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
667                                                        ArrayRef<Register> Ops) {
668   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
669   // we need some temporary storage for the DstOp objects. Here we use a
670   // sufficiently large SmallVector to not go through the heap.
671   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
672   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
673 }
674 
675 MachineInstrBuilder
676 MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
677                                            ArrayRef<APInt> Ops) {
678   SmallVector<SrcOp> TmpVec;
679   TmpVec.reserve(Ops.size());
680   LLT EltTy = Res.getLLTTy(*getMRI()).getElementType();
681   for (const auto &Op : Ops)
682     TmpVec.push_back(buildConstant(EltTy, Op));
683   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
684 }
685 
686 MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
687                                                        const SrcOp &Src) {
688   SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
689   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
690 }
691 
692 MachineInstrBuilder
693 MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
694                                         ArrayRef<Register> Ops) {
695   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
696   // we need some temporary storage for the DstOp objects. Here we use a
697   // sufficiently large SmallVector to not go through the heap.
698   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
699   if (TmpVec[0].getLLTTy(*getMRI()).getSizeInBits() ==
700       Res.getLLTTy(*getMRI()).getElementType().getSizeInBits())
701     return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
702   return buildInstr(TargetOpcode::G_BUILD_VECTOR_TRUNC, Res, TmpVec);
703 }
704 
705 MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
706                                                         const SrcOp &Src) {
707   LLT DstTy = Res.getLLTTy(*getMRI());
708   assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
709          "Expected Src to match Dst elt ty");
710   auto UndefVec = buildUndef(DstTy);
711   auto Zero = buildConstant(LLT::scalar(64), 0);
712   auto InsElt = buildInsertVectorElement(DstTy, UndefVec, Src, Zero);
713   SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
714   return buildShuffleVector(DstTy, InsElt, UndefVec, ZeroMask);
715 }
716 
717 MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
718                                                          const SrcOp &Src1,
719                                                          const SrcOp &Src2,
720                                                          ArrayRef<int> Mask) {
721   LLT DstTy = Res.getLLTTy(*getMRI());
722   LLT Src1Ty = Src1.getLLTTy(*getMRI());
723   LLT Src2Ty = Src2.getLLTTy(*getMRI());
724   assert((size_t)(Src1Ty.getNumElements() + Src2Ty.getNumElements()) >=
725          Mask.size());
726   assert(DstTy.getElementType() == Src1Ty.getElementType() &&
727          DstTy.getElementType() == Src2Ty.getElementType());
728   (void)DstTy;
729   (void)Src1Ty;
730   (void)Src2Ty;
731   ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
732   return buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {Res}, {Src1, Src2})
733       .addShuffleMask(MaskAlloc);
734 }
735 
736 MachineInstrBuilder
737 MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
738   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
739   // we need some temporary storage for the DstOp objects. Here we use a
740   // sufficiently large SmallVector to not go through the heap.
741   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
742   return buildInstr(TargetOpcode::G_CONCAT_VECTORS, Res, TmpVec);
743 }
744 
745 MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
746                                                   const SrcOp &Src,
747                                                   const SrcOp &Op,
748                                                   unsigned Index) {
749   assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
750              Res.getLLTTy(*getMRI()).getSizeInBits() &&
751          "insertion past the end of a register");
752 
753   if (Res.getLLTTy(*getMRI()).getSizeInBits() ==
754       Op.getLLTTy(*getMRI()).getSizeInBits()) {
755     return buildCast(Res, Op);
756   }
757 
758   return buildInstr(TargetOpcode::G_INSERT, Res, {Src, Op, uint64_t(Index)});
759 }
760 
761 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
762                                                      ArrayRef<Register> ResultRegs,
763                                                      bool HasSideEffects) {
764   auto MIB =
765       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
766                                 : TargetOpcode::G_INTRINSIC);
767   for (unsigned ResultReg : ResultRegs)
768     MIB.addDef(ResultReg);
769   MIB.addIntrinsicID(ID);
770   return MIB;
771 }
772 
773 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
774                                                      ArrayRef<DstOp> Results,
775                                                      bool HasSideEffects) {
776   auto MIB =
777       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
778                                 : TargetOpcode::G_INTRINSIC);
779   for (DstOp Result : Results)
780     Result.addDefToMIB(*getMRI(), MIB);
781   MIB.addIntrinsicID(ID);
782   return MIB;
783 }
784 
785 MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
786                                                  const SrcOp &Op) {
787   return buildInstr(TargetOpcode::G_TRUNC, Res, Op);
788 }
789 
790 MachineInstrBuilder
791 MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
792                                std::optional<unsigned> Flags) {
793   return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op, Flags);
794 }
795 
796 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
797                                                 const DstOp &Res,
798                                                 const SrcOp &Op0,
799                                                 const SrcOp &Op1) {
800   return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
801 }
802 
803 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
804                                                 const DstOp &Res,
805                                                 const SrcOp &Op0,
806                                                 const SrcOp &Op1,
807                                                 std::optional<unsigned> Flags) {
808 
809   return buildInstr(TargetOpcode::G_FCMP, Res, {Pred, Op0, Op1}, Flags);
810 }
811 
812 MachineInstrBuilder
813 MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
814                               const SrcOp &Op0, const SrcOp &Op1,
815                               std::optional<unsigned> Flags) {
816 
817   return buildInstr(TargetOpcode::G_SELECT, {Res}, {Tst, Op0, Op1}, Flags);
818 }
819 
820 MachineInstrBuilder
821 MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
822                                            const SrcOp &Elt, const SrcOp &Idx) {
823   return buildInstr(TargetOpcode::G_INSERT_VECTOR_ELT, Res, {Val, Elt, Idx});
824 }
825 
826 MachineInstrBuilder
827 MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
828                                             const SrcOp &Idx) {
829   return buildInstr(TargetOpcode::G_EXTRACT_VECTOR_ELT, Res, {Val, Idx});
830 }
831 
832 MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
833     Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal,
834     Register NewVal, MachineMemOperand &MMO) {
835 #ifndef NDEBUG
836   LLT OldValResTy = getMRI()->getType(OldValRes);
837   LLT SuccessResTy = getMRI()->getType(SuccessRes);
838   LLT AddrTy = getMRI()->getType(Addr);
839   LLT CmpValTy = getMRI()->getType(CmpVal);
840   LLT NewValTy = getMRI()->getType(NewVal);
841   assert(OldValResTy.isScalar() && "invalid operand type");
842   assert(SuccessResTy.isScalar() && "invalid operand type");
843   assert(AddrTy.isPointer() && "invalid operand type");
844   assert(CmpValTy.isValid() && "invalid operand type");
845   assert(NewValTy.isValid() && "invalid operand type");
846   assert(OldValResTy == CmpValTy && "type mismatch");
847   assert(OldValResTy == NewValTy && "type mismatch");
848 #endif
849 
850   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
851       .addDef(OldValRes)
852       .addDef(SuccessRes)
853       .addUse(Addr)
854       .addUse(CmpVal)
855       .addUse(NewVal)
856       .addMemOperand(&MMO);
857 }
858 
859 MachineInstrBuilder
860 MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr,
861                                      Register CmpVal, Register NewVal,
862                                      MachineMemOperand &MMO) {
863 #ifndef NDEBUG
864   LLT OldValResTy = getMRI()->getType(OldValRes);
865   LLT AddrTy = getMRI()->getType(Addr);
866   LLT CmpValTy = getMRI()->getType(CmpVal);
867   LLT NewValTy = getMRI()->getType(NewVal);
868   assert(OldValResTy.isScalar() && "invalid operand type");
869   assert(AddrTy.isPointer() && "invalid operand type");
870   assert(CmpValTy.isValid() && "invalid operand type");
871   assert(NewValTy.isValid() && "invalid operand type");
872   assert(OldValResTy == CmpValTy && "type mismatch");
873   assert(OldValResTy == NewValTy && "type mismatch");
874 #endif
875 
876   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG)
877       .addDef(OldValRes)
878       .addUse(Addr)
879       .addUse(CmpVal)
880       .addUse(NewVal)
881       .addMemOperand(&MMO);
882 }
883 
884 MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
885   unsigned Opcode, const DstOp &OldValRes,
886   const SrcOp &Addr, const SrcOp &Val,
887   MachineMemOperand &MMO) {
888 
889 #ifndef NDEBUG
890   LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
891   LLT AddrTy = Addr.getLLTTy(*getMRI());
892   LLT ValTy = Val.getLLTTy(*getMRI());
893   assert(OldValResTy.isScalar() && "invalid operand type");
894   assert(AddrTy.isPointer() && "invalid operand type");
895   assert(ValTy.isValid() && "invalid operand type");
896   assert(OldValResTy == ValTy && "type mismatch");
897   assert(MMO.isAtomic() && "not atomic mem operand");
898 #endif
899 
900   auto MIB = buildInstr(Opcode);
901   OldValRes.addDefToMIB(*getMRI(), MIB);
902   Addr.addSrcToMIB(MIB);
903   Val.addSrcToMIB(MIB);
904   MIB.addMemOperand(&MMO);
905   return MIB;
906 }
907 
908 MachineInstrBuilder
909 MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
910                                      Register Val, MachineMemOperand &MMO) {
911   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
912                         MMO);
913 }
914 MachineInstrBuilder
915 MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
916                                     Register Val, MachineMemOperand &MMO) {
917   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
918                         MMO);
919 }
920 MachineInstrBuilder
921 MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
922                                     Register Val, MachineMemOperand &MMO) {
923   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
924                         MMO);
925 }
926 MachineInstrBuilder
927 MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
928                                     Register Val, MachineMemOperand &MMO) {
929   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
930                         MMO);
931 }
932 MachineInstrBuilder
933 MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
934                                      Register Val, MachineMemOperand &MMO) {
935   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
936                         MMO);
937 }
938 MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
939                                                        Register Addr,
940                                                        Register Val,
941                                                        MachineMemOperand &MMO) {
942   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
943                         MMO);
944 }
945 MachineInstrBuilder
946 MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
947                                     Register Val, MachineMemOperand &MMO) {
948   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
949                         MMO);
950 }
951 MachineInstrBuilder
952 MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
953                                     Register Val, MachineMemOperand &MMO) {
954   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
955                         MMO);
956 }
957 MachineInstrBuilder
958 MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
959                                     Register Val, MachineMemOperand &MMO) {
960   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
961                         MMO);
962 }
963 MachineInstrBuilder
964 MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
965                                      Register Val, MachineMemOperand &MMO) {
966   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
967                         MMO);
968 }
969 MachineInstrBuilder
970 MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
971                                      Register Val, MachineMemOperand &MMO) {
972   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
973                         MMO);
974 }
975 
976 MachineInstrBuilder
977 MachineIRBuilder::buildAtomicRMWFAdd(
978   const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
979   MachineMemOperand &MMO) {
980   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
981                         MMO);
982 }
983 
984 MachineInstrBuilder
985 MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
986                                      MachineMemOperand &MMO) {
987   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
988                         MMO);
989 }
990 
991 MachineInstrBuilder
992 MachineIRBuilder::buildAtomicRMWFMax(const DstOp &OldValRes, const SrcOp &Addr,
993                                      const SrcOp &Val, MachineMemOperand &MMO) {
994   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FMAX, OldValRes, Addr, Val,
995                         MMO);
996 }
997 
998 MachineInstrBuilder
999 MachineIRBuilder::buildAtomicRMWFMin(const DstOp &OldValRes, const SrcOp &Addr,
1000                                      const SrcOp &Val, MachineMemOperand &MMO) {
1001   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FMIN, OldValRes, Addr, Val,
1002                         MMO);
1003 }
1004 
1005 MachineInstrBuilder
1006 MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
1007   return buildInstr(TargetOpcode::G_FENCE)
1008     .addImm(Ordering)
1009     .addImm(Scope);
1010 }
1011 
1012 MachineInstrBuilder
1013 MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
1014 #ifndef NDEBUG
1015   assert(getMRI()->getType(Res).isPointer() && "invalid res type");
1016 #endif
1017 
1018   return buildInstr(TargetOpcode::G_BLOCK_ADDR).addDef(Res).addBlockAddress(BA);
1019 }
1020 
1021 void MachineIRBuilder::validateTruncExt(const LLT DstTy, const LLT SrcTy,
1022                                         bool IsExtend) {
1023 #ifndef NDEBUG
1024   if (DstTy.isVector()) {
1025     assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
1026     assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
1027            "different number of elements in a trunc/ext");
1028   } else
1029     assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
1030 
1031   if (IsExtend)
1032     assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
1033            "invalid narrowing extend");
1034   else
1035     assert(DstTy.getSizeInBits() < SrcTy.getSizeInBits() &&
1036            "invalid widening trunc");
1037 #endif
1038 }
1039 
1040 void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
1041                                         const LLT Op0Ty, const LLT Op1Ty) {
1042 #ifndef NDEBUG
1043   assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
1044          "invalid operand type");
1045   assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
1046   if (ResTy.isScalar() || ResTy.isPointer())
1047     assert(TstTy.isScalar() && "type mismatch");
1048   else
1049     assert((TstTy.isScalar() ||
1050             (TstTy.isVector() &&
1051              TstTy.getNumElements() == Op0Ty.getNumElements())) &&
1052            "type mismatch");
1053 #endif
1054 }
1055 
1056 MachineInstrBuilder
1057 MachineIRBuilder::buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
1058                              ArrayRef<SrcOp> SrcOps,
1059                              std::optional<unsigned> Flags) {
1060   switch (Opc) {
1061   default:
1062     break;
1063   case TargetOpcode::G_SELECT: {
1064     assert(DstOps.size() == 1 && "Invalid select");
1065     assert(SrcOps.size() == 3 && "Invalid select");
1066     validateSelectOp(
1067         DstOps[0].getLLTTy(*getMRI()), SrcOps[0].getLLTTy(*getMRI()),
1068         SrcOps[1].getLLTTy(*getMRI()), SrcOps[2].getLLTTy(*getMRI()));
1069     break;
1070   }
1071   case TargetOpcode::G_FNEG:
1072   case TargetOpcode::G_ABS:
1073     // All these are unary ops.
1074     assert(DstOps.size() == 1 && "Invalid Dst");
1075     assert(SrcOps.size() == 1 && "Invalid Srcs");
1076     validateUnaryOp(DstOps[0].getLLTTy(*getMRI()),
1077                     SrcOps[0].getLLTTy(*getMRI()));
1078     break;
1079   case TargetOpcode::G_ADD:
1080   case TargetOpcode::G_AND:
1081   case TargetOpcode::G_MUL:
1082   case TargetOpcode::G_OR:
1083   case TargetOpcode::G_SUB:
1084   case TargetOpcode::G_XOR:
1085   case TargetOpcode::G_UDIV:
1086   case TargetOpcode::G_SDIV:
1087   case TargetOpcode::G_UREM:
1088   case TargetOpcode::G_SREM:
1089   case TargetOpcode::G_SMIN:
1090   case TargetOpcode::G_SMAX:
1091   case TargetOpcode::G_UMIN:
1092   case TargetOpcode::G_UMAX:
1093   case TargetOpcode::G_UADDSAT:
1094   case TargetOpcode::G_SADDSAT:
1095   case TargetOpcode::G_USUBSAT:
1096   case TargetOpcode::G_SSUBSAT: {
1097     // All these are binary ops.
1098     assert(DstOps.size() == 1 && "Invalid Dst");
1099     assert(SrcOps.size() == 2 && "Invalid Srcs");
1100     validateBinaryOp(DstOps[0].getLLTTy(*getMRI()),
1101                      SrcOps[0].getLLTTy(*getMRI()),
1102                      SrcOps[1].getLLTTy(*getMRI()));
1103     break;
1104   }
1105   case TargetOpcode::G_SHL:
1106   case TargetOpcode::G_ASHR:
1107   case TargetOpcode::G_LSHR:
1108   case TargetOpcode::G_USHLSAT:
1109   case TargetOpcode::G_SSHLSAT: {
1110     assert(DstOps.size() == 1 && "Invalid Dst");
1111     assert(SrcOps.size() == 2 && "Invalid Srcs");
1112     validateShiftOp(DstOps[0].getLLTTy(*getMRI()),
1113                     SrcOps[0].getLLTTy(*getMRI()),
1114                     SrcOps[1].getLLTTy(*getMRI()));
1115     break;
1116   }
1117   case TargetOpcode::G_SEXT:
1118   case TargetOpcode::G_ZEXT:
1119   case TargetOpcode::G_ANYEXT:
1120     assert(DstOps.size() == 1 && "Invalid Dst");
1121     assert(SrcOps.size() == 1 && "Invalid Srcs");
1122     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1123                      SrcOps[0].getLLTTy(*getMRI()), true);
1124     break;
1125   case TargetOpcode::G_TRUNC:
1126   case TargetOpcode::G_FPTRUNC: {
1127     assert(DstOps.size() == 1 && "Invalid Dst");
1128     assert(SrcOps.size() == 1 && "Invalid Srcs");
1129     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1130                      SrcOps[0].getLLTTy(*getMRI()), false);
1131     break;
1132   }
1133   case TargetOpcode::G_BITCAST: {
1134     assert(DstOps.size() == 1 && "Invalid Dst");
1135     assert(SrcOps.size() == 1 && "Invalid Srcs");
1136     assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1137            SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1138     break;
1139   }
1140   case TargetOpcode::COPY:
1141     assert(DstOps.size() == 1 && "Invalid Dst");
1142     // If the caller wants to add a subreg source it has to be done separately
1143     // so we may not have any SrcOps at this point yet.
1144     break;
1145   case TargetOpcode::G_FCMP:
1146   case TargetOpcode::G_ICMP: {
1147     assert(DstOps.size() == 1 && "Invalid Dst Operands");
1148     assert(SrcOps.size() == 3 && "Invalid Src Operands");
1149     // For F/ICMP, the first src operand is the predicate, followed by
1150     // the two comparands.
1151     assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1152            "Expecting predicate");
1153     assert([&]() -> bool {
1154       CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1155       return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1156                                          : CmpInst::isFPPredicate(Pred);
1157     }() && "Invalid predicate");
1158     assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1159            "Type mismatch");
1160     assert([&]() -> bool {
1161       LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1162       LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1163       if (Op0Ty.isScalar() || Op0Ty.isPointer())
1164         return DstTy.isScalar();
1165       else
1166         return DstTy.isVector() &&
1167                DstTy.getNumElements() == Op0Ty.getNumElements();
1168     }() && "Type Mismatch");
1169     break;
1170   }
1171   case TargetOpcode::G_UNMERGE_VALUES: {
1172     assert(!DstOps.empty() && "Invalid trivial sequence");
1173     assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1174     assert(llvm::all_of(DstOps,
1175                         [&, this](const DstOp &Op) {
1176                           return Op.getLLTTy(*getMRI()) ==
1177                                  DstOps[0].getLLTTy(*getMRI());
1178                         }) &&
1179            "type mismatch in output list");
1180     assert((TypeSize::ScalarTy)DstOps.size() *
1181                    DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1182                SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1183            "input operands do not cover output register");
1184     break;
1185   }
1186   case TargetOpcode::G_MERGE_VALUES: {
1187     assert(SrcOps.size() >= 2 && "invalid trivial sequence");
1188     assert(DstOps.size() == 1 && "Invalid Dst");
1189     assert(llvm::all_of(SrcOps,
1190                         [&, this](const SrcOp &Op) {
1191                           return Op.getLLTTy(*getMRI()) ==
1192                                  SrcOps[0].getLLTTy(*getMRI());
1193                         }) &&
1194            "type mismatch in input list");
1195     assert((TypeSize::ScalarTy)SrcOps.size() *
1196                    SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1197                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1198            "input operands do not cover output register");
1199     assert(!DstOps[0].getLLTTy(*getMRI()).isVector() &&
1200            "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");
1201     break;
1202   }
1203   case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1204     assert(DstOps.size() == 1 && "Invalid Dst size");
1205     assert(SrcOps.size() == 2 && "Invalid Src size");
1206     assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1207     assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1208             DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1209            "Invalid operand type");
1210     assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1211     assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1212                DstOps[0].getLLTTy(*getMRI()) &&
1213            "Type mismatch");
1214     break;
1215   }
1216   case TargetOpcode::G_INSERT_VECTOR_ELT: {
1217     assert(DstOps.size() == 1 && "Invalid dst size");
1218     assert(SrcOps.size() == 3 && "Invalid src size");
1219     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1220            SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1221     assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1222                SrcOps[1].getLLTTy(*getMRI()) &&
1223            "Type mismatch");
1224     assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1225     assert(DstOps[0].getLLTTy(*getMRI()).getNumElements() ==
1226                SrcOps[0].getLLTTy(*getMRI()).getNumElements() &&
1227            "Type mismatch");
1228     break;
1229   }
1230   case TargetOpcode::G_BUILD_VECTOR: {
1231     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1232            "Must have at least 2 operands");
1233     assert(DstOps.size() == 1 && "Invalid DstOps");
1234     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1235            "Res type must be a vector");
1236     assert(llvm::all_of(SrcOps,
1237                         [&, this](const SrcOp &Op) {
1238                           return Op.getLLTTy(*getMRI()) ==
1239                                  SrcOps[0].getLLTTy(*getMRI());
1240                         }) &&
1241            "type mismatch in input list");
1242     assert((TypeSize::ScalarTy)SrcOps.size() *
1243                    SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1244                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1245            "input scalars do not exactly cover the output vector register");
1246     break;
1247   }
1248   case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1249     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1250            "Must have at least 2 operands");
1251     assert(DstOps.size() == 1 && "Invalid DstOps");
1252     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1253            "Res type must be a vector");
1254     assert(llvm::all_of(SrcOps,
1255                         [&, this](const SrcOp &Op) {
1256                           return Op.getLLTTy(*getMRI()) ==
1257                                  SrcOps[0].getLLTTy(*getMRI());
1258                         }) &&
1259            "type mismatch in input list");
1260     break;
1261   }
1262   case TargetOpcode::G_CONCAT_VECTORS: {
1263     assert(DstOps.size() == 1 && "Invalid DstOps");
1264     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1265            "Must have at least 2 operands");
1266     assert(llvm::all_of(SrcOps,
1267                         [&, this](const SrcOp &Op) {
1268                           return (Op.getLLTTy(*getMRI()).isVector() &&
1269                                   Op.getLLTTy(*getMRI()) ==
1270                                       SrcOps[0].getLLTTy(*getMRI()));
1271                         }) &&
1272            "type mismatch in input list");
1273     assert((TypeSize::ScalarTy)SrcOps.size() *
1274                    SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1275                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1276            "input vectors do not exactly cover the output vector register");
1277     break;
1278   }
1279   case TargetOpcode::G_UADDE: {
1280     assert(DstOps.size() == 2 && "Invalid no of dst operands");
1281     assert(SrcOps.size() == 3 && "Invalid no of src operands");
1282     assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1283     assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1284            (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1285            "Invalid operand");
1286     assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1287     assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1288            "type mismatch");
1289     break;
1290   }
1291   }
1292 
1293   auto MIB = buildInstr(Opc);
1294   for (const DstOp &Op : DstOps)
1295     Op.addDefToMIB(*getMRI(), MIB);
1296   for (const SrcOp &Op : SrcOps)
1297     Op.addSrcToMIB(MIB);
1298   if (Flags)
1299     MIB->setFlags(*Flags);
1300   return MIB;
1301 }
1302