xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp (revision 5b56413d04e608379c9a306373554a8e4d321bc0)
1 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the MachineIRBuidler class.
10 //===----------------------------------------------------------------------===//
11 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12 #include "llvm/CodeGen/MachineFunction.h"
13 #include "llvm/CodeGen/MachineInstr.h"
14 #include "llvm/CodeGen/MachineInstrBuilder.h"
15 #include "llvm/CodeGen/MachineRegisterInfo.h"
16 #include "llvm/CodeGen/TargetInstrInfo.h"
17 #include "llvm/CodeGen/TargetLowering.h"
18 #include "llvm/CodeGen/TargetOpcodes.h"
19 #include "llvm/CodeGen/TargetSubtargetInfo.h"
20 #include "llvm/IR/DebugInfoMetadata.h"
21 
22 using namespace llvm;
23 
24 void MachineIRBuilder::setMF(MachineFunction &MF) {
25   State.MF = &MF;
26   State.MBB = nullptr;
27   State.MRI = &MF.getRegInfo();
28   State.TII = MF.getSubtarget().getInstrInfo();
29   State.DL = DebugLoc();
30   State.PCSections = nullptr;
31   State.II = MachineBasicBlock::iterator();
32   State.Observer = nullptr;
33 }
34 
35 //------------------------------------------------------------------------------
36 // Build instruction variants.
37 //------------------------------------------------------------------------------
38 
39 MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
40   return BuildMI(getMF(), {getDL(), getPCSections()}, getTII().get(Opcode));
41 }
42 
43 MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
44   getMBB().insert(getInsertPt(), MIB);
45   recordInsertion(MIB);
46   return MIB;
47 }
48 
49 MachineInstrBuilder
50 MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
51                                       const MDNode *Expr) {
52   assert(isa<DILocalVariable>(Variable) && "not a variable");
53   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
54   assert(
55       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
56       "Expected inlined-at fields to agree");
57   return insertInstr(BuildMI(getMF(), getDL(),
58                              getTII().get(TargetOpcode::DBG_VALUE),
59                              /*IsIndirect*/ false, Reg, Variable, Expr));
60 }
61 
62 MachineInstrBuilder
63 MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
64                                         const MDNode *Expr) {
65   assert(isa<DILocalVariable>(Variable) && "not a variable");
66   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
67   assert(
68       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
69       "Expected inlined-at fields to agree");
70   return insertInstr(BuildMI(getMF(), getDL(),
71                              getTII().get(TargetOpcode::DBG_VALUE),
72                              /*IsIndirect*/ true, Reg, Variable, Expr));
73 }
74 
75 MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
76                                                       const MDNode *Variable,
77                                                       const MDNode *Expr) {
78   assert(isa<DILocalVariable>(Variable) && "not a variable");
79   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
80   assert(
81       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
82       "Expected inlined-at fields to agree");
83   return insertInstr(buildInstrNoInsert(TargetOpcode::DBG_VALUE)
84                          .addFrameIndex(FI)
85                          .addImm(0)
86                          .addMetadata(Variable)
87                          .addMetadata(Expr));
88 }
89 
90 MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
91                                                          const MDNode *Variable,
92                                                          const MDNode *Expr) {
93   assert(isa<DILocalVariable>(Variable) && "not a variable");
94   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
95   assert(
96       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
97       "Expected inlined-at fields to agree");
98   auto MIB = buildInstrNoInsert(TargetOpcode::DBG_VALUE);
99 
100   auto *NumericConstant = [&] () -> const Constant* {
101     if (const auto *CE = dyn_cast<ConstantExpr>(&C))
102       if (CE->getOpcode() == Instruction::IntToPtr)
103         return CE->getOperand(0);
104     return &C;
105   }();
106 
107   if (auto *CI = dyn_cast<ConstantInt>(NumericConstant)) {
108     if (CI->getBitWidth() > 64)
109       MIB.addCImm(CI);
110     else
111       MIB.addImm(CI->getZExtValue());
112   } else if (auto *CFP = dyn_cast<ConstantFP>(NumericConstant)) {
113     MIB.addFPImm(CFP);
114   } else if (isa<ConstantPointerNull>(NumericConstant)) {
115     MIB.addImm(0);
116   } else {
117     // Insert $noreg if we didn't find a usable constant and had to drop it.
118     MIB.addReg(Register());
119   }
120 
121   MIB.addImm(0).addMetadata(Variable).addMetadata(Expr);
122   return insertInstr(MIB);
123 }
124 
125 MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
126   assert(isa<DILabel>(Label) && "not a label");
127   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
128          "Expected inlined-at fields to agree");
129   auto MIB = buildInstr(TargetOpcode::DBG_LABEL);
130 
131   return MIB.addMetadata(Label);
132 }
133 
134 MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
135                                                          const SrcOp &Size,
136                                                          Align Alignment) {
137   assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
138   auto MIB = buildInstr(TargetOpcode::G_DYN_STACKALLOC);
139   Res.addDefToMIB(*getMRI(), MIB);
140   Size.addSrcToMIB(MIB);
141   MIB.addImm(Alignment.value());
142   return MIB;
143 }
144 
145 MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
146                                                       int Idx) {
147   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
148   auto MIB = buildInstr(TargetOpcode::G_FRAME_INDEX);
149   Res.addDefToMIB(*getMRI(), MIB);
150   MIB.addFrameIndex(Idx);
151   return MIB;
152 }
153 
154 MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
155                                                        const GlobalValue *GV) {
156   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
157   assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
158              GV->getType()->getAddressSpace() &&
159          "address space mismatch");
160 
161   auto MIB = buildInstr(TargetOpcode::G_GLOBAL_VALUE);
162   Res.addDefToMIB(*getMRI(), MIB);
163   MIB.addGlobalAddress(GV);
164   return MIB;
165 }
166 
167 MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
168                                                         unsigned Idx) {
169   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
170   auto MIB = buildInstr(TargetOpcode::G_CONSTANT_POOL);
171   Res.addDefToMIB(*getMRI(), MIB);
172   MIB.addConstantPoolIndex(Idx);
173   return MIB;
174 }
175 
176 MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
177                                                      unsigned JTI) {
178   return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {})
179       .addJumpTableIndex(JTI);
180 }
181 
182 void MachineIRBuilder::validateUnaryOp(const LLT Res, const LLT Op0) {
183   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
184   assert((Res == Op0) && "type mismatch");
185 }
186 
187 void MachineIRBuilder::validateBinaryOp(const LLT Res, const LLT Op0,
188                                         const LLT Op1) {
189   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
190   assert((Res == Op0 && Res == Op1) && "type mismatch");
191 }
192 
193 void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0,
194                                        const LLT Op1) {
195   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
196   assert((Res == Op0) && "type mismatch");
197 }
198 
199 MachineInstrBuilder
200 MachineIRBuilder::buildPtrAdd(const DstOp &Res, const SrcOp &Op0,
201                               const SrcOp &Op1, std::optional<unsigned> Flags) {
202   assert(Res.getLLTTy(*getMRI()).getScalarType().isPointer() &&
203          Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
204   assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
205 
206   return buildInstr(TargetOpcode::G_PTR_ADD, {Res}, {Op0, Op1}, Flags);
207 }
208 
209 std::optional<MachineInstrBuilder>
210 MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
211                                     const LLT ValueTy, uint64_t Value) {
212   assert(Res == 0 && "Res is a result argument");
213   assert(ValueTy.isScalar()  && "invalid offset type");
214 
215   if (Value == 0) {
216     Res = Op0;
217     return std::nullopt;
218   }
219 
220   Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0));
221   auto Cst = buildConstant(ValueTy, Value);
222   return buildPtrAdd(Res, Op0, Cst.getReg(0));
223 }
224 
225 MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
226                                                           const SrcOp &Op0,
227                                                           uint32_t NumBits) {
228   LLT PtrTy = Res.getLLTTy(*getMRI());
229   LLT MaskTy = LLT::scalar(PtrTy.getSizeInBits());
230   Register MaskReg = getMRI()->createGenericVirtualRegister(MaskTy);
231   buildConstant(MaskReg, maskTrailingZeros<uint64_t>(NumBits));
232   return buildPtrMask(Res, Op0, MaskReg);
233 }
234 
235 MachineInstrBuilder
236 MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
237                                                   const SrcOp &Op0) {
238   LLT ResTy = Res.getLLTTy(*getMRI());
239   LLT Op0Ty = Op0.getLLTTy(*getMRI());
240 
241   assert(ResTy.isVector() && "Res non vector type");
242 
243   SmallVector<Register, 8> Regs;
244   if (Op0Ty.isVector()) {
245     assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
246            "Different vector element types");
247     assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
248            "Op0 has more elements");
249     auto Unmerge = buildUnmerge(Op0Ty.getElementType(), Op0);
250 
251     for (auto Op : Unmerge.getInstr()->defs())
252       Regs.push_back(Op.getReg());
253   } else {
254     assert((ResTy.getSizeInBits() > Op0Ty.getSizeInBits()) &&
255            "Op0 has more size");
256     Regs.push_back(Op0.getReg());
257   }
258   Register Undef =
259       buildUndef(Op0Ty.isVector() ? Op0Ty.getElementType() : Op0Ty).getReg(0);
260   unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
261   for (unsigned i = 0; i < NumberOfPadElts; ++i)
262     Regs.push_back(Undef);
263   return buildMergeLikeInstr(Res, Regs);
264 }
265 
266 MachineInstrBuilder
267 MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
268                                                     const SrcOp &Op0) {
269   LLT ResTy = Res.getLLTTy(*getMRI());
270   LLT Op0Ty = Op0.getLLTTy(*getMRI());
271 
272   assert((ResTy.isVector() && Op0Ty.isVector()) && "Non vector type");
273   assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
274          "Different vector element types");
275   assert((ResTy.getNumElements() < Op0Ty.getNumElements()) &&
276          "Op0 has fewer elements");
277 
278   SmallVector<Register, 8> Regs;
279   auto Unmerge = buildUnmerge(Op0Ty.getElementType(), Op0);
280   for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
281     Regs.push_back(Unmerge.getReg(i));
282   return buildMergeLikeInstr(Res, Regs);
283 }
284 
285 MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
286   return buildInstr(TargetOpcode::G_BR).addMBB(&Dest);
287 }
288 
289 MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
290   assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
291   return buildInstr(TargetOpcode::G_BRINDIRECT).addUse(Tgt);
292 }
293 
294 MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
295                                                 unsigned JTI,
296                                                 Register IndexReg) {
297   assert(getMRI()->getType(TablePtr).isPointer() &&
298          "Table reg must be a pointer");
299   return buildInstr(TargetOpcode::G_BRJT)
300       .addUse(TablePtr)
301       .addJumpTableIndex(JTI)
302       .addUse(IndexReg);
303 }
304 
305 MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
306                                                 const SrcOp &Op) {
307   return buildInstr(TargetOpcode::COPY, Res, Op);
308 }
309 
310 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
311                                                     const ConstantInt &Val) {
312   LLT Ty = Res.getLLTTy(*getMRI());
313   LLT EltTy = Ty.getScalarType();
314   assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
315          "creating constant with the wrong size");
316 
317   assert(!Ty.isScalableVector() &&
318          "unexpected scalable vector in buildConstant");
319 
320   if (Ty.isFixedVector()) {
321     auto Const = buildInstr(TargetOpcode::G_CONSTANT)
322     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
323     .addCImm(&Val);
324     return buildSplatVector(Res, Const);
325   }
326 
327   auto Const = buildInstr(TargetOpcode::G_CONSTANT);
328   Const->setDebugLoc(DebugLoc());
329   Res.addDefToMIB(*getMRI(), Const);
330   Const.addCImm(&Val);
331   return Const;
332 }
333 
334 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
335                                                     int64_t Val) {
336   auto IntN = IntegerType::get(getMF().getFunction().getContext(),
337                                Res.getLLTTy(*getMRI()).getScalarSizeInBits());
338   ConstantInt *CI = ConstantInt::get(IntN, Val, true);
339   return buildConstant(Res, *CI);
340 }
341 
342 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
343                                                      const ConstantFP &Val) {
344   LLT Ty = Res.getLLTTy(*getMRI());
345   LLT EltTy = Ty.getScalarType();
346 
347   assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
348          == EltTy.getSizeInBits() &&
349          "creating fconstant with the wrong size");
350 
351   assert(!Ty.isPointer() && "invalid operand type");
352 
353   assert(!Ty.isScalableVector() &&
354          "unexpected scalable vector in buildFConstant");
355 
356   if (Ty.isFixedVector()) {
357     auto Const = buildInstr(TargetOpcode::G_FCONSTANT)
358     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
359     .addFPImm(&Val);
360 
361     return buildSplatVector(Res, Const);
362   }
363 
364   auto Const = buildInstr(TargetOpcode::G_FCONSTANT);
365   Const->setDebugLoc(DebugLoc());
366   Res.addDefToMIB(*getMRI(), Const);
367   Const.addFPImm(&Val);
368   return Const;
369 }
370 
371 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
372                                                     const APInt &Val) {
373   ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val);
374   return buildConstant(Res, *CI);
375 }
376 
377 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
378                                                      double Val) {
379   LLT DstTy = Res.getLLTTy(*getMRI());
380   auto &Ctx = getMF().getFunction().getContext();
381   auto *CFP =
382       ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits()));
383   return buildFConstant(Res, *CFP);
384 }
385 
386 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
387                                                      const APFloat &Val) {
388   auto &Ctx = getMF().getFunction().getContext();
389   auto *CFP = ConstantFP::get(Ctx, Val);
390   return buildFConstant(Res, *CFP);
391 }
392 
393 MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
394                                                   MachineBasicBlock &Dest) {
395   assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
396 
397   auto MIB = buildInstr(TargetOpcode::G_BRCOND);
398   Tst.addSrcToMIB(MIB);
399   MIB.addMBB(&Dest);
400   return MIB;
401 }
402 
403 MachineInstrBuilder
404 MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
405                             MachinePointerInfo PtrInfo, Align Alignment,
406                             MachineMemOperand::Flags MMOFlags,
407                             const AAMDNodes &AAInfo) {
408   MMOFlags |= MachineMemOperand::MOLoad;
409   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
410 
411   LLT Ty = Dst.getLLTTy(*getMRI());
412   MachineMemOperand *MMO =
413       getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo);
414   return buildLoad(Dst, Addr, *MMO);
415 }
416 
417 MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
418                                                      const DstOp &Res,
419                                                      const SrcOp &Addr,
420                                                      MachineMemOperand &MMO) {
421   assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
422   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
423 
424   auto MIB = buildInstr(Opcode);
425   Res.addDefToMIB(*getMRI(), MIB);
426   Addr.addSrcToMIB(MIB);
427   MIB.addMemOperand(&MMO);
428   return MIB;
429 }
430 
431 MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
432   const DstOp &Dst, const SrcOp &BasePtr,
433   MachineMemOperand &BaseMMO, int64_t Offset) {
434   LLT LoadTy = Dst.getLLTTy(*getMRI());
435   MachineMemOperand *OffsetMMO =
436       getMF().getMachineMemOperand(&BaseMMO, Offset, LoadTy);
437 
438   if (Offset == 0) // This may be a size or type changing load.
439     return buildLoad(Dst, BasePtr, *OffsetMMO);
440 
441   LLT PtrTy = BasePtr.getLLTTy(*getMRI());
442   LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits());
443   auto ConstOffset = buildConstant(OffsetTy, Offset);
444   auto Ptr = buildPtrAdd(PtrTy, BasePtr, ConstOffset);
445   return buildLoad(Dst, Ptr, *OffsetMMO);
446 }
447 
448 MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
449                                                  const SrcOp &Addr,
450                                                  MachineMemOperand &MMO) {
451   assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
452   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
453 
454   auto MIB = buildInstr(TargetOpcode::G_STORE);
455   Val.addSrcToMIB(MIB);
456   Addr.addSrcToMIB(MIB);
457   MIB.addMemOperand(&MMO);
458   return MIB;
459 }
460 
461 MachineInstrBuilder
462 MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
463                              MachinePointerInfo PtrInfo, Align Alignment,
464                              MachineMemOperand::Flags MMOFlags,
465                              const AAMDNodes &AAInfo) {
466   MMOFlags |= MachineMemOperand::MOStore;
467   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
468 
469   LLT Ty = Val.getLLTTy(*getMRI());
470   MachineMemOperand *MMO =
471       getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo);
472   return buildStore(Val, Addr, *MMO);
473 }
474 
475 MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
476                                                   const SrcOp &Op) {
477   return buildInstr(TargetOpcode::G_ANYEXT, Res, Op);
478 }
479 
480 MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
481                                                 const SrcOp &Op) {
482   return buildInstr(TargetOpcode::G_SEXT, Res, Op);
483 }
484 
485 MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
486                                                 const SrcOp &Op) {
487   return buildInstr(TargetOpcode::G_ZEXT, Res, Op);
488 }
489 
490 unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
491   const auto *TLI = getMF().getSubtarget().getTargetLowering();
492   switch (TLI->getBooleanContents(IsVec, IsFP)) {
493   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
494     return TargetOpcode::G_SEXT;
495   case TargetLoweringBase::ZeroOrOneBooleanContent:
496     return TargetOpcode::G_ZEXT;
497   default:
498     return TargetOpcode::G_ANYEXT;
499   }
500 }
501 
502 MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
503                                                    const SrcOp &Op,
504                                                    bool IsFP) {
505   unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP);
506   return buildInstr(ExtOp, Res, Op);
507 }
508 
509 MachineInstrBuilder MachineIRBuilder::buildBoolExtInReg(const DstOp &Res,
510                                                         const SrcOp &Op,
511                                                         bool IsVector,
512                                                         bool IsFP) {
513   const auto *TLI = getMF().getSubtarget().getTargetLowering();
514   switch (TLI->getBooleanContents(IsVector, IsFP)) {
515   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
516     return buildSExtInReg(Res, Op, 1);
517   case TargetLoweringBase::ZeroOrOneBooleanContent:
518     return buildZExtInReg(Res, Op, 1);
519   case TargetLoweringBase::UndefinedBooleanContent:
520     return buildCopy(Res, Op);
521   }
522 
523   llvm_unreachable("unexpected BooleanContent");
524 }
525 
526 MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
527                                                       const DstOp &Res,
528                                                       const SrcOp &Op) {
529   assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
530           TargetOpcode::G_SEXT == ExtOpc) &&
531          "Expecting Extending Opc");
532   assert(Res.getLLTTy(*getMRI()).isScalar() ||
533          Res.getLLTTy(*getMRI()).isVector());
534   assert(Res.getLLTTy(*getMRI()).isScalar() ==
535          Op.getLLTTy(*getMRI()).isScalar());
536 
537   unsigned Opcode = TargetOpcode::COPY;
538   if (Res.getLLTTy(*getMRI()).getSizeInBits() >
539       Op.getLLTTy(*getMRI()).getSizeInBits())
540     Opcode = ExtOpc;
541   else if (Res.getLLTTy(*getMRI()).getSizeInBits() <
542            Op.getLLTTy(*getMRI()).getSizeInBits())
543     Opcode = TargetOpcode::G_TRUNC;
544   else
545     assert(Res.getLLTTy(*getMRI()) == Op.getLLTTy(*getMRI()));
546 
547   return buildInstr(Opcode, Res, Op);
548 }
549 
550 MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
551                                                        const SrcOp &Op) {
552   return buildExtOrTrunc(TargetOpcode::G_SEXT, Res, Op);
553 }
554 
555 MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
556                                                        const SrcOp &Op) {
557   return buildExtOrTrunc(TargetOpcode::G_ZEXT, Res, Op);
558 }
559 
560 MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
561                                                          const SrcOp &Op) {
562   return buildExtOrTrunc(TargetOpcode::G_ANYEXT, Res, Op);
563 }
564 
565 MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
566                                                      const SrcOp &Op,
567                                                      int64_t ImmOp) {
568   LLT ResTy = Res.getLLTTy(*getMRI());
569   auto Mask = buildConstant(
570       ResTy, APInt::getLowBitsSet(ResTy.getScalarSizeInBits(), ImmOp));
571   return buildAnd(Res, Op, Mask);
572 }
573 
574 MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
575                                                 const SrcOp &Src) {
576   LLT SrcTy = Src.getLLTTy(*getMRI());
577   LLT DstTy = Dst.getLLTTy(*getMRI());
578   if (SrcTy == DstTy)
579     return buildCopy(Dst, Src);
580 
581   unsigned Opcode;
582   if (SrcTy.isPointer() && DstTy.isScalar())
583     Opcode = TargetOpcode::G_PTRTOINT;
584   else if (DstTy.isPointer() && SrcTy.isScalar())
585     Opcode = TargetOpcode::G_INTTOPTR;
586   else {
587     assert(!SrcTy.isPointer() && !DstTy.isPointer() && "n G_ADDRCAST yet");
588     Opcode = TargetOpcode::G_BITCAST;
589   }
590 
591   return buildInstr(Opcode, Dst, Src);
592 }
593 
594 MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
595                                                    const SrcOp &Src,
596                                                    uint64_t Index) {
597   LLT SrcTy = Src.getLLTTy(*getMRI());
598   LLT DstTy = Dst.getLLTTy(*getMRI());
599 
600 #ifndef NDEBUG
601   assert(SrcTy.isValid() && "invalid operand type");
602   assert(DstTy.isValid() && "invalid operand type");
603   assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
604          "extracting off end of register");
605 #endif
606 
607   if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
608     assert(Index == 0 && "insertion past the end of a register");
609     return buildCast(Dst, Src);
610   }
611 
612   auto Extract = buildInstr(TargetOpcode::G_EXTRACT);
613   Dst.addDefToMIB(*getMRI(), Extract);
614   Src.addSrcToMIB(Extract);
615   Extract.addImm(Index);
616   return Extract;
617 }
618 
619 MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
620   return buildInstr(TargetOpcode::G_IMPLICIT_DEF, {Res}, {});
621 }
622 
623 MachineInstrBuilder MachineIRBuilder::buildMergeValues(const DstOp &Res,
624                                                        ArrayRef<Register> Ops) {
625   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
626   // we need some temporary storage for the DstOp objects. Here we use a
627   // sufficiently large SmallVector to not go through the heap.
628   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
629   assert(TmpVec.size() > 1);
630   return buildInstr(TargetOpcode::G_MERGE_VALUES, Res, TmpVec);
631 }
632 
633 MachineInstrBuilder
634 MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
635                                       ArrayRef<Register> Ops) {
636   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
637   // we need some temporary storage for the DstOp objects. Here we use a
638   // sufficiently large SmallVector to not go through the heap.
639   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
640   assert(TmpVec.size() > 1);
641   return buildInstr(getOpcodeForMerge(Res, TmpVec), Res, TmpVec);
642 }
643 
644 MachineInstrBuilder
645 MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
646                                       std::initializer_list<SrcOp> Ops) {
647   assert(Ops.size() > 1);
648   return buildInstr(getOpcodeForMerge(Res, Ops), Res, Ops);
649 }
650 
651 unsigned MachineIRBuilder::getOpcodeForMerge(const DstOp &DstOp,
652                                              ArrayRef<SrcOp> SrcOps) const {
653   if (DstOp.getLLTTy(*getMRI()).isVector()) {
654     if (SrcOps[0].getLLTTy(*getMRI()).isVector())
655       return TargetOpcode::G_CONCAT_VECTORS;
656     return TargetOpcode::G_BUILD_VECTOR;
657   }
658 
659   return TargetOpcode::G_MERGE_VALUES;
660 }
661 
662 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
663                                                    const SrcOp &Op) {
664   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
665   // we need some temporary storage for the DstOp objects. Here we use a
666   // sufficiently large SmallVector to not go through the heap.
667   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
668   assert(TmpVec.size() > 1);
669   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
670 }
671 
672 MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
673                                                    const SrcOp &Op) {
674   unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits();
675   SmallVector<DstOp, 8> TmpVec(NumReg, Res);
676   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
677 }
678 
679 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
680                                                    const SrcOp &Op) {
681   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
682   // we need some temporary storage for the DstOp objects. Here we use a
683   // sufficiently large SmallVector to not go through the heap.
684   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
685   assert(TmpVec.size() > 1);
686   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
687 }
688 
689 MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
690                                                        ArrayRef<Register> Ops) {
691   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
692   // we need some temporary storage for the DstOp objects. Here we use a
693   // sufficiently large SmallVector to not go through the heap.
694   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
695   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
696 }
697 
698 MachineInstrBuilder
699 MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
700                                            ArrayRef<APInt> Ops) {
701   SmallVector<SrcOp> TmpVec;
702   TmpVec.reserve(Ops.size());
703   LLT EltTy = Res.getLLTTy(*getMRI()).getElementType();
704   for (const auto &Op : Ops)
705     TmpVec.push_back(buildConstant(EltTy, Op));
706   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
707 }
708 
709 MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
710                                                        const SrcOp &Src) {
711   SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
712   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
713 }
714 
715 MachineInstrBuilder
716 MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
717                                         ArrayRef<Register> Ops) {
718   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
719   // we need some temporary storage for the DstOp objects. Here we use a
720   // sufficiently large SmallVector to not go through the heap.
721   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
722   if (TmpVec[0].getLLTTy(*getMRI()).getSizeInBits() ==
723       Res.getLLTTy(*getMRI()).getElementType().getSizeInBits())
724     return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
725   return buildInstr(TargetOpcode::G_BUILD_VECTOR_TRUNC, Res, TmpVec);
726 }
727 
728 MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
729                                                         const SrcOp &Src) {
730   LLT DstTy = Res.getLLTTy(*getMRI());
731   assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
732          "Expected Src to match Dst elt ty");
733   auto UndefVec = buildUndef(DstTy);
734   auto Zero = buildConstant(LLT::scalar(64), 0);
735   auto InsElt = buildInsertVectorElement(DstTy, UndefVec, Src, Zero);
736   SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
737   return buildShuffleVector(DstTy, InsElt, UndefVec, ZeroMask);
738 }
739 
740 MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
741                                                          const SrcOp &Src1,
742                                                          const SrcOp &Src2,
743                                                          ArrayRef<int> Mask) {
744   LLT DstTy = Res.getLLTTy(*getMRI());
745   LLT Src1Ty = Src1.getLLTTy(*getMRI());
746   LLT Src2Ty = Src2.getLLTTy(*getMRI());
747   assert((size_t)(Src1Ty.getNumElements() + Src2Ty.getNumElements()) >=
748          Mask.size());
749   assert(DstTy.getElementType() == Src1Ty.getElementType() &&
750          DstTy.getElementType() == Src2Ty.getElementType());
751   (void)DstTy;
752   (void)Src1Ty;
753   (void)Src2Ty;
754   ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
755   return buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {Res}, {Src1, Src2})
756       .addShuffleMask(MaskAlloc);
757 }
758 
759 MachineInstrBuilder
760 MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
761   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
762   // we need some temporary storage for the DstOp objects. Here we use a
763   // sufficiently large SmallVector to not go through the heap.
764   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
765   return buildInstr(TargetOpcode::G_CONCAT_VECTORS, Res, TmpVec);
766 }
767 
768 MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
769                                                   const SrcOp &Src,
770                                                   const SrcOp &Op,
771                                                   unsigned Index) {
772   assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
773              Res.getLLTTy(*getMRI()).getSizeInBits() &&
774          "insertion past the end of a register");
775 
776   if (Res.getLLTTy(*getMRI()).getSizeInBits() ==
777       Op.getLLTTy(*getMRI()).getSizeInBits()) {
778     return buildCast(Res, Op);
779   }
780 
781   return buildInstr(TargetOpcode::G_INSERT, Res, {Src, Op, uint64_t(Index)});
782 }
783 
784 static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
785   if (HasSideEffects && IsConvergent)
786     return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
787   if (HasSideEffects)
788     return TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
789   if (IsConvergent)
790     return TargetOpcode::G_INTRINSIC_CONVERGENT;
791   return TargetOpcode::G_INTRINSIC;
792 }
793 
794 MachineInstrBuilder
795 MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
796                                  ArrayRef<Register> ResultRegs,
797                                  bool HasSideEffects, bool isConvergent) {
798   auto MIB = buildInstr(getIntrinsicOpcode(HasSideEffects, isConvergent));
799   for (unsigned ResultReg : ResultRegs)
800     MIB.addDef(ResultReg);
801   MIB.addIntrinsicID(ID);
802   return MIB;
803 }
804 
805 MachineInstrBuilder
806 MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
807                                  ArrayRef<Register> ResultRegs) {
808   auto Attrs = Intrinsic::getAttributes(getContext(), ID);
809   bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
810   bool isConvergent = Attrs.hasFnAttr(Attribute::Convergent);
811   return buildIntrinsic(ID, ResultRegs, HasSideEffects, isConvergent);
812 }
813 
814 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
815                                                      ArrayRef<DstOp> Results,
816                                                      bool HasSideEffects,
817                                                      bool isConvergent) {
818   auto MIB = buildInstr(getIntrinsicOpcode(HasSideEffects, isConvergent));
819   for (DstOp Result : Results)
820     Result.addDefToMIB(*getMRI(), MIB);
821   MIB.addIntrinsicID(ID);
822   return MIB;
823 }
824 
825 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
826                                                      ArrayRef<DstOp> Results) {
827   auto Attrs = Intrinsic::getAttributes(getContext(), ID);
828   bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
829   bool isConvergent = Attrs.hasFnAttr(Attribute::Convergent);
830   return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
831 }
832 
833 MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
834                                                  const SrcOp &Op) {
835   return buildInstr(TargetOpcode::G_TRUNC, Res, Op);
836 }
837 
838 MachineInstrBuilder
839 MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
840                                std::optional<unsigned> Flags) {
841   return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op, Flags);
842 }
843 
844 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
845                                                 const DstOp &Res,
846                                                 const SrcOp &Op0,
847                                                 const SrcOp &Op1) {
848   return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
849 }
850 
851 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
852                                                 const DstOp &Res,
853                                                 const SrcOp &Op0,
854                                                 const SrcOp &Op1,
855                                                 std::optional<unsigned> Flags) {
856 
857   return buildInstr(TargetOpcode::G_FCMP, Res, {Pred, Op0, Op1}, Flags);
858 }
859 
860 MachineInstrBuilder
861 MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
862                               const SrcOp &Op0, const SrcOp &Op1,
863                               std::optional<unsigned> Flags) {
864 
865   return buildInstr(TargetOpcode::G_SELECT, {Res}, {Tst, Op0, Op1}, Flags);
866 }
867 
868 MachineInstrBuilder
869 MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
870                                            const SrcOp &Elt, const SrcOp &Idx) {
871   return buildInstr(TargetOpcode::G_INSERT_VECTOR_ELT, Res, {Val, Elt, Idx});
872 }
873 
874 MachineInstrBuilder
875 MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
876                                             const SrcOp &Idx) {
877   return buildInstr(TargetOpcode::G_EXTRACT_VECTOR_ELT, Res, {Val, Idx});
878 }
879 
880 MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
881     Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal,
882     Register NewVal, MachineMemOperand &MMO) {
883 #ifndef NDEBUG
884   LLT OldValResTy = getMRI()->getType(OldValRes);
885   LLT SuccessResTy = getMRI()->getType(SuccessRes);
886   LLT AddrTy = getMRI()->getType(Addr);
887   LLT CmpValTy = getMRI()->getType(CmpVal);
888   LLT NewValTy = getMRI()->getType(NewVal);
889   assert(OldValResTy.isScalar() && "invalid operand type");
890   assert(SuccessResTy.isScalar() && "invalid operand type");
891   assert(AddrTy.isPointer() && "invalid operand type");
892   assert(CmpValTy.isValid() && "invalid operand type");
893   assert(NewValTy.isValid() && "invalid operand type");
894   assert(OldValResTy == CmpValTy && "type mismatch");
895   assert(OldValResTy == NewValTy && "type mismatch");
896 #endif
897 
898   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
899       .addDef(OldValRes)
900       .addDef(SuccessRes)
901       .addUse(Addr)
902       .addUse(CmpVal)
903       .addUse(NewVal)
904       .addMemOperand(&MMO);
905 }
906 
907 MachineInstrBuilder
908 MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr,
909                                      Register CmpVal, Register NewVal,
910                                      MachineMemOperand &MMO) {
911 #ifndef NDEBUG
912   LLT OldValResTy = getMRI()->getType(OldValRes);
913   LLT AddrTy = getMRI()->getType(Addr);
914   LLT CmpValTy = getMRI()->getType(CmpVal);
915   LLT NewValTy = getMRI()->getType(NewVal);
916   assert(OldValResTy.isScalar() && "invalid operand type");
917   assert(AddrTy.isPointer() && "invalid operand type");
918   assert(CmpValTy.isValid() && "invalid operand type");
919   assert(NewValTy.isValid() && "invalid operand type");
920   assert(OldValResTy == CmpValTy && "type mismatch");
921   assert(OldValResTy == NewValTy && "type mismatch");
922 #endif
923 
924   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG)
925       .addDef(OldValRes)
926       .addUse(Addr)
927       .addUse(CmpVal)
928       .addUse(NewVal)
929       .addMemOperand(&MMO);
930 }
931 
932 MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
933   unsigned Opcode, const DstOp &OldValRes,
934   const SrcOp &Addr, const SrcOp &Val,
935   MachineMemOperand &MMO) {
936 
937 #ifndef NDEBUG
938   LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
939   LLT AddrTy = Addr.getLLTTy(*getMRI());
940   LLT ValTy = Val.getLLTTy(*getMRI());
941   assert(OldValResTy.isScalar() && "invalid operand type");
942   assert(AddrTy.isPointer() && "invalid operand type");
943   assert(ValTy.isValid() && "invalid operand type");
944   assert(OldValResTy == ValTy && "type mismatch");
945   assert(MMO.isAtomic() && "not atomic mem operand");
946 #endif
947 
948   auto MIB = buildInstr(Opcode);
949   OldValRes.addDefToMIB(*getMRI(), MIB);
950   Addr.addSrcToMIB(MIB);
951   Val.addSrcToMIB(MIB);
952   MIB.addMemOperand(&MMO);
953   return MIB;
954 }
955 
956 MachineInstrBuilder
957 MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
958                                      Register Val, MachineMemOperand &MMO) {
959   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
960                         MMO);
961 }
962 MachineInstrBuilder
963 MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
964                                     Register Val, MachineMemOperand &MMO) {
965   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
966                         MMO);
967 }
968 MachineInstrBuilder
969 MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
970                                     Register Val, MachineMemOperand &MMO) {
971   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
972                         MMO);
973 }
974 MachineInstrBuilder
975 MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
976                                     Register Val, MachineMemOperand &MMO) {
977   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
978                         MMO);
979 }
980 MachineInstrBuilder
981 MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
982                                      Register Val, MachineMemOperand &MMO) {
983   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
984                         MMO);
985 }
986 MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
987                                                        Register Addr,
988                                                        Register Val,
989                                                        MachineMemOperand &MMO) {
990   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
991                         MMO);
992 }
993 MachineInstrBuilder
994 MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
995                                     Register Val, MachineMemOperand &MMO) {
996   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
997                         MMO);
998 }
999 MachineInstrBuilder
1000 MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
1001                                     Register Val, MachineMemOperand &MMO) {
1002   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
1003                         MMO);
1004 }
1005 MachineInstrBuilder
1006 MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
1007                                     Register Val, MachineMemOperand &MMO) {
1008   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
1009                         MMO);
1010 }
1011 MachineInstrBuilder
1012 MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
1013                                      Register Val, MachineMemOperand &MMO) {
1014   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
1015                         MMO);
1016 }
1017 MachineInstrBuilder
1018 MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
1019                                      Register Val, MachineMemOperand &MMO) {
1020   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
1021                         MMO);
1022 }
1023 
1024 MachineInstrBuilder
1025 MachineIRBuilder::buildAtomicRMWFAdd(
1026   const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1027   MachineMemOperand &MMO) {
1028   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
1029                         MMO);
1030 }
1031 
1032 MachineInstrBuilder
1033 MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1034                                      MachineMemOperand &MMO) {
1035   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
1036                         MMO);
1037 }
1038 
1039 MachineInstrBuilder
1040 MachineIRBuilder::buildAtomicRMWFMax(const DstOp &OldValRes, const SrcOp &Addr,
1041                                      const SrcOp &Val, MachineMemOperand &MMO) {
1042   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FMAX, OldValRes, Addr, Val,
1043                         MMO);
1044 }
1045 
1046 MachineInstrBuilder
1047 MachineIRBuilder::buildAtomicRMWFMin(const DstOp &OldValRes, const SrcOp &Addr,
1048                                      const SrcOp &Val, MachineMemOperand &MMO) {
1049   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FMIN, OldValRes, Addr, Val,
1050                         MMO);
1051 }
1052 
1053 MachineInstrBuilder
1054 MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
1055   return buildInstr(TargetOpcode::G_FENCE)
1056     .addImm(Ordering)
1057     .addImm(Scope);
1058 }
1059 
1060 MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr,
1061                                                     unsigned RW,
1062                                                     unsigned Locality,
1063                                                     unsigned CacheType,
1064                                                     MachineMemOperand &MMO) {
1065   auto MIB = buildInstr(TargetOpcode::G_PREFETCH);
1066   Addr.addSrcToMIB(MIB);
1067   MIB.addImm(RW).addImm(Locality).addImm(CacheType);
1068   MIB.addMemOperand(&MMO);
1069   return MIB;
1070 }
1071 
1072 MachineInstrBuilder
1073 MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
1074 #ifndef NDEBUG
1075   assert(getMRI()->getType(Res).isPointer() && "invalid res type");
1076 #endif
1077 
1078   return buildInstr(TargetOpcode::G_BLOCK_ADDR).addDef(Res).addBlockAddress(BA);
1079 }
1080 
1081 void MachineIRBuilder::validateTruncExt(const LLT DstTy, const LLT SrcTy,
1082                                         bool IsExtend) {
1083 #ifndef NDEBUG
1084   if (DstTy.isVector()) {
1085     assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
1086     assert(SrcTy.getElementCount() == DstTy.getElementCount() &&
1087            "different number of elements in a trunc/ext");
1088   } else
1089     assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
1090 
1091   if (IsExtend)
1092     assert(TypeSize::isKnownGT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1093            "invalid narrowing extend");
1094   else
1095     assert(TypeSize::isKnownLT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1096            "invalid widening trunc");
1097 #endif
1098 }
1099 
1100 void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
1101                                         const LLT Op0Ty, const LLT Op1Ty) {
1102 #ifndef NDEBUG
1103   assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
1104          "invalid operand type");
1105   assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
1106   if (ResTy.isScalar() || ResTy.isPointer())
1107     assert(TstTy.isScalar() && "type mismatch");
1108   else
1109     assert((TstTy.isScalar() ||
1110             (TstTy.isVector() &&
1111              TstTy.getNumElements() == Op0Ty.getNumElements())) &&
1112            "type mismatch");
1113 #endif
1114 }
1115 
1116 MachineInstrBuilder
1117 MachineIRBuilder::buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
1118                              ArrayRef<SrcOp> SrcOps,
1119                              std::optional<unsigned> Flags) {
1120   switch (Opc) {
1121   default:
1122     break;
1123   case TargetOpcode::G_SELECT: {
1124     assert(DstOps.size() == 1 && "Invalid select");
1125     assert(SrcOps.size() == 3 && "Invalid select");
1126     validateSelectOp(
1127         DstOps[0].getLLTTy(*getMRI()), SrcOps[0].getLLTTy(*getMRI()),
1128         SrcOps[1].getLLTTy(*getMRI()), SrcOps[2].getLLTTy(*getMRI()));
1129     break;
1130   }
1131   case TargetOpcode::G_FNEG:
1132   case TargetOpcode::G_ABS:
1133     // All these are unary ops.
1134     assert(DstOps.size() == 1 && "Invalid Dst");
1135     assert(SrcOps.size() == 1 && "Invalid Srcs");
1136     validateUnaryOp(DstOps[0].getLLTTy(*getMRI()),
1137                     SrcOps[0].getLLTTy(*getMRI()));
1138     break;
1139   case TargetOpcode::G_ADD:
1140   case TargetOpcode::G_AND:
1141   case TargetOpcode::G_MUL:
1142   case TargetOpcode::G_OR:
1143   case TargetOpcode::G_SUB:
1144   case TargetOpcode::G_XOR:
1145   case TargetOpcode::G_UDIV:
1146   case TargetOpcode::G_SDIV:
1147   case TargetOpcode::G_UREM:
1148   case TargetOpcode::G_SREM:
1149   case TargetOpcode::G_SMIN:
1150   case TargetOpcode::G_SMAX:
1151   case TargetOpcode::G_UMIN:
1152   case TargetOpcode::G_UMAX:
1153   case TargetOpcode::G_UADDSAT:
1154   case TargetOpcode::G_SADDSAT:
1155   case TargetOpcode::G_USUBSAT:
1156   case TargetOpcode::G_SSUBSAT: {
1157     // All these are binary ops.
1158     assert(DstOps.size() == 1 && "Invalid Dst");
1159     assert(SrcOps.size() == 2 && "Invalid Srcs");
1160     validateBinaryOp(DstOps[0].getLLTTy(*getMRI()),
1161                      SrcOps[0].getLLTTy(*getMRI()),
1162                      SrcOps[1].getLLTTy(*getMRI()));
1163     break;
1164   }
1165   case TargetOpcode::G_SHL:
1166   case TargetOpcode::G_ASHR:
1167   case TargetOpcode::G_LSHR:
1168   case TargetOpcode::G_USHLSAT:
1169   case TargetOpcode::G_SSHLSAT: {
1170     assert(DstOps.size() == 1 && "Invalid Dst");
1171     assert(SrcOps.size() == 2 && "Invalid Srcs");
1172     validateShiftOp(DstOps[0].getLLTTy(*getMRI()),
1173                     SrcOps[0].getLLTTy(*getMRI()),
1174                     SrcOps[1].getLLTTy(*getMRI()));
1175     break;
1176   }
1177   case TargetOpcode::G_SEXT:
1178   case TargetOpcode::G_ZEXT:
1179   case TargetOpcode::G_ANYEXT:
1180     assert(DstOps.size() == 1 && "Invalid Dst");
1181     assert(SrcOps.size() == 1 && "Invalid Srcs");
1182     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1183                      SrcOps[0].getLLTTy(*getMRI()), true);
1184     break;
1185   case TargetOpcode::G_TRUNC:
1186   case TargetOpcode::G_FPTRUNC: {
1187     assert(DstOps.size() == 1 && "Invalid Dst");
1188     assert(SrcOps.size() == 1 && "Invalid Srcs");
1189     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1190                      SrcOps[0].getLLTTy(*getMRI()), false);
1191     break;
1192   }
1193   case TargetOpcode::G_BITCAST: {
1194     assert(DstOps.size() == 1 && "Invalid Dst");
1195     assert(SrcOps.size() == 1 && "Invalid Srcs");
1196     assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1197            SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1198     break;
1199   }
1200   case TargetOpcode::COPY:
1201     assert(DstOps.size() == 1 && "Invalid Dst");
1202     // If the caller wants to add a subreg source it has to be done separately
1203     // so we may not have any SrcOps at this point yet.
1204     break;
1205   case TargetOpcode::G_FCMP:
1206   case TargetOpcode::G_ICMP: {
1207     assert(DstOps.size() == 1 && "Invalid Dst Operands");
1208     assert(SrcOps.size() == 3 && "Invalid Src Operands");
1209     // For F/ICMP, the first src operand is the predicate, followed by
1210     // the two comparands.
1211     assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1212            "Expecting predicate");
1213     assert([&]() -> bool {
1214       CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1215       return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1216                                          : CmpInst::isFPPredicate(Pred);
1217     }() && "Invalid predicate");
1218     assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1219            "Type mismatch");
1220     assert([&]() -> bool {
1221       LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1222       LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1223       if (Op0Ty.isScalar() || Op0Ty.isPointer())
1224         return DstTy.isScalar();
1225       else
1226         return DstTy.isVector() &&
1227                DstTy.getNumElements() == Op0Ty.getNumElements();
1228     }() && "Type Mismatch");
1229     break;
1230   }
1231   case TargetOpcode::G_UNMERGE_VALUES: {
1232     assert(!DstOps.empty() && "Invalid trivial sequence");
1233     assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1234     assert(llvm::all_of(DstOps,
1235                         [&, this](const DstOp &Op) {
1236                           return Op.getLLTTy(*getMRI()) ==
1237                                  DstOps[0].getLLTTy(*getMRI());
1238                         }) &&
1239            "type mismatch in output list");
1240     assert((TypeSize::ScalarTy)DstOps.size() *
1241                    DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1242                SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1243            "input operands do not cover output register");
1244     break;
1245   }
1246   case TargetOpcode::G_MERGE_VALUES: {
1247     assert(SrcOps.size() >= 2 && "invalid trivial sequence");
1248     assert(DstOps.size() == 1 && "Invalid Dst");
1249     assert(llvm::all_of(SrcOps,
1250                         [&, this](const SrcOp &Op) {
1251                           return Op.getLLTTy(*getMRI()) ==
1252                                  SrcOps[0].getLLTTy(*getMRI());
1253                         }) &&
1254            "type mismatch in input list");
1255     assert((TypeSize::ScalarTy)SrcOps.size() *
1256                    SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1257                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1258            "input operands do not cover output register");
1259     assert(!DstOps[0].getLLTTy(*getMRI()).isVector() &&
1260            "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");
1261     break;
1262   }
1263   case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1264     assert(DstOps.size() == 1 && "Invalid Dst size");
1265     assert(SrcOps.size() == 2 && "Invalid Src size");
1266     assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1267     assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1268             DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1269            "Invalid operand type");
1270     assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1271     assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1272                DstOps[0].getLLTTy(*getMRI()) &&
1273            "Type mismatch");
1274     break;
1275   }
1276   case TargetOpcode::G_INSERT_VECTOR_ELT: {
1277     assert(DstOps.size() == 1 && "Invalid dst size");
1278     assert(SrcOps.size() == 3 && "Invalid src size");
1279     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1280            SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1281     assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1282                SrcOps[1].getLLTTy(*getMRI()) &&
1283            "Type mismatch");
1284     assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1285     assert(DstOps[0].getLLTTy(*getMRI()).getNumElements() ==
1286                SrcOps[0].getLLTTy(*getMRI()).getNumElements() &&
1287            "Type mismatch");
1288     break;
1289   }
1290   case TargetOpcode::G_BUILD_VECTOR: {
1291     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1292            "Must have at least 2 operands");
1293     assert(DstOps.size() == 1 && "Invalid DstOps");
1294     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1295            "Res type must be a vector");
1296     assert(llvm::all_of(SrcOps,
1297                         [&, this](const SrcOp &Op) {
1298                           return Op.getLLTTy(*getMRI()) ==
1299                                  SrcOps[0].getLLTTy(*getMRI());
1300                         }) &&
1301            "type mismatch in input list");
1302     assert((TypeSize::ScalarTy)SrcOps.size() *
1303                    SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1304                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1305            "input scalars do not exactly cover the output vector register");
1306     break;
1307   }
1308   case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1309     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1310            "Must have at least 2 operands");
1311     assert(DstOps.size() == 1 && "Invalid DstOps");
1312     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1313            "Res type must be a vector");
1314     assert(llvm::all_of(SrcOps,
1315                         [&, this](const SrcOp &Op) {
1316                           return Op.getLLTTy(*getMRI()) ==
1317                                  SrcOps[0].getLLTTy(*getMRI());
1318                         }) &&
1319            "type mismatch in input list");
1320     break;
1321   }
1322   case TargetOpcode::G_CONCAT_VECTORS: {
1323     assert(DstOps.size() == 1 && "Invalid DstOps");
1324     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1325            "Must have at least 2 operands");
1326     assert(llvm::all_of(SrcOps,
1327                         [&, this](const SrcOp &Op) {
1328                           return (Op.getLLTTy(*getMRI()).isVector() &&
1329                                   Op.getLLTTy(*getMRI()) ==
1330                                       SrcOps[0].getLLTTy(*getMRI()));
1331                         }) &&
1332            "type mismatch in input list");
1333     assert((TypeSize::ScalarTy)SrcOps.size() *
1334                    SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1335                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1336            "input vectors do not exactly cover the output vector register");
1337     break;
1338   }
1339   case TargetOpcode::G_UADDE: {
1340     assert(DstOps.size() == 2 && "Invalid no of dst operands");
1341     assert(SrcOps.size() == 3 && "Invalid no of src operands");
1342     assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1343     assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1344            (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1345            "Invalid operand");
1346     assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1347     assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1348            "type mismatch");
1349     break;
1350   }
1351   }
1352 
1353   auto MIB = buildInstr(Opc);
1354   for (const DstOp &Op : DstOps)
1355     Op.addDefToMIB(*getMRI(), MIB);
1356   for (const SrcOp &Op : SrcOps)
1357     Op.addSrcToMIB(MIB);
1358   if (Flags)
1359     MIB->setFlags(*Flags);
1360   return MIB;
1361 }
1362