1 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
35 #define DEBUG_TYPE "systemz-lower"
47   // Chain if this is a strict floating-point comparison.
127   setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());  in SystemZTargetLowering()
129   // TODO: It may be better to default to latency-oriented scheduling, however  in SystemZTargetLowering()
130   // LLVM's current latency-oriented scheduler can't handle physreg definitions  in SystemZTargetLowering()
131   // such as SystemZ has with CC, so set this to the register-pressure  in SystemZTargetLowering()
140   // Instructions are strings of 2-byte aligned 2-byte values.  in SystemZTargetLowering()
142   // For performance reasons we prefer 16-byte alignment.  in SystemZTargetLowering()
151       // Lower SET_CC into an IPM-based sequence.  in SystemZTargetLowering()
279   // Type legalization will convert 8- and 16-bit atomic operations into  in SystemZTargetLowering()
310   // We can use the CC result of compare-and-swap to implement  in SystemZTargetLowering()
322   // Handle unsigned 32-bit types as signed 64-bit types.  in SystemZTargetLowering()
330   // We have native support for a 64-bit CTLZ, via FLOGR.  in SystemZTargetLowering()
335   // On z15 we have native support for a 64-bit CTPOP.  in SystemZTargetLowering()
341   // Give LowerOperation the chance to replace 64-bit ORs with subregs.  in SystemZTargetLowering()
376   // 160-byte area at the bottom of the stack.  in SystemZTargetLowering()
508   // Handle floating-point types.  in SystemZTargetLowering()
536       // Handle constrained floating-point operations.  in SystemZTargetLowering()
556   // Handle floating-point vector types.  in SystemZTargetLowering()
558     // Scalar-to-vector conversion is just a subreg.  in SystemZTargetLowering()
585     // Handle constrained floating-point operations.  in SystemZTargetLowering()
651     // Handle constrained floating-point operations.  in SystemZTargetLowering()
673   // We only have fused f128 multiply-addition on vector registers.  in SystemZTargetLowering()
684   // a load-and-extend of a f80 constant (in cases where the constant  in SystemZTargetLowering()
695   // Floating-point truncation and stores need to be done separately.  in SystemZTargetLowering()
700   // We have 64-bit FPR<->GPR moves, but need special handling for  in SystemZTargetLowering()
701   // 32-bit forms.  in SystemZTargetLowering()
707   // VASTART and VACOPY need to deal with the SystemZ-specific varargs  in SystemZTargetLowering()
708   // structure, but VAEND is a no-op.  in SystemZTargetLowering()
715   // Codes for which we want to perform some z-specific combinations.  in SystemZTargetLowering()
747   // generated by target-independent code don't when the byte value is  in SystemZTargetLowering()
749   // than "STC;MVC".  Handle the choice in target-specific code instead.  in SystemZTargetLowering()
753   // Default to having -disable-strictnode-mutation on  in SystemZTargetLowering()
810   // Try using VECTOR GENERATE BYTE MASK.  This is the architecturally-  in isVectorConstantLegal()
811   // preferred way of creating all-zero and all-one vectors so give it  in isVectorConstantLegal()
832   auto tryValue = [&](uint64_t Value) -> bool {  in isVectorConstantLegal()
844     if (TII->isRxSBGMask(Value, SplatBitSize, Start, End)) {  in isVectorConstantLegal()
845       // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0  in isVectorConstantLegal()
847       // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).  in isVectorConstantLegal()
848       OpVals.push_back(Start - (64 - SplatBitSize));  in isVectorConstantLegal()
849       OpVals.push_back(End - (64 - SplatBitSize));  in isVectorConstantLegal()
860   // being able to use a sign-extended element value in VECTOR REPLICATE  in isVectorConstantLegal()
873   // using a non-wraparound mask.  in isVectorConstantLegal()
881     IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());  in SystemZVectorConstantInfo()
906   assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");  in SystemZVectorConstantInfo()
910   BVN->isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,  in SystemZVectorConstantInfo()
914   BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,  in SystemZVectorConstantInfo()
930   if (MF.getFunction().hasFnAttribute("probe-stack"))  in hasInlineStackProbe()
931     return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==  in hasInlineStackProbe()
932            "inline-asm";  in hasInlineStackProbe()
949   if (RMW->getType()->isIntegerTy(8) || RMW->getType()->isIntegerTy(16))  in shouldExpandAtomicRMWInIR()
954       (RMW->getType()->isIntegerTy(32) || RMW->getType()->isIntegerTy(64)) &&  in shouldExpandAtomicRMWInIR()
955       (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||  in shouldExpandAtomicRMWInIR()
956        RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||  in shouldExpandAtomicRMWInIR()
957        RMW->getOperation() == AtomicRMWInst::BinOp::And ||  in shouldExpandAtomicRMWInIR()
958        RMW->getOperation() == AtomicRMWInst::BinOp::Or ||  in shouldExpandAtomicRMWInIR()
959        RMW->getOperation() == AtomicRMWInst::BinOp::Xor))  in shouldExpandAtomicRMWInIR()
972   return isUInt<32>(Imm) || isUInt<32>(-Imm);  in isLegalAddImmediate()
1001   // With vector support a Load->Store combination may be combined to either  in getLoadStoreAddrMode()
1008   bool MVC = Ty->isIntegerTy(8);  in getLoadStoreAddrMode()
1017     switch (II->getIntrinsicID()) {  in supportedAddressingMode()
1026   if (isa<LoadInst>(I) && I->hasOneUse()) {  in supportedAddressingMode()
1027     auto *SingleUser = cast<Instruction>(*I->user_begin());  in supportedAddressingMode()
1028     if (SingleUser->getParent() == I->getParent()) {  in supportedAddressingMode()
1030         if (auto *C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))  in supportedAddressingMode()
1031           if (C->getBitWidth() <= 64 &&  in supportedAddressingMode()
1032               (isInt<16>(C->getSExtValue()) || isUInt<16>(C->getZExtValue())))  in supportedAddressingMode()
1036         // Load->Store  in supportedAddressingMode()
1037         return getLoadStoreAddrMode(HasVector, I->getType());  in supportedAddressingMode()
1040     if (auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))  in supportedAddressingMode()
1041       if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())  in supportedAddressingMode()
1042         // Load->Store  in supportedAddressingMode()
1043         return getLoadStoreAddrMode(HasVector, LoadI->getType());  in supportedAddressingMode()
1054     Type *MemAccessTy = (isa<LoadInst>(I) ? I->getType() :  in supportedAddressingMode()
1055                          I->getOperand(0)->getType());  in supportedAddressingMode()
1056     bool IsFPAccess = MemAccessTy->isFloatingPointTy();  in supportedAddressingMode()
1057     bool IsVectorAccess = MemAccessTy->isVectorTy();  in supportedAddressingMode()
1062       Value *DataOp = I->getOperand(0);  in supportedAddressingMode()
1069     if (!IsVectorAccess && isa<LoadInst>(I) && I->hasOneUse()) {  in supportedAddressingMode()
1070       User *LoadUser = *I->user_begin();  in supportedAddressingMode()
1089   // Require a 20-bit signed offset.  in isLegalAddressingMode()
1094       Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(128));  in isLegalAddressingMode()
1119     if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)  in findOptimalMemOpLowering()
1135   if (!FromType->isIntegerTy() || !ToType->isIntegerTy())  in isTruncateFree()
1137   unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();  in isTruncateFree()
1138   unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();  in isTruncateFree()
1150 //===----------------------------------------------------------------------===//
1152 //===----------------------------------------------------------------------===//
1160     case 'f': // Floating-point register  in getConstraintType()
1161     case 'h': // High-part register  in getConstraintType()
1162     case 'r': // General-purpose register  in getConstraintType()
1166     case 'Q': // Memory with base and unsigned 12-bit displacement  in getConstraintType()
1168     case 'S': // Memory with base and signed 20-bit displacement  in getConstraintType()
1173     case 'I': // Unsigned 8-bit constant  in getConstraintType()
1174     case 'J': // Unsigned 12-bit constant  in getConstraintType()
1175     case 'K': // Signed 16-bit constant  in getConstraintType()
1176     case 'L': // Signed 20-bit displacement (on all targets we support)  in getConstraintType()
1185     case 'Q': // Address with base and unsigned 12-bit displacement  in getConstraintType()
1187     case 'S': // Address with base and signed 20-bit displacement  in getConstraintType()
1207   Type *type = CallOperandVal->getType();  in getSingleConstraintMatchWeight()
1216   case 'h': // High-part register  in getSingleConstraintMatchWeight()
1217   case 'r': // General-purpose register  in getSingleConstraintMatchWeight()
1218     weight = CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;  in getSingleConstraintMatchWeight()
1221   case 'f': // Floating-point register  in getSingleConstraintMatchWeight()
1223       weight = type->isFloatingPointTy() ? CW_Register : CW_Default;  in getSingleConstraintMatchWeight()
1228       weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register  in getSingleConstraintMatchWeight()
1232   case 'I': // Unsigned 8-bit constant  in getSingleConstraintMatchWeight()
1234       if (isUInt<8>(C->getZExtValue()))  in getSingleConstraintMatchWeight()
1238   case 'J': // Unsigned 12-bit constant  in getSingleConstraintMatchWeight()
1240       if (isUInt<12>(C->getZExtValue()))  in getSingleConstraintMatchWeight()
1244   case 'K': // Signed 16-bit constant  in getSingleConstraintMatchWeight()
1246       if (isInt<16>(C->getSExtValue()))  in getSingleConstraintMatchWeight()
1250   case 'L': // Signed 20-bit displacement (on all targets we support)  in getSingleConstraintMatchWeight()
1252       if (isInt<20>(C->getSExtValue()))  in getSingleConstraintMatchWeight()
1258       if (C->getZExtValue() == 0x7fffffff)  in getSingleConstraintMatchWeight()
1267 // Map maps 0-based register numbers to LLVM register numbers.
1271   assert(*(Constraint.end()-1) == '}' && "Missing '}'");  in parseRegisterNumber()
1275         Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index);  in parseRegisterNumber()
1290     case 'r': // General-purpose register  in getRegForInlineAsmConstraint()
1304     case 'h': // High-part register (an LLVM extension)  in getRegForInlineAsmConstraint()
1307     case 'f': // Floating-point register  in getRegForInlineAsmConstraint()
1412     case 'I': // Unsigned 8-bit constant  in LowerAsmOperandForConstraint()
1414         if (isUInt<8>(C->getZExtValue()))  in LowerAsmOperandForConstraint()
1415           Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),  in LowerAsmOperandForConstraint()
1419     case 'J': // Unsigned 12-bit constant  in LowerAsmOperandForConstraint()
1421         if (isUInt<12>(C->getZExtValue()))  in LowerAsmOperandForConstraint()
1422           Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),  in LowerAsmOperandForConstraint()
1426     case 'K': // Signed 16-bit constant  in LowerAsmOperandForConstraint()
1428         if (isInt<16>(C->getSExtValue()))  in LowerAsmOperandForConstraint()
1429           Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),  in LowerAsmOperandForConstraint()
1433     case 'L': // Signed 20-bit displacement (on all targets we support)  in LowerAsmOperandForConstraint()
1435         if (isInt<20>(C->getSExtValue()))  in LowerAsmOperandForConstraint()
1436           Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),  in LowerAsmOperandForConstraint()
1442         if (C->getZExtValue() == 0x7fffffff)  in LowerAsmOperandForConstraint()
1443           Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),  in LowerAsmOperandForConstraint()
1451 //===----------------------------------------------------------------------===//
1453 //===----------------------------------------------------------------------===//
1470   return CI->isTailCall();  in mayBeEmittedAsTailCall()
1618   FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());  in LowerFormalArguments()
1670       // FIXME: Pre-include call frame size in the offset, should not  in LowerFormalArguments()
1683       // passed as right-justified 8-byte values.  in LowerFormalArguments()
1715     // Save the number of non-varargs registers for later use by va_start, etc.  in LowerFormalArguments()
1716     FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);  in LowerFormalArguments()
1717     FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);  in LowerFormalArguments()
1723     // first stack vararg would be.  The 1-byte size here is arbitrary.  in LowerFormalArguments()
1724     // FIXME: Pre-include call frame size in the offset, should not  in LowerFormalArguments()
1726     int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();  in LowerFormalArguments()
1728     FuncInfo->setVarArgsFrameIndex(FI);  in LowerFormalArguments()
1732     // Save the number of non-varargs registers for later use by va_start, etc.  in LowerFormalArguments()
1733     FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);  in LowerFormalArguments()
1734     FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);  in LowerFormalArguments()
1737     // first stack vararg would be.  The 1-byte size here is arbitrary.  in LowerFormalArguments()
1739     FuncInfo->setVarArgsFrameIndex(  in LowerFormalArguments()
1742     // ...and a similar frame index for the caller-allocated save area  in LowerFormalArguments()
1745       -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, SystemZ::R2D) - 16;  in LowerFormalArguments()
1747     FuncInfo->setRegSaveFrameIndex(RegSaveIndex);  in LowerFormalArguments()
1754         unsigned Offset = TFL->getRegSpillOffset(MF, SystemZ::ELFArgFPRs[I]);  in LowerFormalArguments()
1756           MFI.CreateFixedObject(8, -SystemZMC::ELFCallFrameSize + Offset, true);  in LowerFormalArguments()
1767                                    SystemZ::ELFNumArgFPRs - NumFixedFPRs));  in LowerFormalArguments()
1777     MRI.addLiveIn(Regs->getADARegister(), ADAvReg);  in LowerFormalArguments()
1778     FuncInfo->setADAVirtualRegister(ADAvReg);  in LowerFormalArguments()
1787   // needs the callee-saved argument register R6, or if the call uses  in canUseSiblingCall()
1788   // the callee-saved register arguments SwiftSelf and SwiftError.  in canUseSiblingCall()
1808   unsigned ADAvReg = MFI->getADAVirtualRegister();  in getADAEntry()
1831       (isa<Function>(GV)) || (GA && isa<Function>(GA->getAliaseeObject()));  in getADAEntry()
1832   bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());  in getADAEntry()
1857     bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||  in getzOSCalleeAndADA()
1858                        G->getGlobal()->hasPrivateLinkage());  in getzOSCalleeAndADA()
1862       unsigned ADAvReg = MFI->getADAVirtualRegister();  in getzOSCalleeAndADA()
1864       Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);  in getzOSCalleeAndADA()
1869           G->getGlobal(), DL, PtrVT, 0, SystemZII::MO_ADA_DIRECT_FUNC_DESC);  in getzOSCalleeAndADA()
1875         E->getSymbol(), PtrVT, SystemZII::MO_ADA_DIRECT_FUNC_DESC);  in getzOSCalleeAndADA()
1919   // We don't support GuaranteedTailCallOpt, only automatically-detected  in LowerCall()
1954       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();  in LowerCall()
1978       // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a  in LowerCall()
1979       // MVT::i128 type. We decompose the 128-bit type to a pair of its high  in LowerCall()
1989       // floats are passed as right-justified 8-byte values.  in LowerCall()
1992                                       Regs->getStackPointerRegister(), PtrVT);  in LowerCall()
1993       unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +  in LowerCall()
2005       // they are vararg (non-fixed arguments), if a long double or vector  in LowerCall()
2031                                ->getAddressOfCalleeRegister();  in LowerCall()
2037         static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), ADA));  in LowerCall()
2040       Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);  in LowerCall()
2043       Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT);  in LowerCall()
2052   // Build a sequence of copy-to-reg nodes, chained and glued together.  in LowerCall()
2070   // Add a register mask operand representing the call-preserved registers.  in LowerCall()
2072   const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);  in LowerCall()
2408   Ops.reserve(NumOps - 1);  in emitIntrinsicWithCCAndChain()
2413   assert(Op->getNumValues() == 2 && "Expected only CC result and chain");  in emitIntrinsicWithCCAndChain()
2428   Ops.reserve(NumOps - 1);  in emitIntrinsicWithCC()
2432   SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops);  in emitIntrinsicWithCC()
2437 // floating-point comparison.  Return the condition code mask for
2439 // unsigned comparisons and clear for signed ones.  In the floating-point
2442 #define CONV(X) \  in CCMaskForCondCode()  macro
2451   CONV(EQ);  in CCMaskForCondCode()
2452   CONV(NE);  in CCMaskForCondCode()
2453   CONV(GT);  in CCMaskForCondCode()
2454   CONV(GE);  in CCMaskForCondCode()
2455   CONV(LT);  in CCMaskForCondCode()
2456   CONV(LE);  in CCMaskForCondCode()
2461 #undef CONV  in CCMaskForCondCode()
2471   if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)  in adjustZeroCmp()
2474   int64_t Value = ConstOp1->getSExtValue();  in adjustZeroCmp()
2475   if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||  in adjustZeroCmp()
2476       (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||  in adjustZeroCmp()
2488   // For us to make any changes, it must a comparison between a single-use  in adjustSubwordCmp()
2495   // We must have an 8- or 16-bit load.  in adjustSubwordCmp()
2497   unsigned NumBits = Load->getMemoryVT().getSizeInBits();  in adjustSubwordCmp()
2499       NumBits != Load->getMemoryVT().getStoreSizeInBits())  in adjustSubwordCmp()
2505   if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)  in adjustSubwordCmp()
2507   uint64_t Value = ConstOp1->getZExtValue();  in adjustSubwordCmp()
2508   uint64_t Mask = (1 << NumBits) - 1;  in adjustSubwordCmp()
2509   if (Load->getExtensionType() == ISD::SEXTLOAD) {  in adjustSubwordCmp()
2511     int64_t SignedValue = ConstOp1->getSExtValue();  in adjustSubwordCmp()
2512     if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)  in adjustSubwordCmp()
2515       // Unsigned comparison between two sign-extended values is equivalent  in adjustSubwordCmp()
2516       // to unsigned comparison between two zero-extended values.  in adjustSubwordCmp()
2532   } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {  in adjustSubwordCmp()
2545       Load->getExtensionType() != ExtType) {  in adjustSubwordCmp()
2546     C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(),  in adjustSubwordCmp()
2547                            Load->getBasePtr(), Load->getPointerInfo(),  in adjustSubwordCmp()
2548                            Load->getMemoryVT(), Load->getAlign(),  in adjustSubwordCmp()
2549                            Load->getMemOperand()->getFlags());  in adjustSubwordCmp()
2556       Value != ConstOp1->getZExtValue())  in adjustSubwordCmp()
2561 // for integer register-memory comparisons of type ICmpType.
2566     if (Load->getMemoryVT() == MVT::i8)  in isNaturalMemoryOperand()
2569     switch (Load->getExtensionType()) {  in isNaturalMemoryOperand()
2591   // Always keep a floating-point constant second, since comparisons with  in shouldSwapCmpOperands()
2600   if (ConstOp1 && ConstOp1->getZExtValue() == 0)  in shouldSwapCmpOperands()
2608   // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.  in shouldSwapCmpOperands()
2615     // The unsigned memory-immediate instructions can handle 16-bit  in shouldSwapCmpOperands()
2618         isUInt<16>(ConstOp1->getZExtValue()))  in shouldSwapCmpOperands()
2620     // The signed memory-immediate instructions can handle 16-bit  in shouldSwapCmpOperands()
2623         isInt<16>(ConstOp1->getSExtValue()))  in shouldSwapCmpOperands()
2642 // Check whether C tests for equality between X and Y and whether X - Y
2643 // or Y - X is also computed.  In that case it's better to compare the
2649     for (SDNode *N : C.Op0->uses()) {  in adjustForSubtraction()
2650       if (N->getOpcode() == ISD::SUB &&  in adjustForSubtraction()
2651           ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) ||  in adjustForSubtraction()
2652            (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {  in adjustForSubtraction()
2655         SDNodeFlags Flags = N->getFlags();  in adjustForSubtraction()
2658         N->setFlags(Flags);  in adjustForSubtraction()
2660         C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));  in adjustForSubtraction()
2667 // Check whether C compares a floating-point value with zero and if that
2668 // floating-point value is also negated.  In this case we can use the
2677   if (C1 && C1->isZero()) {  in adjustForFNeg()
2678     for (SDNode *N : C.Op0->uses()) {  in adjustForFNeg()
2679       if (N->getOpcode() == ISD::FNEG) {  in adjustForFNeg()
2689 // also sign-extended.  In that case it is better to test the result
2697       C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {  in adjustForLTGFR()
2699     if (C1 && C1->getZExtValue() == 32) {  in adjustForLTGFR()
2702       for (SDNode *N : ShlOp0->uses()) {  in adjustForLTGFR()
2703         if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&  in adjustForLTGFR()
2704             cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) {  in adjustForLTGFR()
2721       cast<ConstantSDNode>(C.Op1)->getValueSizeInBits(0) <= 64 &&  in adjustICmpTruncate()
2722       C.Op1->getAsZExtVal() == 0) {  in adjustICmpTruncate()
2724     if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=  in adjustICmpTruncate()
2726       unsigned Type = L->getExtensionType();  in adjustICmpTruncate()
2736 // Return true if shift operation N has an in-range constant shift value.
2743   uint64_t Amount = Shift->getZExtValue();  in isSimpleShift()
2802   if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {  in getTestUnderMaskCond()
2808   if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {  in getTestUnderMaskCond()
2816   if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {  in getTestUnderMaskCond()
2822   if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {  in getTestUnderMaskCond()
2857       if (Mask && Mask->getAPIntValue() == 0) {  in adjustForTestUnderMask()
2875   uint64_t CmpVal = ConstOp1->getZExtValue();  in adjustForTestUnderMask()
2887     MaskVal = Mask->getZExtValue();  in adjustForTestUnderMask()
2889     // There is no instruction to compare with a 64-bit immediate  in adjustForTestUnderMask()
2900       if (CmpVal == uint64_t(-1))  in adjustForTestUnderMask()
2907     MaskVal = -(CmpVal & -CmpVal);  in adjustForTestUnderMask()
2949   if (Mask && Mask->getZExtValue() == MaskVal)  in adjustForTestUnderMask()
2965   // (In-)Equality comparisons can be implemented via VCEQGS.  in adjustICmp128()
3010   if (!Mask || Mask->getValueSizeInBits(0) > 64)  in adjustForRedundantAnd()
3013   if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())  in adjustForRedundantAnd()
3019 // Return a Comparison that tests the condition-code result of intrinsic
3022 // and CCValid is the set of possible condition-code results.
3031     C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;  in getIntrinsicCmp()
3034     C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;  in getIntrinsicCmp()
3038     C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;  in getIntrinsicCmp()
3041     C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;  in getIntrinsicCmp()
3045     C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;  in getIntrinsicCmp()
3048     C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;  in getIntrinsicCmp()
3064         CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) &&  in getCmp()
3067                              CmpOp1->getAsZExtVal(), Cond);  in getCmp()
3069         CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&  in getCmp()
3072                              CmpOp1->getAsZExtVal(), Cond);  in getCmp()
3131       return SDValue(Node, Node->getNumValues() - 1);  in emitCmp()
3157 // Implement a 32-bit *MUL_LOHI operation by extending both operands to
3199 // for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3200 // floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3201 // floating-point comparisons.
3272   int Mask[] = { Start, -1, Start + 1, -1 };  in expandV4F32ToV2F64()
3322 // floating-point comparison.  If in addition IsSignaling is true, we have
3323 // a strict signaling floating-point comparison.
3392       DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64));  in lowerVectorSETCC()
3406   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();  in lowerSETCC()
3423   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get();  in lowerSTRICT_FSETCC()
3425   EVT VT = Op.getNode()->getValueType(0);  in lowerSTRICT_FSETCC()
3434   CCReg->setFlags(Op->getFlags());  in lowerSTRICT_FSETCC()
3441   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();  in lowerBR_CC()
3481   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();  in lowerSELECT_CC()
3486   // Check for absolute and negative-absolute selections, including those  in lowerSELECT_CC()
3487   // where the comparison value is sign-extended (for LPGFR and LNGFR).  in lowerSELECT_CC()
3492       cast<ConstantSDNode>(C.Op1)->getValueSizeInBits(0) <= 64 &&  in lowerSELECT_CC()
3493       C.Op1->getAsZExtVal() == 0) {  in lowerSELECT_CC()
3511   const GlobalValue *GV = Node->getGlobal();  in lowerGlobalAddress()
3512   int64_t Offset = Node->getOffset();  in lowerGlobalAddress()
3526       Offset -= Anchor;  in lowerGlobalAddress()
3549   // If there was a non-zero offset that we didn't fold, create an explicit  in lowerGlobalAddress()
3581   Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,  in lowerTLSGetOffset()
3582                                            Node->getValueType(0),  in lowerTLSGetOffset()
3590   // Add a register mask operand representing the call-preserved registers.  in lowerTLSGetOffset()
3593       TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);  in lowerTLSGetOffset()
3622   // Merge them into a single 64-bit address.  in lowerThreadPointer()
3633   const GlobalValue *GV = Node->getGlobal();  in lowerGlobalTLSAddress()
3647       // Load the GOT offset of the tls_index (module ID / per-symbol offset).  in lowerGlobalTLSAddress()
3675       // of the module base offset.  Count total number of local-dynamic  in lowerGlobalTLSAddress()
3679       MFI->incNumLocalDynamicTLSAccesses();  in lowerGlobalTLSAddress()
3681       // Add the per-symbol offset.  in lowerGlobalTLSAddress()
3724   const BlockAddress *BA = Node->getBlockAddress();  in lowerBlockAddress()
3725   int64_t Offset = Node->getOffset();  in lowerBlockAddress()
3737   SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);  in lowerJumpTable()
3749   if (CP->isMachineConstantPoolEntry())  in lowerConstantPool()
3751         DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CP->getAlign());  in lowerConstantPool()
3753     Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlign(),  in lowerConstantPool()
3754                                        CP->getOffset());  in lowerConstantPool()
3775   int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);  in lowerFRAMEADDR()
3783     SDValue Offset = DAG.getConstant(TFL->getBackchainOffset(MF), DL, PtrVT);  in lowerFRAMEADDR()
3784     while (Depth--) {  in lowerFRAMEADDR()
3814     int Offset = TFL->getReturnAddressOffset(MF);  in lowerRETURNADDR()
3822   // implicit live-in.  in lowerRETURNADDR()
3824   Register LinkReg = MF.addLiveIn(CCR->getReturnFunctionAddressRegister(),  in lowerRETURNADDR()
3841       SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),  in lowerBITCAST()
3842                                     LoadN->getBasePtr(), LoadN->getMemOperand());  in lowerBITCAST()
3899   SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);  in lowerVASTART_XPLINK()
3900   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();  in lowerVASTART_XPLINK()
3914   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();  in lowerVASTART_ELF()
3920     DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),  in lowerVASTART_ELF()
3921     DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),  in lowerVASTART_ELF()
3922     DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),  in lowerVASTART_ELF()
3923     DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)  in lowerVASTART_ELF()
3946   const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();  in lowerVACOPY()
3947   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();  in lowerVACOPY()
3972   bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");  in lowerDYNAMIC_STACKALLOC_XPLINK()
3980   uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);  in lowerDYNAMIC_STACKALLOC_XPLINK()
3982   uint64_t StackAlign = TFI->getStackAlignment();  in lowerDYNAMIC_STACKALLOC_XPLINK()
3984   uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;  in lowerDYNAMIC_STACKALLOC_XPLINK()
4023                          DAG.getConstant(~(RequiredAlign - 1), DL, PtrVT));  in lowerDYNAMIC_STACKALLOC_XPLINK()
4035   bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");  in lowerDYNAMIC_STACKALLOC_ELF()
4045   uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);  in lowerDYNAMIC_STACKALLOC_ELF()
4047   uint64_t StackAlign = TFI->getStackAlignment();  in lowerDYNAMIC_STACKALLOC_ELF()
4049   uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;  in lowerDYNAMIC_STACKALLOC_ELF()
4094                   DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));  in lowerDYNAMIC_STACKALLOC_ELF()
4118     // Just do a normal 64-bit multiplication and extract the results.  in lowerSMUL_LOHI()
4129     // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:  in lowerSMUL_LOHI()
4136     //   (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)  in lowerSMUL_LOHI()
4141     //   (ll * rl) - (((lh & rl) + (ll & rh)) << 64)  in lowerSMUL_LOHI()
4166     // Just do a normal 64-bit multiplication and extract the results.  in lowerUMUL_LOHI()
4186   // We use DSGF for 32-bit division.  This means the first operand must  in lowerSDIVREM()
4187   // always be 64-bit, and the second operand should be 32-bit whenever  in lowerSDIVREM()
4215   assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");  in lowerOR()
4217   // Get the known-zero masks for each operand.  in lowerOR()
4244     int64_t Value = int32_t(LowOp->getAsZExtVal());  in lowerOR()
4273   SDValue LHS = N->getOperand(0);  in lowerXALUO()
4274   SDValue RHS = N->getOperand(1);  in lowerXALUO()
4277   if (N->getValueType(0) == MVT::i128) {  in lowerXALUO()
4297     Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));  in lowerXALUO()
4301     return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);  in lowerXALUO()
4332   SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);  in lowerXALUO()
4336   if (N->getValueType(1) == MVT::i1)  in lowerXALUO()
4339   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);  in lowerXALUO()
4359   MVT VT = N->getSimpleValueType(0);  in lowerUADDSUBO_CARRY()
4365   SDValue LHS = N->getOperand(0);  in lowerUADDSUBO_CARRY()
4366   SDValue RHS = N->getOperand(1);  in lowerUADDSUBO_CARRY()
4394     Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));  in lowerUADDSUBO_CARRY()
4398     return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);  in lowerUADDSUBO_CARRY()
4434   if (N->getValueType(1) == MVT::i1)  in lowerUADDSUBO_CARRY()
4437   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);  in lowerUADDSUBO_CARRY()
4489   // Get the known-zero mask for the operand.  in lowerCTPOP()
4495   // Skip known-zero high parts of the operand.  in lowerCTPOP()
4505   // Add up per-byte counts in a binary tree.  All bits of Op at  in lowerCTPOP()
4511                         DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));  in lowerCTPOP()
4518                      DAG.getConstant(BitSize - 8, DL, VT));  in lowerCTPOP()
4531   // The only fence that needs an instruction is a sequentially-consistent  in lowerATOMIC_FENCE()
4532   // cross-thread fence.  in lowerATOMIC_FENCE()
4540   // MEMBARRIER is a compiler barrier; it codegens to a no-op.  in lowerATOMIC_FENCE()
4548       (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&  in lowerATOMIC_LDST_I128()
4550   // Use same code to handle both legal and non-legal i128 types.  in lowerATOMIC_LDST_I128()
4566                             DAG.getConstant(-4, DL, PtrVT));  in getCSAddressAndShifts()
4581 // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation.  Lower the first
4588   // 32-bit operations need no special handling.  in lowerATOMIC_LOAD_OP()
4589   EVT NarrowVT = Node->getMemoryVT();  in lowerATOMIC_LOAD_OP()
4595   SDValue ChainIn = Node->getChain();  in lowerATOMIC_LOAD_OP()
4596   SDValue Addr = Node->getBasePtr();  in lowerATOMIC_LOAD_OP()
4597   SDValue Src2 = Node->getVal();  in lowerATOMIC_LOAD_OP()
4598   MachineMemOperand *MMO = Node->getMemOperand();  in lowerATOMIC_LOAD_OP()
4605       Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType());  in lowerATOMIC_LOAD_OP()
4618                        DAG.getConstant(32 - BitSize, DL, WideVT));  in lowerATOMIC_LOAD_OP()
4622                        DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));  in lowerATOMIC_LOAD_OP()
4641 // Op is an ATOMIC_LOAD_SUB operation.  Lower 8- and 16-bit operations into
4642 // ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
4646   EVT MemVT = Node->getMemoryVT();  in lowerATOMIC_LOAD_SUB()
4648     // A full-width operation: negate and use LAA(G).  in lowerATOMIC_LOAD_SUB()
4652     SDValue Src2 = Node->getVal();  in lowerATOMIC_LOAD_SUB()
4657                          Node->getChain(), Node->getBasePtr(), NegSrc2,  in lowerATOMIC_LOAD_SUB()
4658                          Node->getMemOperand());  in lowerATOMIC_LOAD_SUB()
4664 // Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
4668   SDValue ChainIn = Node->getOperand(0);  in lowerATOMIC_CMP_SWAP()
4669   SDValue Addr = Node->getOperand(1);  in lowerATOMIC_CMP_SWAP()
4670   SDValue CmpVal = Node->getOperand(2);  in lowerATOMIC_CMP_SWAP()
4671   SDValue SwapVal = Node->getOperand(3);  in lowerATOMIC_CMP_SWAP()
4672   MachineMemOperand *MMO = Node->getMemOperand();  in lowerATOMIC_CMP_SWAP()
4675   if (Node->getMemoryVT() == MVT::i128) {  in lowerATOMIC_CMP_SWAP()
4676     // Use same code to handle both legal and non-legal i128 types.  in lowerATOMIC_CMP_SWAP()
4682   // We have native support for 32-bit and 64-bit compare and swap, but we  in lowerATOMIC_CMP_SWAP()
4684   EVT NarrowVT = Node->getMemoryVT();  in lowerATOMIC_CMP_SWAP()
4700   // Convert 8-bit and 16-bit compare and swap to a loop, implemented  in lowerATOMIC_CMP_SWAP()
4732     if (SI->isAtomic())  in getTargetMMOFlags()
4735     if (LI->isAtomic())  in getTargetMMOFlags()
4738     if (AI->isAtomic())  in getTargetMMOFlags()
4741     if (AI->isAtomic())  in getTargetMMOFlags()
4751     report_fatal_error("Variable-sized stack allocations are not supported "  in lowerSTACKSAVE()
4754                             Regs->getStackPointerRegister(), Op.getValueType());  in lowerSTACKSAVE()
4764     report_fatal_error("Variable-sized stack allocations are not supported "  in lowerSTACKRESTORE()
4774         Chain, DL, Regs->getStackPointerRegister(), MVT::i64);  in lowerSTACKRESTORE()
4779   Chain = DAG.getCopyToReg(Chain, DL, Regs->getStackPointerRegister(), NewSP);  in lowerSTACKRESTORE()
4802                                  Node->getVTList(), Ops,  in lowerPREFETCH()
4803                                  Node->getMemoryVT(), Node->getMemOperand());  in lowerPREFETCH()
4819     assert(Op->getNumValues() == 2 && "Expected only CC result and chain");  in lowerINTRINSIC_W_CHAIN()
4835     if (Op->getNumValues() == 1)  in lowerINTRINSIC_WO_CHAIN()
4837     assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");  in lowerINTRINSIC_WO_CHAIN()
4838     return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(),  in lowerINTRINSIC_WO_CHAIN()
4983 // operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
4984 // OpNos[1] is the same for operand 1 of the pattern.  Resolve these -1s and
5001 // Bytes is a VPERM-like permute vector, except that -1 is used for
5012   int OpNos[] = { -1, -1 };  in matchPermute()
5019       if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))  in matchPermute()
5025       if (OpNos[ModelOpNo] == 1 - RealOpNo)  in matchPermute()
5042 // Bytes is a VPERM-like permute vector, except that -1 is used for
5044 // See whether redistributing the -1 bytes gives a shuffle that can be
5045 // implemented using P.  If so, set Transform to a VPERM-like permute vector
5055       Transform[From] = -1;  in matchDoublePermute()
5077 // Convert the mask of the given shuffle op into a byte-level mask,
5086     Bytes.resize(NumElements * BytesPerElement, -1);  in getVPermMask()
5088       int Index = VSN->getMaskElt(I);  in getVPermMask()
5098     Bytes.resize(NumElements * BytesPerElement, -1);  in getVPermMask()
5107 // Bytes is a VPERM-like permute vector, except that -1 is used for
5113   Base = -1;  in getShuffleInput()
5118         Base = Elem - I;  in getShuffleInput()
5122       } else if (unsigned(Base) != Elem - I)  in getShuffleInput()
5129 // Bytes is a VPERM-like permute vector, except that -1 is used for
5137   int OpNos[] = { -1, -1 };  in isShlDoublePermute()
5138   int Shift = -1;  in isShlDoublePermute()
5142       int ExpectedShift = (Index - I) % SystemZ::VectorBytes;  in isShlDoublePermute()
5151       if (OpNos[ModelOpNo] == 1 - RealOpNo)  in isShlDoublePermute()
5189   if (N->getOpcode() == ISD::BITCAST)  in isZeroVector()
5190     N = N->getOperand(0);  in isZeroVector()
5191   if (N->getOpcode() == ISD::SPLAT_VECTOR)  in isZeroVector()
5192     if (auto *Op = dyn_cast<ConstantSDNode>(N->getOperand(0)))  in isZeroVector()
5193       return Op->getZExtValue() == 0;  in isZeroVector()
5205 // Bytes is a VPERM-like permute vector, except that -1 is used for
5226     int ZeroIdx = -1;  in getGeneralPermuteNode()
5242     if (ZeroIdx != -1) {  in getGeneralPermuteNode()
5280 // Describes a general N-operand vector shuffle.
5293   // Index I is -1 if byte I of the result is undefined.  Otherwise the
5310     Bytes.push_back(-1);  in addUndef()
5334                    (FromBytesPerElement - BytesPerElement));  in add()
5393   // Try to redistribute the undefined elements of non-root nodes so that  in getNode()
5394   // the non-root shuffles match something like a pack or merge, then adjust  in getNode()
5403     for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {  in getNode()
5416           NewBytes[J] = -1;  in getNode()
5446         Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;  in getNode()
5485       Log2_32_Ceil(Ops.size()) == Log2_32_Ceil(Ops.size() - 1))  in tryPrepareForUnpack()
5498       if (Bytes[Elt] != -1) {  in tryPrepareForUnpack()
5510           if (SrcBytes[i] != -1 && SrcBytes[i] % 16 != int(i)) {  in tryPrepareForUnpack()
5534     Bytes[B++] = -1;  in tryPrepareForUnpack()
5542         Bytes[I] -= SystemZ::VectorBytes;  in tryPrepareForUnpack()
5564 // Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
5625 // the non-EXTRACT_VECTOR_ELT elements.  See if the given BUILD_VECTOR
5629   EVT VT = BVN->getValueType(0);  in tryBuildVectorShuffle()
5632   // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation  in tryBuildVectorShuffle()
5633   // on byte vectors.  If there are non-EXTRACT_VECTOR_ELT elements that still  in tryBuildVectorShuffle()
5640     SDValue Op = BVN->getOperand(I);  in tryBuildVectorShuffle()
5654       ResidueOps.push_back(BVN->getOperand(I));  in tryBuildVectorShuffle()
5677   if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Op)->isUnindexed())  in isVectorElementLoad()
5680     if (AL->getOpcode() == ISD::ATOMIC_LOAD)  in isVectorElementLoad()
5708   // - if the only defined element is a loaded one, the best sequence  in buildVector()
5711   // - otherwise, if the only defined element is an i64 value, we will  in buildVector()
5712   //   end up with the same VLVGP sequence regardless of whether we short-cut  in buildVector()
5715   // - otherwise, if the only defined element is an i32 or smaller value,  in buildVector()
5734   // Use a 64-bit merge high to combine two doubles.  in buildVector()
5753     // Merging identical replications is a no-op.  in buildVector()
5808       unsigned I1 = NumElements / 2 - 1;  in buildVector()
5809       unsigned I2 = NumElements - 1;  in buildVector()
5838   if (BVN->isConstant()) {  in lowerBUILD_VECTOR()
5869   if (VSN->isSplat()) {  in lowerVECTOR_SHUFFLE()
5871     unsigned Index = VSN->getSplatIndex();  in lowerVECTOR_SHUFFLE()
5878     // Otherwise keep it as a vector-to-vector operation.  in lowerVECTOR_SHUFFLE()
5885     int Elt = VSN->getMaskElt(I);  in lowerVECTOR_SHUFFLE()
5906   // Handle insertions of floating-point values.  in lowerINSERT_VECTOR_ELT()
5920     uint64_t Index = Op2->getAsZExtVal();  in lowerINSERT_VECTOR_ELT()
5921     unsigned Mask = VT.getVectorNumElements() - 1;  in lowerINSERT_VECTOR_ELT()
5938   // Handle extractions of floating-point values.  in lowerEXTRACT_VECTOR_ELT()
5947     uint64_t Index = CIndexN->getZExtValue();  in lowerEXTRACT_VECTOR_ELT()
5948     unsigned Mask = VecVT.getVectorNumElements() - 1;  in lowerEXTRACT_VECTOR_ELT()
5996     unsigned End = MaskElt + NumInPerOut - 1;  in lowerZERO_EXTEND_VECTOR_INREG()
6021     if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,  in lowerShift()
6030     SDValue Splat = BVN->getSplatValue(&UndefElements);  in lowerShift()
6032       // Since i32 is the smallest legal type, we either need a no-op  in lowerShift()
6042     if (VSN->isSplat()) {  in lowerShift()
6043       SDValue VSNOp0 = VSN->getOperand(0);  in lowerShift()
6044       unsigned Index = VSN->getSplatIndex();  in lowerShift()
6049         // Since i32 is the smallest legal type, we either need a no-op  in lowerShift()
6103   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();  in lowerREADCYCLECOUNTER()
6293 // only for 128-bit integer types).
6298   switch (N->getOpcode()) {  in LowerOperationWrapper()
6302     SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };  in LowerOperationWrapper()
6303     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();  in LowerOperationWrapper()
6308     if (N->getValueType(0) == MVT::f128)  in LowerOperationWrapper()
6317     SDValue Val = N->getOperand(1);  in LowerOperationWrapper()
6322     SDValue Ops[] = {N->getOperand(0), Val, N->getOperand(2)};  in LowerOperationWrapper()
6323     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();  in LowerOperationWrapper()
6328     if (cast<AtomicSDNode>(N)->getSuccessOrdering() ==  in LowerOperationWrapper()
6338     SDValue Ops[] = { N->getOperand(0), N->getOperand(1),  in LowerOperationWrapper()
6339                       lowerI128ToGR128(DAG, N->getOperand(2)),  in LowerOperationWrapper()
6340                       lowerI128ToGR128(DAG, N->getOperand(3)) };  in LowerOperationWrapper()
6341     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();  in LowerOperationWrapper()
6346     Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));  in LowerOperationWrapper()
6353     SDValue Src = N->getOperand(0);  in LowerOperationWrapper()
6354     if (N->getValueType(0) == MVT::i128 && Src.getValueType() == MVT::f128 &&  in LowerOperationWrapper()
6543       // Get a VPERM-like permute mask and see whether the bytes covered  in combineExtract()
6572       // Make sure that the least-significant bit of the extracted value  in combineExtract()
6578       Op = Op.getOperand(End / OpBytesPerElement - 1);  in combineExtract()
6603       unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;  in combineExtract()
6610       Byte += SubByte - MinSubByte;  in combineExtract()
6647           // splitting the original elements into Scale equal-sized pieces  in combineTruncateExtract()
6648           // and for truncation purposes want the last (least-significant)  in combineTruncateExtract()
6652           unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;  in combineTruncateExtract()
6672   SDValue N0 = N->getOperand(0);  in combineZERO_EXTEND()
6673   EVT VT = N->getValueType(0);  in combineZERO_EXTEND()
6679       SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),  in combineZERO_EXTEND()
6680                         DAG.getConstant(FalseOp->getZExtValue(), DL, VT),  in combineZERO_EXTEND()
6721   // into (select_cc LHS, RHS, -1, 0, COND)  in combineSIGN_EXTEND_INREG()
6723   SDValue N0 = N->getOperand(0);  in combineSIGN_EXTEND_INREG()
6724   EVT VT = N->getValueType(0);  in combineSIGN_EXTEND_INREG()
6725   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();  in combineSIGN_EXTEND_INREG()
6744   SDValue N0 = N->getOperand(0);  in combineSIGN_EXTEND()
6745   EVT VT = N->getValueType(0);  in combineSIGN_EXTEND()
6751         unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());  in combineSIGN_EXTEND()
6752         unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;  in combineSIGN_EXTEND()
6753         unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;  in combineSIGN_EXTEND()
6772   unsigned Opcode = N->getOpcode();  in combineMERGE()
6773   SDValue Op0 = N->getOperand(0);  in combineMERGE()
6774   SDValue Op1 = N->getOperand(1);  in combineMERGE()
6778     // (z_merge_* 0, 0) -> 0.  This is mostly useful for using VLLEZF  in combineMERGE()
6780     if (Op1 == N->getOperand(0))  in combineMERGE()
6782     // (z_merge_? 0, X) -> (z_unpackl_? 0, X).  in combineMERGE()
6808   for (SDNode::use_iterator UI = LD->use_begin(), UIEnd = LD->use_end();  in isI128MovedToParts()
6817     if (User->getOpcode() == ISD::SRL &&  in isI128MovedToParts()
6818         User->getOperand(1).getOpcode() == ISD::Constant &&  in isI128MovedToParts()
6819         User->getConstantOperandVal(1) == 64 && User->hasOneUse()) {  in isI128MovedToParts()
6820       User = *User->use_begin();  in isI128MovedToParts()
6823     if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(0) != MVT::i64)  in isI128MovedToParts()
6844   for (SDNode::use_iterator UI = LD->use_begin(), UIEnd = LD->use_end();  in isF128MovedToParts()
6852     if (!User->hasOneUse() || !User->isMachineOpcode() ||  in isF128MovedToParts()
6853         User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)  in isF128MovedToParts()
6856     switch (User->getConstantOperandVal(1)) {  in isF128MovedToParts()
6877   EVT LdVT = N->getValueType(0);  in combineLOAD()
6880   // Replace a 128-bit load that is used solely to move its value into GPRs  in combineLOAD()
6883   if (LD->isSimple() && ISD::isNormalLoad(LD)) {  in combineLOAD()
6891             HiPart->getValueType(0), DL, LD->getChain(), LD->getBasePtr(),  in combineLOAD()
6892             LD->getPointerInfo(), LD->getOriginalAlign(),  in combineLOAD()
6893             LD->getMemOperand()->getFlags(), LD->getAAInfo());  in combineLOAD()
6900             LoPart->getValueType(0), DL, LD->getChain(),  in combineLOAD()
6901             DAG.getObjectPtrOffset(DL, LD->getBasePtr(), TypeSize::getFixed(8)),  in combineLOAD()
6902             LD->getPointerInfo().getWithOffset(8), LD->getOriginalAlign(),  in combineLOAD()
6903             LD->getMemOperand()->getFlags(), LD->getAAInfo());  in combineLOAD()
6927   for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();  in combineLOAD()
6929     if (UI->getOpcode() == SystemZISD::REPLICATE) {  in combineLOAD()
6945     for (SDValue Op : U->ops())  in combineLOAD()
6970     if ((unsigned) M[i] != NumElts - 1 - i)  in isVectorElementSwap()
6978   for (auto *U : StoredVal->uses()) {  in isOnlyUsedByStores()
6980       EVT CurrMemVT = ST->getMemoryVT().getScalarType();  in isOnlyUsedByStores()
6996   if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())  in isI128MovedFromParts()
7004   if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||  in isI128MovedFromParts()
7010   if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||  in isI128MovedFromParts()
7013   if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||  in isI128MovedFromParts()
7024   if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||  in isF128MovedFromParts()
7028   if (Val->getNumOperands() != 5 ||  in isF128MovedFromParts()
7029       Val->getOperand(0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||  in isF128MovedFromParts()
7030       Val->getOperand(2)->getAsZExtVal() != SystemZ::subreg_l64 ||  in isF128MovedFromParts()
7031       Val->getOperand(4)->getAsZExtVal() != SystemZ::subreg_h64)  in isF128MovedFromParts()
7034   LoPart = Val->getOperand(1);  in isF128MovedFromParts()
7035   HiPart = Val->getOperand(3);  in isF128MovedFromParts()
7043   auto &Op1 = N->getOperand(1);  in combineSTORE()
7044   EVT MemVT = SN->getMemoryVT();  in combineSTORE()
7049   if (MemVT.isInteger() && SN->isTruncatingStore()) {  in combineSTORE()
7051             combineTruncateExtract(SDLoc(N), MemVT, SN->getValue(), DCI)) {  in combineSTORE()
7055       return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value,  in combineSTORE()
7056                                SN->getBasePtr(), SN->getMemoryVT(),  in combineSTORE()
7057                                SN->getMemOperand());  in combineSTORE()
7061   if (!SN->isTruncatingStore() &&  in combineSTORE()
7063       Op1.getNode()->hasOneUse() &&  in combineSTORE()
7072         N->getOperand(0), BSwapOp, N->getOperand(2)  in combineSTORE()
7077                                 Ops, MemVT, SN->getMemOperand());  in combineSTORE()
7079   // Combine STORE (element-swap) into VSTER  in combineSTORE()
7080   if (!SN->isTruncatingStore() &&  in combineSTORE()
7082       Op1.getNode()->hasOneUse() &&  in combineSTORE()
7085     ArrayRef<int> ShuffleMask = SVN->getMask();  in combineSTORE()
7088         N->getOperand(0), Op1.getOperand(0), N->getOperand(2)  in combineSTORE()
7093                                      Ops, MemVT, SN->getMemOperand());  in combineSTORE()
7098   if (!SN->isTruncatingStore() &&  in combineSTORE()
7101       N->getOperand(0).reachesChainWithoutSideEffects(SDValue(Op1.getNode(), 1))) {  in combineSTORE()
7102       SDValue Ops[] = { Op1.getOperand(0), N->getOperand(2) };  in combineSTORE()
7105                                      Ops, MemVT, SN->getMemOperand());  in combineSTORE()
7108   // Transform a store of a 128-bit value moved from parts into two stores.  in combineSTORE()
7109   if (SN->isSimple() && ISD::isNormalStore(SN)) {  in combineSTORE()
7115         DAG.getStore(SN->getChain(), DL, HiPart, SN->getBasePtr(),  in combineSTORE()
7116                      SN->getPointerInfo(), SN->getOriginalAlign(),  in combineSTORE()
7117                      SN->getMemOperand()->getFlags(), SN->getAAInfo());  in combineSTORE()
7119         DAG.getStore(SN->getChain(), DL, LoPart,  in combineSTORE()
7120                      DAG.getObjectPtrOffset(DL, SN->getBasePtr(),  in combineSTORE()
7122                      SN->getPointerInfo().getWithOffset(8),  in combineSTORE()
7123                      SN->getOriginalAlign(),  in combineSTORE()
7124                      SN->getMemOperand()->getFlags(), SN->getAAInfo());  in combineSTORE()
7132   // it is straight-forward to handle the zero-extend node in the initial  in combineSTORE()
7144       if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||  in combineSTORE()
7145           isInt<16>(C->getSExtValue()) || MemVT.getStoreSize() <= 2)  in combineSTORE()
7147       SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, C->getZExtValue()));  in combineSTORE()
7159       if (MulOp->getOpcode() == ISD::MUL &&  in combineSTORE()
7162         SDValue LHS = MulOp->getOperand(0);  in combineSTORE()
7163         if (LHS->getOpcode() == ISD::ZERO_EXTEND)  in combineSTORE()
7164           WordVT = LHS->getOperand(0).getValueType();  in combineSTORE()
7165         else if (LHS->getOpcode() == ISD::AssertZext)  in combineSTORE()
7166           WordVT = cast<VTSDNode>(LHS->getOperand(1))->getVT();  in combineSTORE()
7170         if (auto *C = dyn_cast<ConstantSDNode>(MulOp->getOperand(1))) {  in combineSTORE()
7172               APInt(MulVT.getSizeInBits(), C->getZExtValue()));  in combineSTORE()
7176             Word = DAG.getZExtOrTrunc(LHS->getOperand(0), SDLoc(SN), WordVT);  in combineSTORE()
7183       SDValue SplatVal = Op1->getOperand(0);  in combineSTORE()
7201       return DAG.getStore(SN->getChain(), SDLoc(SN), SplatVal,  in combineSTORE()
7202                           SN->getBasePtr(), SN->getMemOperand());  in combineSTORE()
7212   // Combine element-swap (LOAD) into VLER  in combineVECTOR_SHUFFLE()
7213   if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&  in combineVECTOR_SHUFFLE()
7214       N->getOperand(0).hasOneUse() &&  in combineVECTOR_SHUFFLE()
7217     ArrayRef<int> ShuffleMask = SVN->getMask();  in combineVECTOR_SHUFFLE()
7218     if (isVectorElementSwap(ShuffleMask, N->getValueType(0))) {  in combineVECTOR_SHUFFLE()
7219       SDValue Load = N->getOperand(0);  in combineVECTOR_SHUFFLE()
7222       // Create the element-swapping load.  in combineVECTOR_SHUFFLE()
7224         LD->getChain(),    // Chain  in combineVECTOR_SHUFFLE()
7225         LD->getBasePtr()   // Ptr  in combineVECTOR_SHUFFLE()
7229                                 DAG.getVTList(LD->getValueType(0), MVT::Other),  in combineVECTOR_SHUFFLE()
7230                                 Ops, LD->getMemoryVT(), LD->getMemOperand());  in combineVECTOR_SHUFFLE()
7256   SDValue Op = N->getOperand(0);  in combineEXTRACT_VECTOR_ELT()
7269                      Op.getOperand(0), N->getOperand(1));  in combineEXTRACT_VECTOR_ELT()
7272     if (EltVT != N->getValueType(0)) {  in combineEXTRACT_VECTOR_ELT()
7274       Op = DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op);  in combineEXTRACT_VECTOR_ELT()
7280   if (auto *IndexN = dyn_cast<ConstantSDNode>(N->getOperand(1))) {  in combineEXTRACT_VECTOR_ELT()
7281     SDValue Op0 = N->getOperand(0);  in combineEXTRACT_VECTOR_ELT()
7283     return combineExtract(SDLoc(N), N->getValueType(0), VecVT, Op0,  in combineEXTRACT_VECTOR_ELT()
7284                           IndexN->getZExtValue(), DCI, false);  in combineEXTRACT_VECTOR_ELT()
7293   if (N->getOperand(0) == N->getOperand(1))  in combineJOIN_DWORDS()
7294     return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0),  in combineJOIN_DWORDS()
7295                        N->getOperand(0));  in combineJOIN_DWORDS()
7300   SDValue Chain1 = N1->getOperand(0);  in MergeInputChains()
7301   SDValue Chain2 = N2->getOperand(0);  in MergeInputChains()
7307   // FIXME - we could handle more complex cases via TokenFactor,  in MergeInputChains()
7319   // (fpround (extract_vector_elt X 1)) ->  in combineFP_ROUND()
7324   unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;  in combineFP_ROUND()
7326   SDValue Op0 = N->getOperand(OpNo);  in combineFP_ROUND()
7327   if (N->getValueType(0) == MVT::f32 && Op0.hasOneUse() &&  in combineFP_ROUND()
7333     for (auto *U : Vec->uses()) {  in combineFP_ROUND()
7334       if (U != Op0.getNode() && U->hasOneUse() &&  in combineFP_ROUND()
7335           U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&  in combineFP_ROUND()
7336           U->getOperand(0) == Vec &&  in combineFP_ROUND()
7337           U->getOperand(1).getOpcode() == ISD::Constant &&  in combineFP_ROUND()
7338           U->getConstantOperandVal(1) == 1) {  in combineFP_ROUND()
7339         SDValue OtherRound = SDValue(*U->use_begin(), 0);  in combineFP_ROUND()
7340         if (OtherRound.getOpcode() == N->getOpcode() &&  in combineFP_ROUND()
7344           if (N->isStrictFPOpcode()) {  in combineFP_ROUND()
7367                                N->getVTList(), Extract0, Chain);  in combineFP_ROUND()
7383   // (fpextend (extract_vector_elt X 2)) ->  in combineFP_EXTEND()
7388   unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;  in combineFP_EXTEND()
7390   SDValue Op0 = N->getOperand(OpNo);  in combineFP_EXTEND()
7391   if (N->getValueType(0) == MVT::f64 && Op0.hasOneUse() &&  in combineFP_EXTEND()
7397     for (auto *U : Vec->uses()) {  in combineFP_EXTEND()
7398       if (U != Op0.getNode() && U->hasOneUse() &&  in combineFP_EXTEND()
7399           U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&  in combineFP_EXTEND()
7400           U->getOperand(0) == Vec &&  in combineFP_EXTEND()
7401           U->getOperand(1).getOpcode() == ISD::Constant &&  in combineFP_EXTEND()
7402           U->getConstantOperandVal(1) == 2) {  in combineFP_EXTEND()
7403         SDValue OtherExtend = SDValue(*U->use_begin(), 0);  in combineFP_EXTEND()
7404         if (OtherExtend.getOpcode() == N->getOpcode() &&  in combineFP_EXTEND()
7408           if (N->isStrictFPOpcode()) {  in combineFP_EXTEND()
7431                                N->getVTList(), Extract0, Chain);  in combineFP_EXTEND()
7446   unsigned Opcode = N->getOpcode();  in combineINT_TO_FP()
7447   EVT OutVT = N->getValueType(0);  in combineINT_TO_FP()
7449   SDValue Op = N->getOperand(0);  in combineINT_TO_FP()
7450   unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();  in combineINT_TO_FP()
7451   unsigned InScalarBits = Op->getValueType(0).getScalarSizeInBits();  in combineINT_TO_FP()
7453   // Insert an extension before type-legalization to avoid scalarization, e.g.:  in combineINT_TO_FP()
7457   if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&  in combineINT_TO_FP()
7459     unsigned NumElts = cast<FixedVectorType>(OutLLVMTy)->getNumElements();  in combineINT_TO_FP()
7461         Ctx, EVT::getIntegerVT(Ctx, OutLLVMTy->getScalarSizeInBits()), NumElts);  in combineINT_TO_FP()
7474   if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&  in combineBSWAP()
7475       N->getOperand(0).hasOneUse() &&  in combineBSWAP()
7476       canLoadStoreByteSwapped(N->getValueType(0))) {  in combineBSWAP()
7477       SDValue Load = N->getOperand(0);  in combineBSWAP()
7480       // Create the byte-swapping load.  in combineBSWAP()
7482         LD->getChain(),    // Chain  in combineBSWAP()
7483         LD->getBasePtr()   // Ptr  in combineBSWAP()
7485       EVT LoadVT = N->getValueType(0);  in combineBSWAP()
7491                                 Ops, LD->getMemoryVT(), LD->getMemOperand());  in combineBSWAP()
7495       if (N->getValueType(0) == MVT::i16)  in combineBSWAP()
7511   SDValue Op = N->getOperand(0);  in combineBSWAP()
7529         (canLoadStoreByteSwapped(N->getValueType(0)) &&  in combineBSWAP()
7531       EVT VecVT = N->getValueType(0);  in combineBSWAP()
7532       EVT EltVT = N->getValueType(0).getVectorElementType();  in combineBSWAP()
7560       EVT VecVT = N->getValueType(0);  in combineBSWAP()
7573       return DAG.getVectorShuffle(VecVT, SDLoc(N), Op0, Op1, SV->getMask());  in combineBSWAP()
7591   if (ICmp->getOpcode() != SystemZISD::ICMP)  in combineCCMask()
7593   auto *CompareLHS = ICmp->getOperand(0).getNode();  in combineCCMask()
7594   auto *CompareRHS = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));  in combineCCMask()
7599   if (CompareLHS->getOpcode() == SystemZISD::SELECT_CCMASK) {  in combineCCMask()
7608     auto *TrueVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(0));  in combineCCMask()
7611     auto *FalseVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));  in combineCCMask()
7614     if (CompareRHS->getZExtValue() == FalseVal->getZExtValue())  in combineCCMask()
7616     else if (CompareRHS->getZExtValue() != TrueVal->getZExtValue())  in combineCCMask()
7620     auto *NewCCValid = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(2));  in combineCCMask()
7621     auto *NewCCMask = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(3));  in combineCCMask()
7624     CCValid = NewCCValid->getZExtValue();  in combineCCMask()
7625     CCMask = NewCCMask->getZExtValue();  in combineCCMask()
7630     CCReg = CompareLHS->getOperand(4);  in combineCCMask()
7635   if (CompareLHS->getOpcode() == ISD::SRA) {  in combineCCMask()
7636     auto *SRACount = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));  in combineCCMask()
7637     if (!SRACount || SRACount->getZExtValue() != 30)  in combineCCMask()
7639     auto *SHL = CompareLHS->getOperand(0).getNode();  in combineCCMask()
7640     if (SHL->getOpcode() != ISD::SHL)  in combineCCMask()
7642     auto *SHLCount = dyn_cast<ConstantSDNode>(SHL->getOperand(1));  in combineCCMask()
7643     if (!SHLCount || SHLCount->getZExtValue() != 30 - SystemZ::IPM_CC)  in combineCCMask()
7645     auto *IPM = SHL->getOperand(0).getNode();  in combineCCMask()
7646     if (IPM->getOpcode() != SystemZISD::IPM)  in combineCCMask()
7650     if (!CompareLHS->hasOneUse())  in combineCCMask()
7653     if (CompareRHS->getZExtValue() != 0)  in combineCCMask()
7660     CCReg = IPM->getOperand(0);  in combineCCMask()
7672   auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));  in combineBR_CCMASK()
7673   auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));  in combineBR_CCMASK()
7677   int CCValidVal = CCValid->getZExtValue();  in combineBR_CCMASK()
7678   int CCMaskVal = CCMask->getZExtValue();  in combineBR_CCMASK()
7679   SDValue Chain = N->getOperand(0);  in combineBR_CCMASK()
7680   SDValue CCReg = N->getOperand(4);  in combineBR_CCMASK()
7683     return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0),  in combineBR_CCMASK()
7687                        N->getOperand(3), CCReg);  in combineBR_CCMASK()
7696   auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));  in combineSELECT_CCMASK()
7697   auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));  in combineSELECT_CCMASK()
7701   int CCValidVal = CCValid->getZExtValue();  in combineSELECT_CCMASK()
7702   int CCMaskVal = CCMask->getZExtValue();  in combineSELECT_CCMASK()
7703   SDValue CCReg = N->getOperand(4);  in combineSELECT_CCMASK()
7706     return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),  in combineSELECT_CCMASK()
7707                        N->getOperand(0), N->getOperand(1),  in combineSELECT_CCMASK()
7719   auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));  in combineGET_CCMASK()
7720   auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));  in combineGET_CCMASK()
7723   int CCValidVal = CCValid->getZExtValue();  in combineGET_CCMASK()
7724   int CCMaskVal = CCMask->getZExtValue();  in combineGET_CCMASK()
7726   SDValue Select = N->getOperand(0);  in combineGET_CCMASK()
7727   if (Select->getOpcode() == ISD::TRUNCATE)  in combineGET_CCMASK()
7728     Select = Select->getOperand(0);  in combineGET_CCMASK()
7729   if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)  in combineGET_CCMASK()
7732   auto *SelectCCValid = dyn_cast<ConstantSDNode>(Select->getOperand(2));  in combineGET_CCMASK()
7733   auto *SelectCCMask = dyn_cast<ConstantSDNode>(Select->getOperand(3));  in combineGET_CCMASK()
7736   int SelectCCValidVal = SelectCCValid->getZExtValue();  in combineGET_CCMASK()
7737   int SelectCCMaskVal = SelectCCMask->getZExtValue();  in combineGET_CCMASK()
7739   auto *TrueVal = dyn_cast<ConstantSDNode>(Select->getOperand(0));  in combineGET_CCMASK()
7740   auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1));  in combineGET_CCMASK()
7743   if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)  in combineGET_CCMASK()
7745   else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)  in combineGET_CCMASK()
7755   return Select->getOperand(4);  in combineGET_CCMASK()
7761   EVT VT = N->getValueType(0);  in combineIntDIVREM()
7770       DAG.isConstantIntBuildVectorOrConstantInt(N->getOperand(1)))  in combineIntDIVREM()
7779   unsigned Id = N->getConstantOperandVal(1);  in combineINTRINSIC()
7785     if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(2)))  in combineINTRINSIC()
7786       if (C->getZExtValue() >= 15)  in combineINTRINSIC()
7787         return DAG.getLoad(N->getValueType(0), SDLoc(N), N->getOperand(0),  in combineINTRINSIC()
7788                            N->getOperand(3), MachinePointerInfo());  in combineINTRINSIC()
7793     if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(3)))  in combineINTRINSIC()
7794       if (C->getZExtValue() >= 15)  in combineINTRINSIC()
7795         return DAG.getStore(N->getOperand(0), SDLoc(N), N->getOperand(2),  in combineINTRINSIC()
7796                             N->getOperand(4), MachinePointerInfo());  in combineINTRINSIC()
7804   if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)  in unwrapAddress()
7805     return N->getOperand(0);  in unwrapAddress()
7811   switch(N->getOpcode()) {  in PerformDAGCombine()
7895       if (!DemandedElts[OpNo - 1])  in getDemandedSrcElements()
7898       unsigned MaskBit = ((OpNo - 1) ? 1 : 4);  in getDemandedSrcElements()
7908       unsigned NumSrc0Els = 16 - FirstIdx;  in getDemandedSrcElements()
7920       SrcDemE = APInt(NumElts, -1);  in getDemandedSrcElements()
8065     unsigned SrcExtraBits = SrcBitWidth - VTBits;  in computeNumSignBitsBinOp()
8067       return (Common - SrcExtraBits);  in computeNumSignBitsBinOp()
8111       Tmp += VTBits - PackedOp.getScalarValueSizeInBits();  in ComputeNumSignBitsForTargetNode()
8133   switch (Op->getOpcode()) {  in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
8144   unsigned StackAlign = TFI->getStackAlignment();  in getStackProbeSize()
8148   // stack-probe-size attribute.  in getStackProbeSize()
8150       MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size", 4096);  in getStackProbeSize()
8152   StackProbeSize &= ~(StackAlign - 1);  in getStackProbeSize()
8156 //===----------------------------------------------------------------------===//
8158 //===----------------------------------------------------------------------===//
8164   MachineFunction &MF = *MBB->getParent();  in forceReg()
8171     BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::COPY), Reg)  in forceReg()
8177   BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LA), Reg)  in forceReg()
8189   MachineBasicBlock::iterator miI(std::next(MachineBasicBlock::iterator(MI)));  in checkCCKill()  local
8190   for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {  in checkCCKill()
8191     const MachineInstr& mi = *miI;  in checkCCKill()
8195       break; // Should have kill-flag - update below.  in checkCCKill()
8200   if (miI == MBB->end()) {  in checkCCKill()
8201     for (const MachineBasicBlock *Succ : MBB->successors())  in checkCCKill()
8202       if (Succ->isLiveIn(SystemZ::CC))  in checkCCKill()
8209 // Return true if it is OK for this Select pseudo-opcode to be cascaded
8210 // together with other Select pseudo-opcodes into a single basic-block with
8237   MachineFunction *MF = TrueMBB->getParent();  in createPHIsForSelects()
8238   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();  in createPHIsForSelects()
8241   unsigned CCValid = FirstMI->getOperand(3).getImm();  in createPHIsForSelects()
8242   unsigned CCMask = FirstMI->getOperand(4).getImm();  in createPHIsForSelects()
8244   MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();  in createPHIsForSelects()
8255     Register DestReg = MI->getOperand(0).getReg();  in createPHIsForSelects()
8256     Register TrueReg = MI->getOperand(1).getReg();  in createPHIsForSelects()
8257     Register FalseReg = MI->getOperand(2).getReg();  in createPHIsForSelects()
8262     if (MI->getOperand(4).getImm() == (CCValid ^ CCMask))  in createPHIsForSelects()
8271     DebugLoc DL = MI->getDebugLoc();  in createPHIsForSelects()
8272     BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg)  in createPHIsForSelects()
8280   MF->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);  in createPHIsForSelects()
8286   MachineFunction &MF = *BB->getParent();  in emitAdjCallStack()
8289   assert(TFL->hasReservedCallFrame(MF) &&  in emitAdjCallStack()
8290          "ADJSTACKDOWN and ADJSTACKUP should be no-ops");  in emitAdjCallStack()
8322            std::next(MachineBasicBlock::iterator(MI)), MBB->end())) {  in emitSelect()
8338       if (NextMI.readsVirtualRegister(SelMI->getOperand(0).getReg())) {  in emitSelect()
8352   bool CCKilled = (LastMI->killsRegister(SystemZ::CC, /*TRI=*/nullptr) ||  in emitSelect()
8359   // live-in to both FalseMBB and JoinMBB.  in emitSelect()
8361     FalseMBB->addLiveIn(SystemZ::CC);  in emitSelect()
8362     JoinMBB->addLiveIn(SystemZ::CC);  in emitSelect()
8369   BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))  in emitSelect()
8371   MBB->addSuccessor(JoinMBB);  in emitSelect()
8372   MBB->addSuccessor(FalseMBB);  in emitSelect()
8377   MBB->addSuccessor(JoinMBB);  in emitSelect()
8385     SelMI->eraseFromParent();  in emitSelect()
8387   MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();  in emitSelect()
8389     MBB->splice(InsertPos, StartMBB, DbgMI);  in emitSelect()
8413   StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp);  in emitCondStore()
8419     if (I->isStore()) {  in emitCondStore()
8425   // order to avoid matching the index register, but the performance trade-offs  in emitCondStore()
8431     BuildMI(*MBB, MI, DL, TII->get(STOCOpcode))  in emitCondStore()
8452   // live-in to both FalseMBB and JoinMBB.  in emitCondStore()
8455     FalseMBB->addLiveIn(SystemZ::CC);  in emitCondStore()
8456     JoinMBB->addLiveIn(SystemZ::CC);  in emitCondStore()
8463   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitCondStore()
8465   MBB->addSuccessor(JoinMBB);  in emitCondStore()
8466   MBB->addSuccessor(FalseMBB);  in emitCondStore()
8472   BuildMI(MBB, DL, TII->get(StoreOpcode))  in emitCondStore()
8478   MBB->addSuccessor(JoinMBB);  in emitCondStore()
8489   MachineFunction &MF = *MBB->getParent();  in emitICmp128Hi()
8493   // Synthetic instruction to compare 128-bit values.  in emitICmp128Hi()
8517   BuildMI(MBB, MI.getDebugLoc(), TII->get(HiOpcode))  in emitICmp128Hi()
8519   BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))  in emitICmp128Hi()
8521   MBB->addSuccessor(JoinMBB);  in emitICmp128Hi()
8522   MBB->addSuccessor(HiEqMBB);  in emitICmp128Hi()
8536   BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::VCHLGS), Temp)  in emitICmp128Hi()
8538   MBB->addSuccessor(JoinMBB);  in emitICmp128Hi()
8540   // Mark CC as live-in to JoinMBB.  in emitICmp128Hi()
8541   JoinMBB->addLiveIn(SystemZ::CC);  in emitICmp128Hi()
8555   MachineFunction &MF = *MBB->getParent();  in emitAtomicLoadBinary()
8571   unsigned LOpcode  = TII->getOpcodeForOffset(SystemZ::L,  Disp);  in emitAtomicLoadBinary()
8572   unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);  in emitAtomicLoadBinary()
8592   BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);  in emitAtomicLoadBinary()
8593   MBB->addSuccessor(LoopMBB);  in emitAtomicLoadBinary()
8604   BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)  in emitAtomicLoadBinary()
8607   BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)  in emitAtomicLoadBinary()
8612     BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2);  in emitAtomicLoadBinary()
8614     BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal)  in emitAtomicLoadBinary()
8615       .addReg(Tmp).addImm(-1U << (32 - BitSize));  in emitAtomicLoadBinary()
8618     BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal)  in emitAtomicLoadBinary()
8624     BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal)  in emitAtomicLoadBinary()
8626       .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize);  in emitAtomicLoadBinary()
8627   BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)  in emitAtomicLoadBinary()
8629   BuildMI(MBB, DL, TII->get(CSOpcode), Dest)  in emitAtomicLoadBinary()
8634   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitAtomicLoadBinary()
8636   MBB->addSuccessor(LoopMBB);  in emitAtomicLoadBinary()
8637   MBB->addSuccessor(DoneMBB);  in emitAtomicLoadBinary()
8646 // minimum or maximum value.  KeepOldMask is the BRC condition-code mask
8651   MachineFunction &MF = *MBB->getParent();  in emitAtomicLoadMinMax()
8666   unsigned LOpcode  = TII->getOpcodeForOffset(SystemZ::L,  Disp);  in emitAtomicLoadMinMax()
8667   unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);  in emitAtomicLoadMinMax()
8690   BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);  in emitAtomicLoadMinMax()
8691   MBB->addSuccessor(LoopMBB);  in emitAtomicLoadMinMax()
8699   BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)  in emitAtomicLoadMinMax()
8702   BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)  in emitAtomicLoadMinMax()
8704   BuildMI(MBB, DL, TII->get(CompareOpcode))  in emitAtomicLoadMinMax()
8706   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitAtomicLoadMinMax()
8708   MBB->addSuccessor(UpdateMBB);  in emitAtomicLoadMinMax()
8709   MBB->addSuccessor(UseAltMBB);  in emitAtomicLoadMinMax()
8715   BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)  in emitAtomicLoadMinMax()
8718   MBB->addSuccessor(UpdateMBB);  in emitAtomicLoadMinMax()
8728   BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)  in emitAtomicLoadMinMax()
8731   BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)  in emitAtomicLoadMinMax()
8733   BuildMI(MBB, DL, TII->get(CSOpcode), Dest)  in emitAtomicLoadMinMax()
8738   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitAtomicLoadMinMax()
8740   MBB->addSuccessor(LoopMBB);  in emitAtomicLoadMinMax()
8741   MBB->addSuccessor(DoneMBB);  in emitAtomicLoadMinMax()
8752   MachineFunction &MF = *MBB->getParent();  in emitAtomicCmpSwapW()
8769   // Get the right opcodes for the displacement and zero-extension.  in emitAtomicCmpSwapW()
8770   unsigned LOpcode  = TII->getOpcodeForOffset(SystemZ::L,  Disp);  in emitAtomicCmpSwapW()
8771   unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);  in emitAtomicCmpSwapW()
8795   BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal)  in emitAtomicCmpSwapW()
8799   MBB->addSuccessor(LoopMBB);  in emitAtomicCmpSwapW()
8807   //   %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0  in emitAtomicCmpSwapW()
8808   //                      ^^ Replace the upper 32-BitSize bits of the  in emitAtomicCmpSwapW()
8815   BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)  in emitAtomicCmpSwapW()
8818   BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)  in emitAtomicCmpSwapW()
8821   BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)  in emitAtomicCmpSwapW()
8823   BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)  in emitAtomicCmpSwapW()
8824     .addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0);  in emitAtomicCmpSwapW()
8825   BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)  in emitAtomicCmpSwapW()
8827   BuildMI(MBB, DL, TII->get(SystemZ::CR))  in emitAtomicCmpSwapW()
8829   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitAtomicCmpSwapW()
8832   MBB->addSuccessor(DoneMBB);  in emitAtomicCmpSwapW()
8833   MBB->addSuccessor(SetMBB);  in emitAtomicCmpSwapW()
8836   //   %StoreVal     = RLL %RetrySwapVal, -BitSize(%NegBitShift)  in emitAtomicCmpSwapW()
8842   BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)  in emitAtomicCmpSwapW()
8843     .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);  in emitAtomicCmpSwapW()
8844   BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)  in emitAtomicCmpSwapW()
8849   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitAtomicCmpSwapW()
8851   MBB->addSuccessor(LoopMBB);  in emitAtomicCmpSwapW()
8852   MBB->addSuccessor(DoneMBB);  in emitAtomicCmpSwapW()
8854   // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in  in emitAtomicCmpSwapW()
8858     DoneMBB->addLiveIn(SystemZ::CC);  in emitAtomicCmpSwapW()
8872   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest)  in emitPair128()
8887   MachineFunction &MF = *MBB->getParent();  in emitExt128()
8896   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128);  in emitExt128()
8901     BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64)  in emitExt128()
8903     BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128)  in emitExt128()
8907   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)  in emitExt128()
8918   MachineFunction &MF = *MBB->getParent();  in emitMemMemWrapper()
8929   auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {  in emitMemMemWrapper()
8932       unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);  in emitMemMemWrapper()
8933       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opcode), Reg)  in emitMemMemWrapper()
8958                             unsigned Length) -> void {  in emitMemMemWrapper()
8963         BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::MVI))  in emitMemMemWrapper()
8966         BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::STC))  in emitMemMemWrapper()
8968       if (--Length == 0)  in emitMemMemWrapper()
8971     BuildMI(*MBB, InsPos, DL, TII->get(Opcode))  in emitMemMemWrapper()
8989         // A two-CLC sequence is a clear win over a loop, not least because  in emitMemMemWrapper()
8990         // it needs only one branch.  A three-CLC sequence needs the same  in emitMemMemWrapper()
9001       // to choose between straight-line code and looping code, since the  in emitMemMemWrapper()
9020       TII->loadImmediate(*MBB, MI, StartCountReg, ImmLength / 256);  in emitMemMemWrapper()
9023       BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg)  in emitMemMemWrapper()
9030     auto loadZeroAddress = [&]() -> MachineOperand {  in emitMemMemWrapper()
9032       BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0);  in emitMemMemWrapper()
9070       BuildMI(MBB, DL, TII->get(SystemZ::CGHI))  in emitMemMemWrapper()
9071         .addReg(LenAdjReg).addImm(IsMemset ? -2 : -1);  in emitMemMemWrapper()
9072       BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitMemMemWrapper()
9075       MBB->addSuccessor(AllDoneMBB);  in emitMemMemWrapper()
9077         MBB->addSuccessor(StartMBB);  in emitMemMemWrapper()
9084         MBB->addSuccessor(MemsetOneCheckMBB);  in emitMemMemWrapper()
9086         BuildMI(MBB, DL, TII->get(SystemZ::CGHI))  in emitMemMemWrapper()
9087           .addReg(LenAdjReg).addImm(-1);  in emitMemMemWrapper()
9088         BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitMemMemWrapper()
9091         MBB->addSuccessor(MemsetOneMBB, {10, 100});  in emitMemMemWrapper()
9092         MBB->addSuccessor(StartMBB, {90, 100});  in emitMemMemWrapper()
9097         insertMemMemOp(MBB, MBB->end(),  in emitMemMemWrapper()
9101         BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(AllDoneMBB);  in emitMemMemWrapper()
9102         MBB->addSuccessor(AllDoneMBB);  in emitMemMemWrapper()
9108       BuildMI(MBB, DL, TII->get(SystemZ::CGHI))  in emitMemMemWrapper()
9110       BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitMemMemWrapper()
9113       MBB->addSuccessor(DoneMBB);  in emitMemMemWrapper()
9114       MBB->addSuccessor(LoopMBB);  in emitMemMemWrapper()
9124       MBB->addSuccessor(LoopMBB);  in emitMemMemWrapper()
9130         // CC live-through into EndMBB, so add it as live-in.  in emitMemMemWrapper()
9131         DoneMBB->addLiveIn(SystemZ::CC);  in emitMemMemWrapper()
9147     BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg)  in emitMemMemWrapper()
9151       BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg)  in emitMemMemWrapper()
9154     BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg)  in emitMemMemWrapper()
9158       BuildMI(MBB, DL, TII->get(SystemZ::PFD))  in emitMemMemWrapper()
9160         .addReg(ThisDestReg).addImm(DestDisp - IsMemset + 768).addReg(0);  in emitMemMemWrapper()
9161     insertMemMemOp(MBB, MBB->end(),  in emitMemMemWrapper()
9165       BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitMemMemWrapper()
9168       MBB->addSuccessor(EndMBB);  in emitMemMemWrapper()
9169       MBB->addSuccessor(NextMBB);  in emitMemMemWrapper()
9175     //   %NextCountReg = AGHI %ThisCountReg, -1  in emitMemMemWrapper()
9182     BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg)  in emitMemMemWrapper()
9185       BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg)  in emitMemMemWrapper()
9187     BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg)  in emitMemMemWrapper()
9188       .addReg(ThisCountReg).addImm(-1);  in emitMemMemWrapper()
9189     BuildMI(MBB, DL, TII->get(SystemZ::CGHI))  in emitMemMemWrapper()
9191     BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitMemMemWrapper()
9194     MBB->addSuccessor(LoopMBB);  in emitMemMemWrapper()
9195     MBB->addSuccessor(DoneMBB);  in emitMemMemWrapper()
9209       BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg)  in emitMemMemWrapper()
9213         BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)  in emitMemMemWrapper()
9217         insertMemMemOp(MBB, MBB->end(),  in emitMemMemWrapper()
9221         BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))  in emitMemMemWrapper()
9226       MBB->addSuccessor(AllDoneMBB);  in emitMemMemWrapper()
9231           MBB->addLiveIn(SystemZ::CC);  in emitMemMemWrapper()
9237   // Handle any remaining bytes with straight-line code.  in emitMemMemWrapper()
9240     // The previous iteration might have created out-of-range displacements.  in emitMemMemWrapper()
9247     ImmLength -= ThisLength;  in emitMemMemWrapper()
9252       BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitMemMemWrapper()
9255       MBB->addSuccessor(EndMBB);  in emitMemMemWrapper()
9256       MBB->addSuccessor(NextMBB);  in emitMemMemWrapper()
9261     MBB->addSuccessor(EndMBB);  in emitMemMemWrapper()
9263     MBB->addLiveIn(SystemZ::CC);  in emitMemMemWrapper()
9270 // Decompose string pseudo-instruction MI into a loop that continually performs
9274   MachineFunction &MF = *MBB->getParent();  in emitStringWrapper()
9295   MBB->addSuccessor(LoopMBB);  in emitStringWrapper()
9301   //   %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L  in emitStringWrapper()
9305   // The load of R0L can be hoisted by post-RA LICM.  in emitStringWrapper()
9308   BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg)  in emitStringWrapper()
9311   BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg)  in emitStringWrapper()
9314   BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg);  in emitStringWrapper()
9315   BuildMI(MBB, DL, TII->get(Opcode))  in emitStringWrapper()
9318   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitStringWrapper()
9320   MBB->addSuccessor(LoopMBB);  in emitStringWrapper()
9321   MBB->addSuccessor(DoneMBB);  in emitStringWrapper()
9323   DoneMBB->addLiveIn(SystemZ::CC);  in emitStringWrapper()
9333   MachineFunction &MF = *MBB->getParent();  in emitTransactionBegin()
9338   MI.setDesc(TII->get(Opcode));  in emitTransactionBegin()
9348   if (TFI->hasFP(MF))  in emitTransactionBegin()
9378   MachineFunction &MF = *MBB->getParent();  in emitLoadAndTestCmp0()
9386   const TargetRegisterClass *RC = MRI->getRegClass(SrcReg);  in emitLoadAndTestCmp0()
9387   Register DstReg = MRI->createVirtualRegister(RC);  in emitLoadAndTestCmp0()
9389   // Replace pseudo with a normal load-and-test that models the def as  in emitLoadAndTestCmp0()
9391   BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg)  in emitLoadAndTestCmp0()
9401   MachineFunction &MF = *MBB->getParent();  in emitProbedAlloca()
9419   Register PHIReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);  in emitProbedAlloca()
9420   Register IncReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);  in emitProbedAlloca()
9425   StartMBB->addSuccessor(LoopTestMBB);  in emitProbedAlloca()
9427   BuildMI(MBB, DL, TII->get(SystemZ::PHI), PHIReg)  in emitProbedAlloca()
9432   BuildMI(MBB, DL, TII->get(SystemZ::CLGFI))  in emitProbedAlloca()
9435   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitProbedAlloca()
9438   MBB->addSuccessor(LoopBodyMBB);  in emitProbedAlloca()
9439   MBB->addSuccessor(TailTestMBB);  in emitProbedAlloca()
9444   BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), IncReg)  in emitProbedAlloca()
9447   BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), SystemZ::R15D)  in emitProbedAlloca()
9450   BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D)  in emitProbedAlloca()
9451     .addReg(SystemZ::R15D).addImm(ProbeSize - 8).addReg(0)  in emitProbedAlloca()
9453   BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(LoopTestMBB);  in emitProbedAlloca()
9454   MBB->addSuccessor(LoopTestMBB);  in emitProbedAlloca()
9460   BuildMI(MBB, DL, TII->get(SystemZ::CGHI))  in emitProbedAlloca()
9463   BuildMI(MBB, DL, TII->get(SystemZ::BRC))  in emitProbedAlloca()
9466   MBB->addSuccessor(TailMBB);  in emitProbedAlloca()
9467   MBB->addSuccessor(DoneMBB);  in emitProbedAlloca()
9472   BuildMI(MBB, DL, TII->get(SystemZ::SLGR), SystemZ::R15D)  in emitProbedAlloca()
9475   BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D)  in emitProbedAlloca()
9476     .addReg(SystemZ::R15D).addImm(-8).addReg(PHIReg)  in emitProbedAlloca()
9478   MBB->addSuccessor(DoneMBB);  in emitProbedAlloca()
9482   BuildMI(*MBB, MBB->begin(), DL, TII->get(TargetOpcode::COPY), DstReg)  in emitProbedAlloca()
9495                      DAG.getIntPtrConstant(TFL->getBackchainOffset(MF), DL));  in getBackchainAddress()
9657 // compiler from crashing when list-ilp is used.
9669    The rounding method is in FPC Byte 3 bits 6-7, and has the following  in lowerGET_ROUNDING()
9674      11 Round to -inf  in lowerGET_ROUNDING()
9677     -1 Undefined  in lowerGET_ROUNDING()
9681      3 Round to -inf  in lowerGET_ROUNDING()
9737       DAG.getConstant(OpVT.getVectorNumElements() - 1, DL, MVT::i32));  in lowerVECREDUCE_ADD()