Lines Matching +full:fsin +full:- +full:output
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
62 /// Keeps track of state when getting the sign of a floating-point value as an
76 //===----------------------------------------------------------------------===//
126 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
210 UpdatedNodes->insert(N); in ReplacedNode()
214 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); in ReplaceNode()
215 dbgs() << " with: "; New->dump(&DAG)); in ReplaceNode()
217 assert(Old->getNumValues() == New->getNumValues() && in ReplaceNode()
222 UpdatedNodes->insert(New); in ReplaceNode()
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); in ReplaceNode()
228 dbgs() << " with: "; New->dump(&DAG)); in ReplaceNode()
232 UpdatedNodes->insert(New.getNode()); in ReplaceNode()
237 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); in ReplaceNode()
240 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { in ReplaceNode()
242 New[i]->dump(&DAG)); in ReplaceNode()
244 UpdatedNodes->insert(New[i].getNode()); in ReplaceNode()
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); in ReplaceNodeWithValue()
251 dbgs() << " with: "; New->dump(&DAG)); in ReplaceNodeWithValue()
255 UpdatedNodes->insert(New.getNode()); in ReplaceNodeWithValue()
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex(); in getStackAlignedMMO()
280 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
298 NewMask.push_back(-1); in ShuffleWithNarrowerEltType()
303 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); in ShuffleWithNarrowerEltType()
321 EVT VT = CFP->getValueType(0); in ExpandConstantFP()
322 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); in ExpandConstantFP()
325 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, in ExpandConstantFP()
329 APFloat APF = CFP->getValueAPF(); in ExpandConstantFP()
337 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); in ExpandConstantFP()
354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); in ExpandConstantFP()
371 EVT VT = CP->getValueType(0); in ExpandConstant()
372 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), in ExpandConstant()
374 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); in ExpandConstant()
403 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); in ExpandINSERT_VECTOR_ELT()
416 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' in OptimizeFloatStore()
422 SDValue Chain = ST->getChain(); in OptimizeFloatStore()
423 SDValue Ptr = ST->getBasePtr(); in OptimizeFloatStore()
424 SDValue Value = ST->getValue(); in OptimizeFloatStore()
425 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); in OptimizeFloatStore()
426 AAMDNodes AAInfo = ST->getAAInfo(); in OptimizeFloatStore()
434 if (CFP->getValueType(0) == MVT::f32 && in OptimizeFloatStore()
436 SDValue Con = DAG.getConstant(CFP->getValueAPF(). in OptimizeFloatStore()
439 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), in OptimizeFloatStore()
440 ST->getOriginalAlign(), MMOFlags, AAInfo); in OptimizeFloatStore()
443 if (CFP->getValueType(0) == MVT::f64 && in OptimizeFloatStore()
444 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) { in OptimizeFloatStore()
445 // If this target supports 64-bit registers, do a single 64-bit store. in OptimizeFloatStore()
447 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). in OptimizeFloatStore()
449 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), in OptimizeFloatStore()
450 ST->getOriginalAlign(), MMOFlags, AAInfo); in OptimizeFloatStore()
453 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { in OptimizeFloatStore()
454 // Otherwise, if the target supports 32-bit registers, use 2 32-bit in OptimizeFloatStore()
455 // stores. If the target supports neither 32- nor 64-bits, this in OptimizeFloatStore()
457 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); in OptimizeFloatStore()
463 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), in OptimizeFloatStore()
464 ST->getOriginalAlign(), MMOFlags, AAInfo); in OptimizeFloatStore()
467 ST->getPointerInfo().getWithOffset(4), in OptimizeFloatStore()
468 ST->getOriginalAlign(), MMOFlags, AAInfo); in OptimizeFloatStore()
479 SDValue Chain = ST->getChain(); in LegalizeStoreOps()
480 SDValue Ptr = ST->getBasePtr(); in LegalizeStoreOps()
483 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); in LegalizeStoreOps()
484 AAMDNodes AAInfo = ST->getAAInfo(); in LegalizeStoreOps()
486 if (!ST->isTruncatingStore()) { in LegalizeStoreOps()
493 SDValue Value = ST->getValue(); in LegalizeStoreOps()
500 EVT MemVT = ST->getMemoryVT(); in LegalizeStoreOps()
503 *ST->getMemOperand())) { in LegalizeStoreOps()
523 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), in LegalizeStoreOps()
524 ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
533 SDValue Value = ST->getValue(); in LegalizeStoreOps()
534 EVT StVT = ST->getMemoryVT(); in LegalizeStoreOps()
540 // Promote to a byte-sized store with upper bits zero if not in LegalizeStoreOps()
542 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) in LegalizeStoreOps()
546 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, in LegalizeStoreOps()
547 ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
550 // If not storing a power-of-2 number of bits, expand as two stores. in LegalizeStoreOps()
557 unsigned ExtraWidth = StWidthBits - RoundWidth; in LegalizeStoreOps()
567 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) in LegalizeStoreOps()
569 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), in LegalizeStoreOps()
570 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
581 ST->getPointerInfo().getWithOffset(IncrementSize), in LegalizeStoreOps()
582 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
584 // Big endian - avoid unaligned stores. in LegalizeStoreOps()
585 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X in LegalizeStoreOps()
591 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT, in LegalizeStoreOps()
592 ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
600 ST->getPointerInfo().getWithOffset(IncrementSize), in LegalizeStoreOps()
601 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
608 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { in LegalizeStoreOps()
611 EVT MemVT = ST->getMemoryVT(); in LegalizeStoreOps()
615 *ST->getMemOperand())) { in LegalizeStoreOps()
633 // TRUNCSTORE:i16 i32 -> STORE i16 in LegalizeStoreOps()
636 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), in LegalizeStoreOps()
637 ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
639 // The in-memory type isn't legal. Truncate to the type it would promote in LegalizeStoreOps()
645 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT, in LegalizeStoreOps()
646 ST->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeStoreOps()
657 SDValue Chain = LD->getChain(); // The chain. in LegalizeLoadOps()
658 SDValue Ptr = LD->getBasePtr(); // The base pointer. in LegalizeLoadOps()
662 ISD::LoadExtType ExtType = LD->getExtensionType(); in LegalizeLoadOps()
664 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n"); in LegalizeLoadOps()
665 MVT VT = Node->getSimpleValueType(0); in LegalizeLoadOps()
669 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { in LegalizeLoadOps()
672 EVT MemVT = LD->getMemoryVT(); in LegalizeLoadOps()
677 *LD->getMemOperand())) { in LegalizeLoadOps()
690 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in LegalizeLoadOps()
694 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); in LegalizeLoadOps()
705 UpdatedNodes->insert(RVal.getNode()); in LegalizeLoadOps()
706 UpdatedNodes->insert(RChain.getNode()); in LegalizeLoadOps()
714 EVT SrcVT = LD->getMemoryVT(); in LegalizeLoadOps()
716 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); in LegalizeLoadOps()
717 AAMDNodes AAInfo = LD->getAAInfo(); in LegalizeLoadOps()
728 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == in LegalizeLoadOps()
730 // Promote to a byte-sized load if not loading an integral number of in LegalizeLoadOps()
731 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. in LegalizeLoadOps()
742 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), in LegalizeLoadOps()
743 Chain, Ptr, LD->getPointerInfo(), NVT, in LegalizeLoadOps()
744 LD->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeLoadOps()
754 // All the top bits are guaranteed to be zero - inform the optimizers. in LegalizeLoadOps()
762 // If not loading a power-of-2 number of bits, expand as two loads. in LegalizeLoadOps()
769 unsigned ExtraWidth = SrcWidthBits - RoundWidth; in LegalizeLoadOps()
780 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) in LegalizeLoadOps()
782 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
783 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), in LegalizeLoadOps()
790 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
791 LD->getPointerInfo().getWithOffset(IncrementSize), in LegalizeLoadOps()
792 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeLoadOps()
806 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
808 // Big endian - avoid unaligned loads. in LegalizeLoadOps()
809 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 in LegalizeLoadOps()
811 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
812 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(), in LegalizeLoadOps()
819 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
820 LD->getPointerInfo().getWithOffset(IncrementSize), in LegalizeLoadOps()
821 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo); in LegalizeLoadOps()
835 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
841 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), in LegalizeLoadOps()
859 EVT MemVT = LD->getMemoryVT(); in LegalizeLoadOps()
862 *LD->getMemOperand())) { in LegalizeLoadOps()
869 EVT DestVT = Node->getValueType(0); in LegalizeLoadOps()
877 // If we are loading a legal type, this is a non-extload followed by a in LegalizeLoadOps()
883 SrcVT, LD->getMemOperand()); in LegalizeLoadOps()
886 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); in LegalizeLoadOps()
892 // normal undefined upper bits behavior to allow using an in-reg extend in LegalizeLoadOps()
894 // from-integer conversion. in LegalizeLoadOps()
902 Ptr, ISrcVT, LD->getMemOperand()); in LegalizeLoadOps()
914 // FIXME: This does not work for vectors on most targets. Sign- in LegalizeLoadOps()
915 // and zero-extend operations are currently folded into extending in LegalizeLoadOps()
923 Node->getValueType(0), in LegalizeLoadOps()
925 LD->getMemOperand()); in LegalizeLoadOps()
947 UpdatedNodes->insert(Value.getNode()); in LegalizeLoadOps()
948 UpdatedNodes->insert(Chain.getNode()); in LegalizeLoadOps()
956 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); in LegalizeOp()
959 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
960 Node->getOpcode() == ISD::Register) in LegalizeOp()
964 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) in LegalizeOp()
965 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == in LegalizeOp()
969 for (const SDValue &Op : Node->op_values()) in LegalizeOp()
980 switch (Node->getOpcode()) { in LegalizeOp()
985 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
988 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
989 Node->getValueType(0)); in LegalizeOp()
992 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
993 Node->getValueType(0)); in LegalizeOp()
995 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
999 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1000 Node->getOperand(1).getValueType()); in LegalizeOp()
1011 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1012 Node->getOperand(0).getValueType()); in LegalizeOp()
1022 // These pseudo-ops are the same as the other STRICT_ ops except in LegalizeOp()
1024 // instead of the output type. in LegalizeOp()
1025 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1026 Node->getOperand(1).getValueType()); in LegalizeOp()
1029 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); in LegalizeOp()
1030 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); in LegalizeOp()
1034 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1035 Node->getOperand(1).getValueType()); in LegalizeOp()
1044 unsigned Opc = Node->getOpcode(); in LegalizeOp()
1055 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); in LegalizeOp()
1057 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); in LegalizeOp()
1060 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1061 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1062 Node->getValueType(0)); in LegalizeOp()
1064 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
1078 // special case should be done as part of making LegalizeDAG non-recursive. in LegalizeOp()
1092 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1103 // they should actually be custom-lowered. in LegalizeOp()
1104 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1111 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1117 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); in LegalizeOp()
1127 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1131 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1132 Node->getOperand(0)); in LegalizeOp()
1139 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1143 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1144 Node->getOperand(0)); in LegalizeOp()
1160 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1170 unsigned Scale = Node->getConstantOperandVal(2); in LegalizeOp()
1171 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), in LegalizeOp()
1172 Node->getValueType(0), Scale); in LegalizeOp()
1176 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1177 cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1180 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1181 cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1185 Node->getOpcode(), in LegalizeOp()
1186 cast<VPScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1190 Node->getOpcode(), in LegalizeOp()
1191 cast<VPStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1195 Node->getOpcode(), in LegalizeOp()
1196 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1215 Node->getOpcode(), Node->getOperand(0).getValueType()); in LegalizeOp()
1237 Node->getOpcode(), Node->getOperand(1).getValueType()); in LegalizeOp()
1241 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1242 Node->getOperand(0).getValueType()); in LegalizeOp()
1245 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1248 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1255 switch (Node->getOpcode()) { in LegalizeOp()
1264 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1265 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1285 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1286 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1287 SDValue Op2 = Node->getOperand(2); in LegalizeOp()
1316 if (Node->getNumValues() == 1) { in LegalizeOp()
1319 assert((Res.getValueType() == Node->getValueType(0) || in LegalizeOp()
1320 Node->getValueType(0) == MVT::Glue) && in LegalizeOp()
1329 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { in LegalizeOp()
1332 assert((Res->getValueType(i) == Node->getValueType(i) || in LegalizeOp()
1333 Node->getValueType(i) == MVT::Glue) && in LegalizeOp()
1356 switch (Node->getOpcode()) { in LegalizeOp()
1360 Node->dump( &DAG); in LegalizeOp()
1393 for (SDNode *User : Vec.getNode()->uses()) { in ExpandExtractFromVectorThroughStack()
1395 if (ST->isIndexed() || ST->isTruncatingStore() || in ExpandExtractFromVectorThroughStack()
1396 ST->getValue() != Vec) in ExpandExtractFromVectorThroughStack()
1401 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) in ExpandExtractFromVectorThroughStack()
1410 ST->hasPredecessor(Op.getNode())) in ExpandExtractFromVectorThroughStack()
1413 StackPtr = ST->getBasePtr(); in ExpandExtractFromVectorThroughStack()
1431 std::min(cast<StoreSDNode>(Ch)->getAlign(), in ExpandExtractFromVectorThroughStack()
1452 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), in ExpandExtractFromVectorThroughStack()
1453 NewLoad->op_end()); in ExpandExtractFromVectorThroughStack()
1461 assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); in ExpandInsertToVectorThroughStack()
1472 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in ExpandInsertToVectorThroughStack()
1507 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1508 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1515 EVT VT = Node->getValueType(0); in ExpandVectorBuildThroughStack()
1517 : Node->getOperand(0).getValueType(); in ExpandVectorBuildThroughStack()
1520 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); in ExpandVectorBuildThroughStack()
1532 MemVT.bitsLT(Node->getOperand(0).getValueType()); in ExpandVectorBuildThroughStack()
1535 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { in ExpandVectorBuildThroughStack()
1537 if (Node->getOperand(i).isUndef()) continue; in ExpandVectorBuildThroughStack()
1546 Node->getOperand(i), Idx, in ExpandVectorBuildThroughStack()
1549 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), in ExpandVectorBuildThroughStack()
1563 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1565 /// holding all bits of the floating-point value.
1577 State.SignBit = NumBits - 1; in getSignAsIntValue()
1586 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in getSignAsIntValue()
1602 unsigned ByteOffset = (NumBits / 8) - 1; in getSignAsIntValue()
1617 /// and cast the result back to a floating-point type.
1633 SDValue Mag = Node->getOperand(0); in ExpandFCOPYSIGN()
1634 SDValue Sign = Node->getOperand(1); in ExpandFCOPYSIGN()
1645 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) in ExpandFCOPYSIGN()
1665 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; in ExpandFCOPYSIGN()
1676 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); in ExpandFCOPYSIGN()
1698 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); in ExpandFNEG()
1712 SDValue Value = Node->getOperand(0); in ExpandFABS()
1737 EVT VT = Node->getValueType(0); in ExpandDYNAMIC_STACKALLOC()
1740 SDValue Tmp3 = Node->getOperand(2); in ExpandDYNAMIC_STACKALLOC()
1750 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue(); in ExpandDYNAMIC_STACKALLOC()
1753 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ? in ExpandDYNAMIC_STACKALLOC()
1756 Align StackAlign = TFL->getStackAlign(); in ExpandDYNAMIC_STACKALLOC()
1760 DAG.getConstant(-Alignment.value(), dl, VT)); in ExpandDYNAMIC_STACKALLOC()
1761 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain in ExpandDYNAMIC_STACKALLOC()
1798 int SPFI = StackPtrFI->getIndex(); in EmitStackConvert()
1827 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); in ExpandSCALAR_TO_VECTOR()
1830 int SPFI = StackPtrFI->getIndex(); in ExpandSCALAR_TO_VECTOR()
1833 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, in ExpandSCALAR_TO_VECTOR()
1835 Node->getValueType(0).getVectorElementType()); in ExpandSCALAR_TO_VECTOR()
1837 Node->getValueType(0), dl, Ch, StackPtr, in ExpandSCALAR_TO_VECTOR()
1844 unsigned NumElems = Node->getNumOperands(); in ExpandBVWithShuffles()
1846 EVT VT = Node->getValueType(0); in ExpandBVWithShuffles()
1859 SDValue V = Node->getOperand(i); in ExpandBVWithShuffles()
1874 SmallVector<int, 16> ShuffleVec(NumElems, -1); in ExpandBVWithShuffles()
1920 SmallVector<int, 16> ShuffleVec(NumElems, -1); in ExpandBVWithShuffles()
1938 unsigned NumElems = Node->getNumOperands(); in ExpandBUILD_VECTOR()
1941 EVT VT = Node->getValueType(0); in ExpandBUILD_VECTOR()
1942 EVT OpVT = Node->getOperand(0).getValueType(); in ExpandBUILD_VECTOR()
1945 // If the only non-undef value is the low element, turn this into a in ExpandBUILD_VECTOR()
1951 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
1973 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1980 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1981 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); in ExpandBUILD_VECTOR()
1983 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1985 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); in ExpandBUILD_VECTOR()
1990 const ConstantInt *CI = V->getConstantIntValue(); in ExpandBUILD_VECTOR()
1992 CI->getZExtValue())); in ExpandBUILD_VECTOR()
1995 assert(Node->getOperand(i).isUndef()); in ExpandBUILD_VECTOR()
2003 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); in ExpandBUILD_VECTOR()
2012 if (Node->getOperand(i).isUndef()) in ExpandBUILD_VECTOR()
2014 DefinedValues.insert(Node->getOperand(i)); in ExpandBUILD_VECTOR()
2019 SmallVector<int, 8> ShuffleVec(NumElems, -1); in ExpandBUILD_VECTOR()
2021 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
2026 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { in ExpandBUILD_VECTOR()
2051 EVT VT = Node->getValueType(0); in ExpandSPLAT_VECTOR()
2052 SDValue SplatVal = Node->getOperand(0); in ExpandSPLAT_VECTOR()
2059 // register, return the lo part and set the hi part to the by-reg argument in
2071 DAG.getContext()->emitError(Twine("no libcall available for ") + in ExpandLibCall()
2072 Node->getOperationName(&DAG)); in ExpandLibCall()
2075 EVT RetVT = Node->getValueType(0); in ExpandLibCall()
2081 // node which is being folded has a non-entry input chain. in ExpandLibCall()
2090 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()); in ExpandLibCall()
2121 for (const SDValue &Op : Node->op_values()) { in ExpandLibCall()
2137 EVT VT = Node->getValueType(0); in ExpandFrexpLibCall()
2138 EVT ExpVT = Node->getValueType(1); in ExpandFrexpLibCall()
2140 SDValue FPOp = Node->getOperand(0); in ExpandFrexpLibCall()
2162 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex(); in ExpandFrexpLibCall()
2181 if (Node->isStrictFPOpcode()) { in ExpandFPLibCall()
2182 EVT RetVT = Node->getValueType(0); in ExpandFPLibCall()
2183 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); in ExpandFPLibCall()
2189 Node->getOperand(0)); in ExpandFPLibCall()
2193 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP; in ExpandFPLibCall()
2207 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0), in ExpandFPLibCall()
2220 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandIntLibCall()
2240 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); in ExpandArgFPLibCall()
2251 unsigned Opcode = Node->getOpcode(); in ExpandDivRemLibCall()
2255 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandDivRemLibCall()
2269 EVT RetVT = Node->getValueType(0); in ExpandDivRemLibCall()
2274 for (const SDValue &Op : Node->op_values()) { in ExpandDivRemLibCall()
2287 Entry.Ty = PointerType::getUnqual(RetTy->getContext()); in ExpandDivRemLibCall()
2316 switch (Node->getSimpleValueType(0).SimpleTy) { in isSinCosLibcallAvailable()
2329 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2330 ? ISD::FCOS : ISD::FSIN; in useSinCos()
2332 SDValue Op0 = Node->getOperand(0); in useSinCos()
2333 for (const SDNode *User : Op0.getNode()->uses()) { in useSinCos()
2337 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) in useSinCos()
2348 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandSinCosLibCall()
2362 EVT RetVT = Node->getValueType(0); in ExpandSinCosLibCall()
2369 Entry.Node = Node->getOperand(0); in ExpandSinCosLibCall()
2378 Entry.Ty = PointerType::getUnqual(RetTy->getContext()); in ExpandSinCosLibCall()
2386 Entry.Ty = PointerType::getUnqual(RetTy->getContext()); in ExpandSinCosLibCall()
2410 EVT VT = Node->getValueType(0); in expandLdexp()
2411 SDValue X = Node->getOperand(0); in expandLdexp()
2412 SDValue N = Node->getOperand(1); in expandLdexp()
2418 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO in expandLdexp()
2474 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT); in expandLdexp()
2475 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT); in expandLdexp()
2511 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl); in expandLdexp()
2522 SDValue Val = Node->getOperand(0); in expandFrexp()
2524 EVT ExpVT = Node->getValueType(1); in expandFrexp()
2537 // extracted_exp = (bitcast value to uint) >> precision - 1 in expandFrexp()
2543 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp in expandFrexp()
2563 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1); in expandFrexp()
2564 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit in expandFrexp()
2610 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl); in expandFrexp()
2616 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT); in expandFrexp()
2645 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2646 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2647 EVT DestVT = Node->getValueType(0); in ExpandLegalINT_TO_FP()
2649 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; in ExpandLegalINT_TO_FP()
2650 SDValue Op0 = Node->getOperand(OpNo); in ExpandLegalINT_TO_FP()
2653 // TODO: Should any fast-math-flags be set for the created nodes? in ExpandLegalINT_TO_FP()
2657 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2660 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double " in ExpandLegalINT_TO_FP()
2703 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2705 {Node->getOperand(0), Load, Bias}); in ExpandLegalINT_TO_FP()
2732 // should be valid for i32->f32 as well. in ExpandLegalINT_TO_FP()
2743 // pseudo-op, or, even better, for whole-function isel. in ExpandLegalINT_TO_FP()
2757 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2762 { Node->getOperand(0), InCvt }); in ExpandLegalINT_TO_FP()
2770 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); in ExpandLegalINT_TO_FP()
2771 Fast->setFlags(Flags); in ExpandLegalINT_TO_FP()
2773 Slow->setFlags(Flags); in ExpandLegalINT_TO_FP()
2785 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2790 // size of DestVT is >= than the number of bits in SrcVT -1. in ExpandLegalINT_TO_FP()
2792 SrcVT.getSizeInBits() - 1 && in ExpandLegalINT_TO_FP()
2796 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2798 { Node->getOperand(0), Op0 }); in ExpandLegalINT_TO_FP()
2828 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); in ExpandLegalINT_TO_FP()
2847 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2864 bool IsStrict = N->isStrictFPOpcode(); in PromoteLegalINT_TO_FP()
2865 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || in PromoteLegalINT_TO_FP()
2866 N->getOpcode() == ISD::STRICT_SINT_TO_FP; in PromoteLegalINT_TO_FP()
2867 EVT DestVT = N->getValueType(0); in PromoteLegalINT_TO_FP()
2868 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); in PromoteLegalINT_TO_FP()
2904 {N->getOperand(0), in PromoteLegalINT_TO_FP()
2925 bool IsStrict = N->isStrictFPOpcode(); in PromoteLegalFP_TO_INT()
2926 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || in PromoteLegalFP_TO_INT()
2927 N->getOpcode() == ISD::STRICT_FP_TO_SINT; in PromoteLegalFP_TO_INT()
2928 EVT DestVT = N->getValueType(0); in PromoteLegalFP_TO_INT()
2929 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0); in PromoteLegalFP_TO_INT()
2958 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp); in PromoteLegalFP_TO_INT()
2975 unsigned Opcode = Node->getOpcode(); in PromoteLegalFP_TO_INT_SAT()
2978 EVT NewOutTy = Node->getValueType(0); in PromoteLegalFP_TO_INT_SAT()
2989 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), in PromoteLegalFP_TO_INT_SAT()
2990 Node->getOperand(1)); in PromoteLegalFP_TO_INT_SAT()
2991 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
3008 DAG.getConstant(1ULL << (--i), dl, ShVT)); in ExpandPARITY()
3017 MVT VecVT = Node->getOperand(1).getSimpleValueType(); in PromoteReduction()
3018 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT); in PromoteReduction()
3019 MVT ScalarVT = Node->getSimpleValueType(0); in PromoteReduction()
3023 SmallVector<SDValue, 4> Operands(Node->getNumOperands()); in PromoteReduction()
3027 assert(Node->getOperand(0).getValueType().isFloatingPoint() && in PromoteReduction()
3030 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
3032 for (unsigned j = 1; j != Node->getNumOperands(); ++j) in PromoteReduction()
3033 if (Node->getOperand(j).getValueType().isVector() && in PromoteReduction()
3034 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteReduction()
3035 ISD::getVPMaskIdx(Node->getOpcode()) == j)) { // Skip mask operand. in PromoteReduction()
3038 assert(Node->getOperand(j).getValueType().isFloatingPoint() && in PromoteReduction()
3041 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
3043 Operands[j] = Node->getOperand(j); // Skip VL operand. in PromoteReduction()
3046 SDValue Res = DAG.getNode(Node->getOpcode(), DL, NewScalarVT, Operands, in PromoteReduction()
3047 Node->getFlags()); in PromoteReduction()
3060 switch (Node->getOpcode()) { in ExpandNode()
3100 Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); in ExpandNode()
3105 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3108 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, in ExpandNode()
3123 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); in ExpandNode()
3124 Results.push_back(Node->getOperand(0)); in ExpandNode()
3133 Results.push_back(Node->getOperand(0)); in ExpandNode()
3139 Results.append(Node->getNumValues() - 1, in ExpandNode()
3140 DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3141 Results.push_back(Node->getOperand(0)); in ExpandNode()
3147 Results.push_back(Node->getOperand(0)); in ExpandNode()
3151 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); in ExpandNode()
3152 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3154 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3155 Node->getOperand(0), Node->getOperand(1), Zero, Zero, in ExpandNode()
3156 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
3164 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), in ExpandNode()
3165 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1), in ExpandNode()
3166 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
3174 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3176 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3177 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), in ExpandNode()
3178 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); in ExpandNode()
3182 SDValue RHS = Node->getOperand(1); in ExpandNode()
3184 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); in ExpandNode()
3185 EVT OuterType = Node->getValueType(0); in ExpandNode()
3191 Node->getOperand(2), DAG.getValueType(AtomicType)); in ExpandNode()
3197 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3202 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3209 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
3218 EVT VT = Node->getValueType(0); in ExpandNode()
3219 SDValue RHS = Node->getOperand(2); in ExpandNode()
3221 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG && in ExpandNode()
3222 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT()) in ExpandNode()
3223 RHS = RHS->getOperand(0); in ExpandNode()
3226 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(), in ExpandNode()
3227 Node->getOperand(0), Node->getOperand(1), in ExpandNode()
3228 NewRHS, AN->getMemOperand()); in ExpandNode()
3237 for (unsigned i = 0; i < Node->getNumValues(); i++) in ExpandNode()
3238 Results.push_back(Node->getOperand(i)); in ExpandNode()
3241 EVT VT = Node->getValueType(0); in ExpandNode()
3257 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3258 Node->getValueType(0)) in ExpandNode()
3263 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), in ExpandNode()
3264 Node->getValueType(0), dl, in ExpandNode()
3265 Node->getOperand(0)))) { in ExpandNode()
3280 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), in ExpandNode()
3281 Node->getValueType(0), dl))) in ExpandNode()
3291 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3292 Node->getValueType(0)) in ExpandNode()
3298 Node->getOperand(1), Node->getOperand(1).getValueType(), in ExpandNode()
3299 Node->getValueType(0), dl, Node->getOperand(0)))) { in ExpandNode()
3306 SDValue Op = Node->getOperand(0); in ExpandNode()
3308 EVT DstVT = Node->getValueType(0); in ExpandNode()
3323 SDValue Op = Node->getOperand(0); in ExpandNode()
3335 // Add fp_extend in case the output is bigger than f32. in ExpandNode()
3336 if (Node->getValueType(0) != MVT::f32) in ExpandNode()
3337 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op); in ExpandNode()
3342 SDValue Op = Node->getOperand(0); in ExpandNode()
3348 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags()); in ExpandNode()
3356 if (Node->getValueType(0) == MVT::bf16) { in ExpandNode()
3360 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0)); in ExpandNode()
3366 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); in ExpandNode()
3367 EVT VT = Node->getValueType(0); in ExpandNode()
3369 // An in-register sign-extend of a boolean is a negation: in ExpandNode()
3370 // 'true' (1) sign-extended is -1. in ExpandNode()
3371 // 'false' (0) sign-extended is 0. in ExpandNode()
3378 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
3388 unsigned BitsDiff = VT.getScalarSizeInBits() - in ExpandNode()
3391 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
3392 Node->getOperand(0), ShiftCst); in ExpandNode()
3393 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
3401 if (Node->isStrictFPOpcode()) in ExpandNode()
3410 if (Node->isStrictFPOpcode()) in ExpandNode()
3451 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar()) in ExpandNode()
3453 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3454 Node->getOperand(0)); in ExpandNode()
3476 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in ExpandNode()
3478 EVT VT = Node->getValueType(0); in ExpandNode()
3480 SDValue Op0 = Node->getOperand(0); in ExpandNode()
3481 SDValue Op1 = Node->getOperand(1); in ExpandNode()
3491 // cast operands to v8i32 and re-build the mask. in ExpandNode()
3539 DAG.getVectorIdxConstant(Idx - NumElems, dl))); in ExpandNode()
3544 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3553 EVT OpTy = Node->getOperand(0).getValueType(); in ExpandNode()
3554 if (Node->getConstantOperandVal(1)) { in ExpandNode()
3555 // 1 -> Hi in ExpandNode()
3556 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3559 Node->getOperand(0).getValueType(), in ExpandNode()
3561 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3563 // 0 -> Lo in ExpandNode()
3564 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3565 Node->getOperand(0)); in ExpandNode()
3574 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, in ExpandNode()
3575 Node->getValueType(0))); in ExpandNode()
3578 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ExpandNode()
3579 Results.push_back(Node->getOperand(0)); in ExpandNode()
3586 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, in ExpandNode()
3587 Node->getOperand(1))); in ExpandNode()
3589 Results.push_back(Node->getOperand(0)); in ExpandNode()
3593 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3606 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1)); in ExpandNode()
3608 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0), in ExpandNode()
3609 Test, Node->getFlags(), SDLoc(Node), DAG)) in ExpandNode()
3617 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B in ExpandNode()
3619 switch (Node->getOpcode()) { in ExpandNode()
3626 Tmp1 = Node->getOperand(0); in ExpandNode()
3627 Tmp2 = Node->getOperand(1); in ExpandNode()
3644 case ISD::FSIN: in ExpandNode()
3646 EVT VT = Node->getValueType(0); in ExpandNode()
3647 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / in ExpandNode()
3653 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3654 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3662 EVT VT = Node->getValueType(0); in ExpandNode()
3671 if (Node->getOpcode() == ISD::STRICT_FLDEXP) in ExpandNode()
3678 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0)); in ExpandNode()
3694 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3696 // the result. Since "f16 -> f32" is much more commonly available, give in ExpandNode()
3699 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3701 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3706 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3708 // the result. Since "f16 -> f32" is much more commonly available, give in ExpandNode()
3710 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other}, in ExpandNode()
3711 {Node->getOperand(0), Node->getOperand(1)}); in ExpandNode()
3713 {Node->getValueType(0), MVT::Other}, in ExpandNode()
3722 SDValue Op = Node->getOperand(0); in ExpandNode()
3727 // a float-half conversion. in ExpandNode()
3732 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3740 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), in ExpandNode()
3751 EVT VT = Node->getValueType(0); in ExpandNode()
3754 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
3755 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3756 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3762 EVT VT = Node->getValueType(0); in ExpandNode()
3766 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT); in ExpandNode()
3768 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3778 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3780 EVT VT = Node->getValueType(0); in ExpandNode()
3783 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), in ExpandNode()
3784 Node->getOperand(1)); in ExpandNode()
3792 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3793 EVT VT = Node->getValueType(0); in ExpandNode()
3796 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), in ExpandNode()
3797 Node->getOperand(1)); in ExpandNode()
3803 SDValue LHS = Node->getOperand(0); in ExpandNode()
3804 SDValue RHS = Node->getOperand(1); in ExpandNode()
3807 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3818 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, in ExpandNode()
3835 EVT VT = Node->getValueType(0); in ExpandNode()
3837 // See if multiply or divide can be lowered using two-result operations. in ExpandNode()
3857 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), in ExpandNode()
3858 Node->getOperand(1))); in ExpandNode()
3914 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), in ExpandNode()
3915 Node->getOperand(0), in ExpandNode()
3916 Node->getOperand(1), in ExpandNode()
3917 Node->getConstantOperandVal(2), in ExpandNode()
3925 // <= 128 (which is the case for all of the default Embedded-C types), in ExpandNode()
3932 SDValue LHS = Node->getOperand(0); in ExpandNode()
3933 SDValue RHS = Node->getOperand(1); in ExpandNode()
3934 SDValue Carry = Node->getOperand(2); in ExpandNode()
3936 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY; in ExpandNode()
3944 EVT CarryType = Node->getValueType(1); in ExpandNode()
3945 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); in ExpandNode()
3999 EVT PairTy = Node->getValueType(0); in ExpandNode()
4000 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
4001 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
4010 Tmp1 = Node->getOperand(0); in ExpandNode()
4011 Tmp2 = Node->getOperand(1); in ExpandNode()
4012 Tmp3 = Node->getOperand(2); in ExpandNode()
4016 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); in ExpandNode()
4022 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4026 SDValue Chain = Node->getOperand(0); in ExpandNode()
4027 SDValue Table = Node->getOperand(1); in ExpandNode()
4028 SDValue Index = Node->getOperand(2); in ExpandNode()
4029 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex(); in ExpandNode()
4035 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); in ExpandNode()
4037 // For power-of-two jumptable entry sizes convert multiplication to a shift. in ExpandNode()
4071 Tmp1 = Node->getOperand(0); in ExpandNode()
4072 Tmp2 = Node->getOperand(1); in ExpandNode()
4078 Node->getOperand(2)); in ExpandNode()
4090 Node->getOperand(2)); in ExpandNode()
4098 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandNode()
4099 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandNode()
4100 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4101 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4102 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ExpandNode()
4104 Tmp1 = Node->getOperand(0 + Offset); in ExpandNode()
4105 Tmp2 = Node->getOperand(1 + Offset); in ExpandNode()
4106 Tmp3 = Node->getOperand(2 + Offset); in ExpandNode()
4109 Mask = Node->getOperand(3 + Offset); in ExpandNode()
4110 EVL = Node->getOperand(4 + Offset); in ExpandNode()
4113 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl, in ExpandNode()
4121 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(), in ExpandNode()
4122 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags()); in ExpandNode()
4125 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), in ExpandNode()
4126 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags()); in ExpandNode()
4128 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1, in ExpandNode()
4129 Tmp2, Tmp3, Node->getFlags()); in ExpandNode()
4137 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); in ExpandNode()
4140 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0)); in ExpandNode()
4157 EVT VT = Node->getValueType(0); in ExpandNode()
4162 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4168 Tmp1 = Node->getOperand(0); // LHS in ExpandNode()
4169 Tmp2 = Node->getOperand(1); // RHS in ExpandNode()
4170 Tmp3 = Node->getOperand(2); // True in ExpandNode()
4171 Tmp4 = Node->getOperand(3); // False in ExpandNode()
4172 EVT VT = Node->getValueType(0); in ExpandNode()
4174 SDValue CC = Node->getOperand(4); in ExpandNode()
4175 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); in ExpandNode()
4185 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
4187 DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4, Node->getFlags())); in ExpandNode()
4201 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4211 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4230 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
4235 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4238 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4246 Tmp1 = Node->getOperand(0); // Chain in ExpandNode()
4247 Tmp2 = Node->getOperand(2); // LHS in ExpandNode()
4248 Tmp3 = Node->getOperand(3); // RHS in ExpandNode()
4249 Tmp4 = Node->getOperand(1); // CC in ExpandNode()
4262 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4263 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
4267 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
4268 Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
4283 EVT VT = Node->getValueType(0); in ExpandNode()
4284 assert(VT.isVector() && "Unable to legalize non-vector shift"); in ExpandNode()
4292 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4295 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4296 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, in ExpandNode()
4300 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); in ExpandNode()
4326 // The default expansion of llvm.clear_cache is simply a no-op for those in ExpandNode()
4328 Results.push_back(Node->getOperand(0)); in ExpandNode()
4345 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { in ExpandNode()
4346 // FIXME: We were asked to expand a strict floating-point operation, in ExpandNode()
4348 // the "strict" properties. For now, we just fall back to the non-strict in ExpandNode()
4351 switch (Node->getOpcode()) { in ExpandNode()
4353 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
4354 Node->getValueType(0)) in ExpandNode()
4360 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
4363 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
4366 EVT VT = Node->getValueType(0); in ExpandNode()
4367 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
4368 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
4369 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
4370 {Node->getOperand(0), Node->getOperand(1), Neg}, in ExpandNode()
4385 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
4386 Node->getOperand(1).getValueType()) in ExpandNode()
4409 unsigned Opc = Node->getOpcode(); in ConvertNodeToLibcall()
4418 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
4446 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); in ConvertNodeToLibcall()
4447 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering(); in ConvertNodeToLibcall()
4449 EVT RetVT = Node->getValueType(0); in ConvertNodeToLibcall()
4454 Ops.append(Node->op_begin() + 2, Node->op_end()); in ConvertNodeToLibcall()
4455 Ops.push_back(Node->getOperand(1)); in ConvertNodeToLibcall()
4462 Ops.append(Node->op_begin() + 1, Node->op_end()); in ConvertNodeToLibcall()
4467 Node->getOperand(0)); in ConvertNodeToLibcall()
4477 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
4489 SDValue InputChain = Node->getOperand(0); in ConvertNodeToLibcall()
4490 SDValue StartVal = Node->getOperand(1); in ConvertNodeToLibcall()
4491 SDValue EndVal = Node->getOperand(2); in ConvertNodeToLibcall()
4524 case ISD::FSIN: in ConvertNodeToLibcall()
4663 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0)); in ConvertNodeToLibcall()
4667 if (Node->isStrictFPOpcode()) { in ConvertNodeToLibcall()
4670 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4671 {Node->getOperand(0), Node->getOperand(2)}); in ConvertNodeToLibcall()
4674 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4675 {Exponent.getValue(1), Node->getOperand(1), Exponent}); in ConvertNodeToLibcall()
4680 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4681 Node->getOperand(1)); in ConvertNodeToLibcall()
4683 Node->getValueType(0), in ConvertNodeToLibcall()
4684 Node->getOperand(0), Exponent)); in ConvertNodeToLibcall()
4688 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; in ConvertNodeToLibcall()
4691 Node->getOperand(1 + Offset).getValueType().getSizeInBits(); in ConvertNodeToLibcall()
4695 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); in ConvertNodeToLibcall()
4696 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ConvertNodeToLibcall()
4766 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4771 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4774 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1), in ConvertNodeToLibcall()
4775 CallOptions, SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4781 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4784 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, in ConvertNodeToLibcall()
4785 SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4793 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); in ConvertNodeToLibcall()
4800 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16); in ConvertNodeToLibcall()
4809 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP in ConvertNodeToLibcall()
4810 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4811 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4812 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4813 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); in ConvertNodeToLibcall()
4814 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4819 // have i1 -> fp conversions. So, it needs to be promoted to a larger type, in ConvertNodeToLibcall()
4820 // eg: i13 -> fp. Then, look for an appropriate libcall. in ConvertNodeToLibcall()
4833 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4836 NVT, Node->getOperand(IsStrict ? 1 : 0)); in ConvertNodeToLibcall()
4850 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT. in ConvertNodeToLibcall()
4851 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4852 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4853 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4855 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4857 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4862 // have fp -> i1 conversions. So, it needs to be promoted to a larger type, in ConvertNodeToLibcall()
4863 // eg: fp -> i32. Then, look for an appropriate libcall. in ConvertNodeToLibcall()
4876 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4895 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4896 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4897 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4898 EVT VT = Node->getValueType(0); in ConvertNodeToLibcall()
4899 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() && in ConvertNodeToLibcall()
4915 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), in ConvertNodeToLibcall()
4916 Node->getValueType(0)), in ConvertNodeToLibcall()
4924 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16) in ConvertNodeToLibcall()
4925 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16); in ConvertNodeToLibcall()
4926 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16) in ConvertNodeToLibcall()
4927 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::bf16); in ConvertNodeToLibcall()
4929 LC = RTLIB::getFPEXT(Node->getOperand(1).getValueType(), in ConvertNodeToLibcall()
4930 Node->getValueType(0)); in ConvertNodeToLibcall()
4936 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), in ConvertNodeToLibcall()
4937 CallOptions, SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4984 switch (Node->getSimpleValueType(0).SimpleTy) { in ConvertNodeToLibcall()
5000 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc. in ConvertNodeToLibcall()
5001 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl); in ConvertNodeToLibcall()
5002 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
5008 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
5009 SDValue EnvPtr = Node->getOperand(1); in ConvertNodeToLibcall()
5015 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
5016 SDValue EnvPtr = Node->getOperand(1); in ConvertNodeToLibcall()
5024 EVT ModeVT = Node->getValueType(0); in ConvertNodeToLibcall()
5026 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in ConvertNodeToLibcall()
5028 Node->getOperand(0), dl); in ConvertNodeToLibcall()
5039 SDValue Mode = Node->getOperand(1); in ConvertNodeToLibcall()
5042 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in ConvertNodeToLibcall()
5044 Node->getOperand(0), dl, Mode, StackPtr, in ConvertNodeToLibcall()
5052 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the in ConvertNodeToLibcall()
5056 SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy); in ConvertNodeToLibcall()
5058 Node->getOperand(0), dl)); in ConvertNodeToLibcall()
5086 MVT OVT = Node->getSimpleValueType(0); in PromoteNode()
5087 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
5088 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
5089 Node->getOpcode() == ISD::SETCC || in PromoteNode()
5090 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
5091 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
5092 OVT = Node->getOperand(0).getSimpleValueType(); in PromoteNode()
5094 if (Node->getOpcode() == ISD::ATOMIC_STORE || in PromoteNode()
5095 Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
5096 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
5097 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
5098 Node->getOpcode() == ISD::STRICT_FSETCCS || in PromoteNode()
5099 Node->getOpcode() == ISD::VP_REDUCE_FADD || in PromoteNode()
5100 Node->getOpcode() == ISD::VP_REDUCE_FMUL || in PromoteNode()
5101 Node->getOpcode() == ISD::VP_REDUCE_FMAX || in PromoteNode()
5102 Node->getOpcode() == ISD::VP_REDUCE_FMIN || in PromoteNode()
5103 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM || in PromoteNode()
5104 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM || in PromoteNode()
5105 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD) in PromoteNode()
5106 OVT = Node->getOperand(1).getSimpleValueType(); in PromoteNode()
5107 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
5108 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
5109 OVT = Node->getOperand(2).getSimpleValueType(); in PromoteNode()
5110 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); in PromoteNode()
5113 switch (Node->getOpcode()) { in PromoteNode()
5119 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
5120 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
5121 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5123 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5125 unsigned NewOpc = Node->getOpcode(); in PromoteNode()
5140 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) in PromoteNode()
5142 DAG.getConstant(NVT.getSizeInBits() - in PromoteNode()
5154 DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5156 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT)) in PromoteNode()
5158 NVT.getSizeInBits() - OVT.getSizeInBits(), NVT, dl); in PromoteNode()
5163 auto CTLZResult = DAG.getNode(Node->getOpcode(), dl, NVT, LeftShiftResult); in PromoteNode()
5169 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); in PromoteNode()
5170 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5171 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5197 SDValue Chain = Node->getOperand(0); // Get the chain. in PromoteNode()
5198 SDValue Ptr = Node->getOperand(1); // Get the pointer. in PromoteNode()
5210 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), in PromoteNode()
5211 Node->getConstantOperandVal(3)); in PromoteNode()
5216 // Modified the chain result - switch anything that used the old chain to in PromoteNode()
5221 UpdatedNodes->insert(Tmp2.getNode()); in PromoteNode()
5222 UpdatedNodes->insert(Chain.getNode()); in PromoteNode()
5246 switch (Node->getOpcode()) { in PromoteNode()
5271 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5272 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5274 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5281 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
5283 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5284 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5298 if (Node->getValueType(0).isVector() || in PromoteNode()
5299 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { in PromoteNode()
5302 } else if (Node->getValueType(0).isInteger()) { in PromoteNode()
5309 Tmp1 = Node->getOperand(0); in PromoteNode()
5311 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5312 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5315 Tmp1->setFlags(Node->getFlags()); in PromoteNode()
5317 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); in PromoteNode()
5319 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5325 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in PromoteNode()
5328 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
5329 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
5338 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5339 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5341 Node->getOperand(2)); in PromoteNode()
5346 SDValue Cond = Node->getOperand(4); in PromoteNode()
5347 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get(); in PromoteNode()
5349 MVT CVT = Node->getSimpleValueType(0); in PromoteNode()
5359 Tmp1 = Node->getOperand(0); in PromoteNode()
5360 Tmp2 = Node->getOperand(1); in PromoteNode()
5362 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5363 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5366 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5367 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5370 Node->getFlags()); in PromoteNode()
5387 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
5389 TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT)) in PromoteNode()
5394 if (Node->isStrictFPOpcode()) { in PromoteNode()
5395 SDValue InChain = Node->getOperand(0); in PromoteNode()
5397 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); in PromoteNode()
5399 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); in PromoteNode()
5402 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in PromoteNode()
5403 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, in PromoteNode()
5404 {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, in PromoteNode()
5405 Node->getFlags())); in PromoteNode()
5409 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5410 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5411 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5412 Tmp2, Node->getOperand(2), Node->getFlags())); in PromoteNode()
5419 cast<CondCodeSDNode>(Node->getOperand(1))->get(); in PromoteNode()
5422 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5423 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5424 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
5425 Node->getOperand(0), Node->getOperand(1), in PromoteNode()
5426 Tmp1, Tmp2, Node->getOperand(4))); in PromoteNode()
5439 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5440 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5441 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, in PromoteNode()
5442 Node->getFlags()); in PromoteNode()
5456 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5458 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
5461 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5469 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5470 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5471 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
5474 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), in PromoteNode()
5479 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5481 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
5483 {Node->getOperand(0), Node->getOperand(3)}); in PromoteNode()
5486 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5496 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5497 Tmp2 = Node->getOperand(1); in PromoteNode()
5498 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5504 // which is a no-op. Mark it as a TRUNCating FP_ROUND. in PromoteNode()
5505 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
5513 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5514 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5515 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)}); in PromoteNode()
5522 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5523 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1); in PromoteNode()
5541 case ISD::FSIN: in PromoteNode()
5558 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5559 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5587 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5588 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5612 for (const SDValue &Op : Node->op_values()) in PromoteNode()
5645 SDValue Idx = Node->getOperand(1); in PromoteNode()
5651 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5691 SDValue Val = Node->getOperand(1); in PromoteNode()
5692 SDValue Idx = Node->getOperand(2); in PromoteNode()
5699 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5729 SDValue Val = Node->getOperand(0); in PromoteNode()
5749 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); in PromoteNode()
5752 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && in PromoteNode()
5755 SDValue Op0 = AM->getBasePtr(); in PromoteNode()
5760 if (AM->getOpcode() == ISD::ATOMIC_STORE) in PromoteNode()
5763 SDValue NewAtomic = DAG.getAtomic(AM->getOpcode(), SL, NVT, AM->getChain(), in PromoteNode()
5764 Op0, Op1, AM->getMemOperand()); in PromoteNode()
5766 if (AM->getOpcode() != ISD::ATOMIC_STORE) { in PromoteNode()
5778 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && in PromoteNode()
5783 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand()); in PromoteNode()
5789 SDValue Scalar = Node->getOperand(0); in PromoteNode()
5794 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5799 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5846 --NI; in Legalize()
5849 if (N->use_empty() && N != getRoot().getNode()) { in Legalize()
5859 if (N->use_empty() && N != getRoot().getNode()) { in Legalize()