Lines Matching full:instruction
16 // instruction). This is performed by checking the real and imaginary components
25 // shufflevector instruction and intrinsics.
38 // transformation, and every Instruction contained within said nodes. It also
39 // holds onto a reference to the root Instruction, and the root node that should
44 // transform into a single complex instruction (ignoring vector splitting, which
47 // instruction in the order they appear in the IR.
232 SmallPtrSet<Instruction *, 16> FinalInstructions;
235 std::map<Instruction *, NodePtr> RootToNode;
238 SmallVector<Instruction *, 1> OrderedRoots;
246 /// ReductionInfo maps from %ReductionOp to %PHInode and Instruction
262 MapVector<Instruction *, std::pair<PHINode *, Instruction *>> ReductionInfo;
313 NodePtr identifyPartialMul(Instruction *Real, Instruction *Imag);
319 identifyNodeWithImplicitAdd(Instruction *I, Instruction *J,
329 NodePtr identifyAdd(Instruction *Real, Instruction *Imag);
330 NodePtr identifySymmetricOperation(Instruction *Real, Instruction *Imag);
367 NodePtr identifyReassocNodes(Instruction *I, Instruction *J);
369 NodePtr identifyRoot(Instruction *I);
374 /// * Using two shufflevectors with even indices for /pReal instruction and
378 NodePtr identifyDeinterleave(Instruction *Real, Instruction *Imag);
386 NodePtr identifyPHINode(Instruction *Real, Instruction *Imag);
390 NodePtr identifySelectNode(Instruction *Real, Instruction *Imag);
410 bool identifyNodes(Instruction *RootI);
419 /// Check that every instruction, from the roots to the leaves, has internal
423 /// Perform the actual replacement of the underlying instruction graph.
525 auto *I = cast<Instruction>(V); in getNegOperand()
526 if (I->getOpcode() == Instruction::FNeg) in getNegOperand()
550 Instruction *Real, Instruction *Imag, in identifyNodeWithImplicitAdd()
560 if ((Real->getOpcode() != Instruction::FMul && in identifyNodeWithImplicitAdd()
561 Real->getOpcode() != Instruction::Mul) || in identifyNodeWithImplicitAdd()
562 (Imag->getOpcode() != Instruction::FMul && in identifyNodeWithImplicitAdd()
563 Imag->getOpcode() != Instruction::Mul)) { in identifyNodeWithImplicitAdd()
565 dbgs() << " - Real or imaginary instruction is not fmul or mul\n"); in identifyNodeWithImplicitAdd()
652 ComplexDeinterleavingGraph::identifyPartialMul(Instruction *Real, in identifyPartialMul()
653 Instruction *Imag) { in identifyPartialMul()
658 return Op == Instruction::FAdd || Op == Instruction::Add; in identifyPartialMul()
661 return Op == Instruction::FSub || Op == Instruction::Sub; in identifyPartialMul()
685 Instruction *RealMulI = dyn_cast<Instruction>(Real->getOperand(1)); in identifyPartialMul()
689 Instruction *ImagMulI = dyn_cast<Instruction>(Imag->getOperand(1)); in identifyPartialMul()
694 LLVM_DEBUG(dbgs() << " - Mul instruction has multiple uses\n"); in identifyPartialMul()
733 auto *CRInst = dyn_cast<Instruction>(CR); in identifyPartialMul()
734 auto *CIInst = dyn_cast<Instruction>(CI); in identifyPartialMul()
770 ComplexDeinterleavingGraph::identifyAdd(Instruction *Real, Instruction *Imag) { in identifyAdd()
775 if ((Real->getOpcode() == Instruction::FSub && in identifyAdd()
776 Imag->getOpcode() == Instruction::FAdd) || in identifyAdd()
777 (Real->getOpcode() == Instruction::Sub && in identifyAdd()
778 Imag->getOpcode() == Instruction::Add)) in identifyAdd()
780 else if ((Real->getOpcode() == Instruction::FAdd && in identifyAdd()
781 Imag->getOpcode() == Instruction::FSub) || in identifyAdd()
782 (Real->getOpcode() == Instruction::Add && in identifyAdd()
783 Imag->getOpcode() == Instruction::Sub)) in identifyAdd()
790 auto *AR = dyn_cast<Instruction>(Real->getOperand(0)); in identifyAdd()
791 auto *BI = dyn_cast<Instruction>(Real->getOperand(1)); in identifyAdd()
792 auto *AI = dyn_cast<Instruction>(Imag->getOperand(0)); in identifyAdd()
793 auto *BR = dyn_cast<Instruction>(Imag->getOperand(1)); in identifyAdd()
819 static bool isInstructionPairAdd(Instruction *A, Instruction *B) { in isInstructionPairAdd()
823 return (OpcA == Instruction::FSub && OpcB == Instruction::FAdd) || in isInstructionPairAdd()
824 (OpcA == Instruction::FAdd && OpcB == Instruction::FSub) || in isInstructionPairAdd()
825 (OpcA == Instruction::Sub && OpcB == Instruction::Add) || in isInstructionPairAdd()
826 (OpcA == Instruction::Add && OpcB == Instruction::Sub); in isInstructionPairAdd()
829 static bool isInstructionPairMul(Instruction *A, Instruction *B) { in isInstructionPairMul()
836 static bool isInstructionPotentiallySymmetric(Instruction *I) { in isInstructionPotentiallySymmetric()
838 case Instruction::FAdd: in isInstructionPotentiallySymmetric()
839 case Instruction::FSub: in isInstructionPotentiallySymmetric()
840 case Instruction::FMul: in isInstructionPotentiallySymmetric()
841 case Instruction::FNeg: in isInstructionPotentiallySymmetric()
842 case Instruction::Add: in isInstructionPotentiallySymmetric()
843 case Instruction::Sub: in isInstructionPotentiallySymmetric()
844 case Instruction::Mul: in isInstructionPotentiallySymmetric()
852 ComplexDeinterleavingGraph::identifySymmetricOperation(Instruction *Real, in identifySymmetricOperation()
853 Instruction *Imag) { in identifySymmetricOperation()
909 auto *Real = dyn_cast<Instruction>(R); in identifyNode()
910 auto *Imag = dyn_cast<Instruction>(I); in identifyNode()
955 ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real, in identifyReassocNodes()
956 Instruction *Imag) { in identifyReassocNodes()
958 return Opcode == Instruction::FAdd || Opcode == Instruction::FSub || in identifyReassocNodes()
959 Opcode == Instruction::FNeg || Opcode == Instruction::Add || in identifyReassocNodes()
960 Opcode == Instruction::Sub; in identifyReassocNodes()
984 // Collect multiplications and addend instructions from the given instruction in identifyReassocNodes()
987 auto Collect = [&Flags](Instruction *Insn, std::vector<Product> &Muls, in identifyReassocNodes()
997 Instruction *I = dyn_cast<Instruction>(V); in identifyReassocNodes()
1003 // If an instruction has more than one user, it indicates that it either in identifyReassocNodes()
1015 case Instruction::FAdd: in identifyReassocNodes()
1016 case Instruction::Add: in identifyReassocNodes()
1020 case Instruction::FSub: in identifyReassocNodes()
1024 case Instruction::Sub: in identifyReassocNodes()
1032 case Instruction::FMul: in identifyReassocNodes()
1033 case Instruction::Mul: { in identifyReassocNodes()
1051 case Instruction::FNeg: in identifyReassocNodes()
1060 LLVM_DEBUG(dbgs() << "The instruction's fast math flags are " in identifyReassocNodes()
1164 // Map to store common instruction to node pointers in identifyMultiplications()
1365 TmpNode->Opcode = Instruction::FAdd; in identifyAdditions()
1368 TmpNode->Opcode = Instruction::Add; in identifyAdditions()
1375 TmpNode->Opcode = Instruction::FSub; in identifyAdditions()
1378 TmpNode->Opcode = Instruction::Sub; in identifyAdditions()
1422 bool ComplexDeinterleavingGraph::identifyNodes(Instruction *RootI) { in identifyNodes()
1423 // This potential root instruction might already have been recognized as in identifyNodes()
1425 // CompositeNode we should choose only one either Real or Imag instruction to in identifyNodes()
1426 // use as an anchor for generating complex instruction. in identifyNodes()
1434 auto *R = cast<Instruction>(RootNode->Real); in identifyNodes()
1435 auto *I = cast<Instruction>(RootNode->Imag); in identifyNodes()
1479 auto *ReductionOp = dyn_cast<Instruction>(PHI.getIncomingValueForBlock(B)); in collectPotentialReductions()
1483 // Check if final instruction is reduced outside of current block in collectPotentialReductions()
1484 Instruction *FinalReduction = nullptr; in collectPotentialReductions()
1490 FinalReduction = dyn_cast<Instruction>(U); in collectPotentialReductions()
1504 // If the initial value of PHINode is an Instruction, consider it a leaf in collectPotentialReductions()
1507 dyn_cast<Instruction>(PHI.getIncomingValueForBlock(Incoming))) in collectPotentialReductions()
1515 SmallVector<Instruction *> OperationInstruction; in identifyReductionNodes()
1568 SmallPtrSet<Instruction *, 16> AllInstructions; in checkNodes()
1569 SmallVector<Instruction *, 8> Worklist; in checkNodes()
1583 if (auto *OpI = dyn_cast<Instruction>(Op)) { in checkNodes()
1591 SmallVector<Instruction *, 2> OuterInstructions; in checkNodes()
1598 if (AllInstructions.count(cast<Instruction>(U))) in checkNodes()
1601 // Found an instruction that is not used by XCMLA/XCADD chain in checkNodes()
1609 SmallPtrSet<Instruction *, 16> Visited; in checkNodes()
1619 LLVM_DEBUG(dbgs() << "Instruction " << *I in checkNodes()
1629 Worklist.emplace_back(cast<Instruction>(U)); in checkNodes()
1632 if (auto *OpI = dyn_cast<Instruction>(Op)) in checkNodes()
1640 ComplexDeinterleavingGraph::identifyRoot(Instruction *RootI) { in identifyRoot()
1645 auto *Real = dyn_cast<Instruction>(Intrinsic->getOperand(0)); in identifyRoot()
1646 auto *Imag = dyn_cast<Instruction>(Intrinsic->getOperand(1)); in identifyRoot()
1662 Instruction *Real; in identifyRoot()
1663 Instruction *Imag; in identifyRoot()
1671 ComplexDeinterleavingGraph::identifyDeinterleave(Instruction *Real, in identifyDeinterleave()
1672 Instruction *Imag) { in identifyDeinterleave()
1673 Instruction *I = nullptr; in identifyDeinterleave()
1789 // value is a constant or an Instruction in identifySplat()
1791 if (Const->getOpcode() != Instruction::ShuffleVector) in identifySplat()
1814 auto *Real = dyn_cast<Instruction>(R); in identifySplat()
1815 auto *Imag = dyn_cast<Instruction>(I); in identifySplat()
1833 ComplexDeinterleavingGraph::identifyPHINode(Instruction *Real, in identifyPHINode()
1834 Instruction *Imag) { in identifyPHINode()
1845 ComplexDeinterleavingGraph::identifySelectNode(Instruction *Real, in identifySelectNode()
1846 Instruction *Imag) { in identifySelectNode()
1852 Instruction *MaskA, *MaskB; in identifySelectNode()
1853 Instruction *AR, *AI, *RA, *BI; in identifySelectNode()
1888 case Instruction::FNeg: in replaceSymmetricNode()
1891 case Instruction::FAdd: in replaceSymmetricNode()
1894 case Instruction::Add: in replaceSymmetricNode()
1897 case Instruction::FSub: in replaceSymmetricNode()
1900 case Instruction::Sub: in replaceSymmetricNode()
1903 case Instruction::FMul: in replaceSymmetricNode()
1906 case Instruction::Mul: in replaceSymmetricNode()
1913 cast<Instruction>(I)->setFastMathFlags(*Flags); in replaceSymmetricNode()
1956 auto *R = dyn_cast<Instruction>(Node->Real); in replaceNode()
1957 auto *I = dyn_cast<Instruction>(Node->Imag); in replaceNode()
1960 Instruction *InsertPoint = (I->comesBefore(R) ? R : I)->getNextNode(); in replaceNode()
1985 auto *MaskReal = cast<Instruction>(Node->Real)->getOperand(0); in replaceNode()
1986 auto *MaskImag = cast<Instruction>(Node->Imag)->getOperand(0); in replaceNode()
2006 auto *Real = cast<Instruction>(Node->Real); in processReductionOperation()
2007 auto *Imag = cast<Instruction>(Node->Imag); in processReductionOperation()
2046 SmallVector<Instruction *, 16> DeadInstrRoots; in replaceNodes()
2059 auto *RootReal = cast<Instruction>(RootNode->Real); in replaceNodes()
2060 auto *RootImag = cast<Instruction>(RootNode->Imag); in replaceNodes()
2063 DeadInstrRoots.push_back(cast<Instruction>(RootReal)); in replaceNodes()
2064 DeadInstrRoots.push_back(cast<Instruction>(RootImag)); in replaceNodes()