Lines Matching +full:real +full:-
1 //===- ComplexDeinterleavingPass.cpp --------------------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
16 // instruction). This is performed by checking the real and imaginary components
24 // these intrinsics, whereas, fixed-width vectors are recognized for both
31 // use-def chain. This step is assumed to finish successfully, as all
46 // depth-first manner, traversing and identifying the operands of each
48 // Each node maintains a reference to its Real and Imaginary instructions,
56 // Note: If the operation of a Node is Shuffle, only the Real, Imaginary, and
58 // should be pre-populated.
60 //===----------------------------------------------------------------------===//
80 #define DEBUG_TYPE "complex-deinterleaving"
85 "enable-complex-deinterleaving",
142 : Operation(Op), Real(R), Imag(I) {} in ComplexDeinterleavingCompositeNode()
151 Value *Real; member
171 V->print(OS, true); in dump()
183 OS << "- CompositeNode: " << this << "\n"; in dump()
184 OS << " Real: "; in dump()
185 PrintValue(Real); in dump()
194 OS << " - "; in dump()
241 /// one-block loop, then the only incoming block is 'Incoming' and the
265 /// %ReductionOP, which we refer to as real and imag (or vice versa), and
266 /// traverse the use-tree to detect complex operations. As this is a reduction
277 /// OldToNewPHI maps the original real PHINode to a new, double-sized PHINode.
290 "Reduction related nodes must have Real and Imaginary parts"); in prepareCompositeNode()
297 if (Node->Real && Node->Imag) in submitCompositeNode()
298 CachedResult[{Node->Real, Node->Imag}] = Node; in submitCompositeNode()
307 /// 90: r: cr - ai * bi
309 /// 180: r: cr - ar * br
310 /// i: ci - ar * bi
312 /// i: ci - ai * br
313 NodePtr identifyPartialMul(Instruction *Real, Instruction *Imag);
325 /// 90: r: ar - bi
328 /// i: ai - br
329 NodePtr identifyAdd(Instruction *Real, Instruction *Imag);
330 NodePtr identifySymmetricOperation(Instruction *Real, Instruction *Imag);
343 /// Extract one addend that have both real and imaginary parts positive.
354 /// Go through pairs of multiplication (one Real and one Imag) and find all
361 /// If the code is compiled with -Ofast or expressions have `reassoc` flag,
363 /// and the real and imaginary parts may not be executed in parallel. This
375 /// odd indices for /pImag instructions (only for fixed-width vectors)
378 NodePtr identifyDeinterleave(Instruction *Real, Instruction *Imag);
384 NodePtr identifySplat(Value *Real, Value *Imag);
386 NodePtr identifyPHINode(Instruction *Real, Instruction *Imag);
390 NodePtr identifySelectNode(Instruction *Real, Instruction *Imag);
405 Node->dump(OS); in dump()
412 /// In case \pB is one-block loop, this function seeks potential reductions
451 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering(); in run()
466 const auto *TL = TM->getSubtargetImpl(F)->getTargetLowering(); in runOnFunction()
478 if (!TL->isComplexDeinterleavingSupported()) { in runOnFunction()
526 if (I->getOpcode() == Instruction::FNeg) in getNegOperand()
527 return I->getOperand(0); in getNegOperand()
529 return I->getOperand(1); in getNegOperand()
550 Instruction *Real, Instruction *Imag, in identifyNodeWithImplicitAdd() argument
552 LLVM_DEBUG(dbgs() << "identifyNodeWithImplicitAdd " << *Real << " / " << *Imag in identifyNodeWithImplicitAdd()
555 if (!Real->hasOneUse() || !Imag->hasOneUse()) { in identifyNodeWithImplicitAdd()
556 LLVM_DEBUG(dbgs() << " - Mul operand has multiple uses.\n"); in identifyNodeWithImplicitAdd()
560 if ((Real->getOpcode() != Instruction::FMul && in identifyNodeWithImplicitAdd()
561 Real->getOpcode() != Instruction::Mul) || in identifyNodeWithImplicitAdd()
562 (Imag->getOpcode() != Instruction::FMul && in identifyNodeWithImplicitAdd()
563 Imag->getOpcode() != Instruction::Mul)) { in identifyNodeWithImplicitAdd()
565 dbgs() << " - Real or imaginary instruction is not fmul or mul\n"); in identifyNodeWithImplicitAdd()
569 Value *R0 = Real->getOperand(0); in identifyNodeWithImplicitAdd()
570 Value *R1 = Real->getOperand(1); in identifyNodeWithImplicitAdd()
571 Value *I0 = Imag->getOperand(0); in identifyNodeWithImplicitAdd()
572 Value *I1 = Imag->getOperand(1); in identifyNodeWithImplicitAdd()
609 LLVM_DEBUG(dbgs() << " - No equal operand\n"); in identifyNodeWithImplicitAdd()
627 LLVM_DEBUG(dbgs() << " - Incomplete partial match\n"); in identifyNodeWithImplicitAdd()
633 LLVM_DEBUG(dbgs() << " - No CommonNode identified\n"); in identifyNodeWithImplicitAdd()
639 LLVM_DEBUG(dbgs() << " - No UncommonNode identified\n"); in identifyNodeWithImplicitAdd()
644 ComplexDeinterleavingOperation::CMulPartial, Real, Imag); in identifyNodeWithImplicitAdd()
645 Node->Rotation = Rotation; in identifyNodeWithImplicitAdd()
646 Node->addOperand(CommonNode); in identifyNodeWithImplicitAdd()
647 Node->addOperand(UncommonNode); in identifyNodeWithImplicitAdd()
652 ComplexDeinterleavingGraph::identifyPartialMul(Instruction *Real, in identifyPartialMul() argument
654 LLVM_DEBUG(dbgs() << "identifyPartialMul " << *Real << " / " << *Imag in identifyPartialMul()
664 if (IsAdd(Real->getOpcode()) && IsAdd(Imag->getOpcode())) in identifyPartialMul()
666 else if (IsSub(Real->getOpcode()) && IsAdd(Imag->getOpcode())) in identifyPartialMul()
668 else if (IsSub(Real->getOpcode()) && IsSub(Imag->getOpcode())) in identifyPartialMul()
670 else if (IsAdd(Real->getOpcode()) && IsSub(Imag->getOpcode())) in identifyPartialMul()
673 LLVM_DEBUG(dbgs() << " - Unhandled rotation.\n"); in identifyPartialMul()
677 if (isa<FPMathOperator>(Real) && in identifyPartialMul()
678 (!Real->getFastMathFlags().allowContract() || in identifyPartialMul()
679 !Imag->getFastMathFlags().allowContract())) { in identifyPartialMul()
680 LLVM_DEBUG(dbgs() << " - Contract is missing from the FastMath flags.\n"); in identifyPartialMul()
684 Value *CR = Real->getOperand(0); in identifyPartialMul()
685 Instruction *RealMulI = dyn_cast<Instruction>(Real->getOperand(1)); in identifyPartialMul()
688 Value *CI = Imag->getOperand(0); in identifyPartialMul()
689 Instruction *ImagMulI = dyn_cast<Instruction>(Imag->getOperand(1)); in identifyPartialMul()
693 if (!RealMulI->hasOneUse() || !ImagMulI->hasOneUse()) { in identifyPartialMul()
694 LLVM_DEBUG(dbgs() << " - Mul instruction has multiple uses\n"); in identifyPartialMul()
698 Value *R0 = RealMulI->getOperand(0); in identifyPartialMul()
699 Value *R1 = RealMulI->getOperand(1); in identifyPartialMul()
700 Value *I0 = ImagMulI->getOperand(0); in identifyPartialMul()
701 Value *I1 = ImagMulI->getOperand(1); in identifyPartialMul()
714 LLVM_DEBUG(dbgs() << " - No equal operand\n"); in identifyPartialMul()
737 LLVM_DEBUG(dbgs() << " - Common operands are not instructions.\n"); in identifyPartialMul()
743 LLVM_DEBUG(dbgs() << " - No cnode identified\n"); in identifyPartialMul()
749 LLVM_DEBUG(dbgs() << " - No UncommonRes identified\n"); in identifyPartialMul()
756 LLVM_DEBUG(dbgs() << " - No CommonRes identified\n"); in identifyPartialMul()
761 ComplexDeinterleavingOperation::CMulPartial, Real, Imag); in identifyPartialMul()
762 Node->Rotation = Rotation; in identifyPartialMul()
763 Node->addOperand(CommonRes); in identifyPartialMul()
764 Node->addOperand(UncommonRes); in identifyPartialMul()
765 Node->addOperand(CNode); in identifyPartialMul()
770 ComplexDeinterleavingGraph::identifyAdd(Instruction *Real, Instruction *Imag) { in identifyAdd() argument
771 LLVM_DEBUG(dbgs() << "identifyAdd " << *Real << " / " << *Imag << "\n"); in identifyAdd()
775 if ((Real->getOpcode() == Instruction::FSub && in identifyAdd()
776 Imag->getOpcode() == Instruction::FAdd) || in identifyAdd()
777 (Real->getOpcode() == Instruction::Sub && in identifyAdd()
778 Imag->getOpcode() == Instruction::Add)) in identifyAdd()
780 else if ((Real->getOpcode() == Instruction::FAdd && in identifyAdd()
781 Imag->getOpcode() == Instruction::FSub) || in identifyAdd()
782 (Real->getOpcode() == Instruction::Add && in identifyAdd()
783 Imag->getOpcode() == Instruction::Sub)) in identifyAdd()
786 LLVM_DEBUG(dbgs() << " - Unhandled case, rotation is not assigned.\n"); in identifyAdd()
790 auto *AR = dyn_cast<Instruction>(Real->getOperand(0)); in identifyAdd()
791 auto *BI = dyn_cast<Instruction>(Real->getOperand(1)); in identifyAdd()
792 auto *AI = dyn_cast<Instruction>(Imag->getOperand(0)); in identifyAdd()
793 auto *BR = dyn_cast<Instruction>(Imag->getOperand(1)); in identifyAdd()
796 LLVM_DEBUG(dbgs() << " - Not all operands are instructions.\n"); in identifyAdd()
802 LLVM_DEBUG(dbgs() << " - AR/AI is not identified as a composite node.\n"); in identifyAdd()
807 LLVM_DEBUG(dbgs() << " - BR/BI is not identified as a composite node.\n"); in identifyAdd()
812 prepareCompositeNode(ComplexDeinterleavingOperation::CAdd, Real, Imag); in identifyAdd()
813 Node->Rotation = Rotation; in identifyAdd()
814 Node->addOperand(ResA); in identifyAdd()
815 Node->addOperand(ResB); in identifyAdd()
820 unsigned OpcA = A->getOpcode(); in isInstructionPairAdd()
821 unsigned OpcB = B->getOpcode(); in isInstructionPairAdd()
837 switch (I->getOpcode()) { in isInstructionPotentiallySymmetric()
852 ComplexDeinterleavingGraph::identifySymmetricOperation(Instruction *Real, in identifySymmetricOperation() argument
854 if (Real->getOpcode() != Imag->getOpcode()) in identifySymmetricOperation()
857 if (!isInstructionPotentiallySymmetric(Real) || in identifySymmetricOperation()
861 auto *R0 = Real->getOperand(0); in identifySymmetricOperation()
862 auto *I0 = Imag->getOperand(0); in identifySymmetricOperation()
869 if (Real->isBinaryOp()) { in identifySymmetricOperation()
870 auto *R1 = Real->getOperand(1); in identifySymmetricOperation()
871 auto *I1 = Imag->getOperand(1); in identifySymmetricOperation()
877 if (isa<FPMathOperator>(Real) && in identifySymmetricOperation()
878 Real->getFastMathFlags() != Imag->getFastMathFlags()) in identifySymmetricOperation()
882 Real, Imag); in identifySymmetricOperation()
883 Node->Opcode = Real->getOpcode(); in identifySymmetricOperation()
884 if (isa<FPMathOperator>(Real)) in identifySymmetricOperation()
885 Node->Flags = Real->getFastMathFlags(); in identifySymmetricOperation()
887 Node->addOperand(Op0); in identifySymmetricOperation()
888 if (Real->isBinaryOp()) in identifySymmetricOperation()
889 Node->addOperand(Op1); in identifySymmetricOperation()
897 assert(R->getType() == I->getType() && in identifyNode()
898 "Real and imaginary parts should not have different types"); in identifyNode()
902 LLVM_DEBUG(dbgs() << " - Folding to existing node\n"); in identifyNode()
903 return It->second; in identifyNode()
909 auto *Real = dyn_cast<Instruction>(R); in identifyNode() local
911 if (!Real || !Imag) in identifyNode()
914 if (NodePtr CN = identifyDeinterleave(Real, Imag)) in identifyNode()
917 if (NodePtr CN = identifyPHINode(Real, Imag)) in identifyNode()
920 if (NodePtr CN = identifySelectNode(Real, Imag)) in identifyNode()
923 auto *VTy = cast<VectorType>(Real->getType()); in identifyNode()
926 bool HasCMulSupport = TL->isComplexDeinterleavingOperationSupported( in identifyNode()
928 bool HasCAddSupport = TL->isComplexDeinterleavingOperationSupported( in identifyNode()
931 if (HasCMulSupport && isInstructionPairMul(Real, Imag)) { in identifyNode()
932 if (NodePtr CN = identifyPartialMul(Real, Imag)) in identifyNode()
936 if (HasCAddSupport && isInstructionPairAdd(Real, Imag)) { in identifyNode()
937 if (NodePtr CN = identifyAdd(Real, Imag)) in identifyNode()
942 if (NodePtr CN = identifyReassocNodes(Real, Imag)) in identifyNode()
946 if (NodePtr CN = identifySymmetricOperation(Real, Imag)) in identifyNode()
949 LLVM_DEBUG(dbgs() << " - Not recognised as a valid pattern.\n"); in identifyNode()
955 ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real, in identifyReassocNodes() argument
957 auto IsOperationSupported = [](unsigned Opcode) -> bool { in identifyReassocNodes()
963 if (!IsOperationSupported(Real->getOpcode()) || in identifyReassocNodes()
964 !IsOperationSupported(Imag->getOpcode())) in identifyReassocNodes()
968 if (isa<FPMathOperator>(Real)) { in identifyReassocNodes()
969 if (Real->getFastMathFlags() != Imag->getFastMathFlags()) { in identifyReassocNodes()
970 LLVM_DEBUG(dbgs() << "The flags in Real and Imaginary instructions are " in identifyReassocNodes()
975 Flags = Real->getFastMathFlags(); in identifyReassocNodes()
976 if (!Flags->allowReassoc()) { in identifyReassocNodes()
988 std::list<Addend> &Addends) -> bool { in identifyReassocNodes()
1009 if (I != Insn && I->getNumUses() > 1) { in identifyReassocNodes()
1010 LLVM_DEBUG(dbgs() << "Found potential sub-expression: " << *I << "\n"); in identifyReassocNodes()
1014 switch (I->getOpcode()) { in identifyReassocNodes()
1017 Worklist.emplace_back(I->getOperand(1), IsPositive); in identifyReassocNodes()
1018 Worklist.emplace_back(I->getOperand(0), IsPositive); in identifyReassocNodes()
1021 Worklist.emplace_back(I->getOperand(1), !IsPositive); in identifyReassocNodes()
1022 Worklist.emplace_back(I->getOperand(0), IsPositive); in identifyReassocNodes()
1028 Worklist.emplace_back(I->getOperand(1), !IsPositive); in identifyReassocNodes()
1029 Worklist.emplace_back(I->getOperand(0), IsPositive); in identifyReassocNodes()
1035 if (isNeg(I->getOperand(0))) { in identifyReassocNodes()
1036 A = getNegOperand(I->getOperand(0)); in identifyReassocNodes()
1039 A = I->getOperand(0); in identifyReassocNodes()
1042 if (isNeg(I->getOperand(1))) { in identifyReassocNodes()
1043 B = getNegOperand(I->getOperand(1)); in identifyReassocNodes()
1046 B = I->getOperand(1); in identifyReassocNodes()
1052 Worklist.emplace_back(I->getOperand(0), !IsPositive); in identifyReassocNodes()
1059 if (Flags && I->getFastMathFlags() != *Flags) { in identifyReassocNodes()
1071 if (!Collect(Real, RealMuls, RealAddends) || in identifyReassocNodes()
1095 // Set the Real and Imag fields of the final node and submit it in identifyReassocNodes()
1096 FinalNode->Real = Real; in identifyReassocNodes()
1097 FinalNode->Imag = Imag; in identifyReassocNodes()
1106 auto FindCommonInstruction = [](const Product &Real, in collectPartialMuls()
1107 const Product &Imag) -> Value * { in collectPartialMuls()
1108 if (Real.Multiplicand == Imag.Multiplicand || in collectPartialMuls()
1109 Real.Multiplicand == Imag.Multiplier) in collectPartialMuls()
1110 return Real.Multiplicand; in collectPartialMuls()
1112 if (Real.Multiplier == Imag.Multiplicand || in collectPartialMuls()
1113 Real.Multiplier == Imag.Multiplier) in collectPartialMuls()
1114 return Real.Multiplier; in collectPartialMuls()
1119 // Iterating over real and imaginary multiplications to find common operands in collectPartialMuls()
1180 auto NodeFromCommon = identifyNode(InfoReal->Common, InfoImag->Common); in identifyMultiplications()
1183 NodeFromCommon = identifyNode(InfoReal->Common, InfoImag->Common); in identifyMultiplications()
1188 CommonToNode[InfoReal->Common] = NodeFromCommon; in identifyMultiplications()
1189 CommonToNode[InfoImag->Common] = NodeFromCommon; in identifyMultiplications()
1204 // A.real() * B where both A and B are complex numbers. in identifyMultiplications()
1209 dbgs().indent(4) << (Mul->IsPositive ? "+" : "-") << *Mul->Multiplier in identifyMultiplications()
1210 << " multiplied by " << *Mul->Multiplicand << "\n"; in identifyMultiplications()
1218 auto NodeA = It->second; in identifyMultiplications()
1220 auto IsMultiplicandReal = PMI.Common == NodeA->Real; in identifyMultiplications()
1225 // Rotation | Real | Imag | in identifyMultiplications()
1226 // ---------+--------+--------+ in identifyMultiplications()
1228 // 90 | -y * v | y * u | in identifyMultiplications()
1229 // 180 | -x * u | -x * v | in identifyMultiplications()
1230 // 270 | y * v | -y * u | in identifyMultiplications()
1262 dbgs().indent(4) << "X: " << *NodeA->Real << "\n"; in identifyMultiplications()
1263 dbgs().indent(4) << "Y: " << *NodeA->Imag << "\n"; in identifyMultiplications()
1264 dbgs().indent(4) << "U: " << *NodeB->Real << "\n"; in identifyMultiplications()
1265 dbgs().indent(4) << "V: " << *NodeB->Imag << "\n"; in identifyMultiplications()
1266 dbgs().indent(4) << "Rotation - " << (int)Rotation * 90 << "\n"; in identifyMultiplications()
1271 NodeMul->Rotation = Rotation; in identifyMultiplications()
1272 NodeMul->addOperand(NodeA); in identifyMultiplications()
1273 NodeMul->addOperand(NodeB); in identifyMultiplications()
1275 NodeMul->addOperand(Result); in identifyMultiplications()
1289 dbgs() << "Unprocessed products (Real):\n"; in identifyMultiplications()
1292 dbgs().indent(4) << (RealMuls[i].IsPositive ? "+" : "-") in identifyMultiplications()
1299 dbgs().indent(4) << (ImagMuls[i].IsPositive ? "+" : "-") in identifyMultiplications()
1321 // Otherwise find an element with both positive real and imaginary parts. in identifyAdditions()
1357 dbgs().indent(4) << "Rotation - " << (int)Rotation * 90 << "\n"; in identifyAdditions()
1365 TmpNode->Opcode = Instruction::FAdd; in identifyAdditions()
1366 TmpNode->Flags = *Flags; in identifyAdditions()
1368 TmpNode->Opcode = Instruction::Add; in identifyAdditions()
1375 TmpNode->Opcode = Instruction::FSub; in identifyAdditions()
1376 TmpNode->Flags = *Flags; in identifyAdditions()
1378 TmpNode->Opcode = Instruction::Sub; in identifyAdditions()
1383 TmpNode->Rotation = Rotation; in identifyAdditions()
1386 TmpNode->addOperand(Result); in identifyAdditions()
1387 TmpNode->addOperand(AddNode); in identifyAdditions()
1424 // reduction. Because RootToNode maps both Real and Imaginary parts to in identifyNodes()
1425 // CompositeNode we should choose only one either Real or Imag instruction to in identifyNodes()
1429 auto RootNode = It->second; in identifyNodes()
1430 assert(RootNode->Operation == in identifyNodes()
1432 // Find out which part, Real or Imag, comes later, and only if we come to in identifyNodes()
1434 auto *R = cast<Instruction>(RootNode->Real); in identifyNodes()
1435 auto *I = cast<Instruction>(RootNode->Imag); in identifyNodes()
1436 auto *ReplacementAnchor = R->comesBefore(I) ? I : R; in identifyNodes()
1448 Function *F = RootI->getFunction(); in identifyNodes()
1449 BasicBlock *B = RootI->getParent(); in identifyNodes()
1450 dbgs() << "Complex deinterleaving graph for " << F->getName() in identifyNodes()
1451 << "::" << B->getName() << ".\n"; in identifyNodes()
1463 auto *Br = dyn_cast<BranchInst>(B->getTerminator()); in collectPotentialReductions()
1464 if (!Br || Br->getNumSuccessors() != 2) in collectPotentialReductions()
1467 // Identify simple one-block loop in collectPotentialReductions()
1468 if (Br->getSuccessor(0) != B && Br->getSuccessor(1) != B) in collectPotentialReductions()
1472 for (auto &PHI : B->phis()) { in collectPotentialReductions()
1476 if (!PHI.getType()->isVectorTy()) in collectPotentialReductions()
1486 for (auto *U : ReductionOp->users()) { in collectPotentialReductions()
1493 if (NumUsers != 2 || !FinalReduction || FinalReduction->getParent() == B || in collectPotentialReductions()
1528 auto *Real = OperationInstruction[i]; in identifyReductionNodes() local
1530 if (Real->getType() != Imag->getType()) in identifyReductionNodes()
1533 RealPHI = ReductionInfo[Real].first; in identifyReductionNodes()
1536 auto Node = identifyNode(Real, Imag); in identifyReductionNodes()
1538 std::swap(Real, Imag); in identifyReductionNodes()
1540 Node = identifyNode(Real, Imag); in identifyReductionNodes()
1545 // re-identification and attach the node to the real part in identifyReductionNodes()
1548 << *Real << " / " << *Imag << "\n"); in identifyReductionNodes()
1552 ComplexDeinterleavingOperation::ReductionOperation, Real, Imag); in identifyReductionNodes()
1553 RootNode->addOperand(Node); in identifyReductionNodes()
1554 RootToNode[Real] = RootNode; in identifyReductionNodes()
1582 for (Value *Op : I->operands()) { in checkNodes()
1597 for (User *U : I->users()) { in checkNodes()
1628 for (User *U : I->users()) in checkNodes()
1631 for (Value *Op : I->operands()) { in checkNodes()
1642 if (Intrinsic->getIntrinsicID() != Intrinsic::vector_interleave2) in identifyRoot()
1645 auto *Real = dyn_cast<Instruction>(Intrinsic->getOperand(0)); in identifyRoot() local
1646 auto *Imag = dyn_cast<Instruction>(Intrinsic->getOperand(1)); in identifyRoot()
1647 if (!Real || !Imag) in identifyRoot()
1650 return identifyNode(Real, Imag); in identifyRoot()
1657 // Look for a shufflevector that takes separate vectors of the real and in identifyRoot()
1659 if (!isInterleavingMask(SVI->getShuffleMask())) in identifyRoot()
1662 Instruction *Real; in identifyRoot() local
1664 if (!match(RootI, m_Shuffle(m_Instruction(Real), m_Instruction(Imag)))) in identifyRoot()
1667 return identifyNode(Real, Imag); in identifyRoot()
1671 ComplexDeinterleavingGraph::identifyDeinterleave(Instruction *Real, in identifyDeinterleave() argument
1675 if (match(Real, m_ExtractValue<0>(m_Instruction(I))) && in identifyDeinterleave()
1680 llvm::ComplexDeinterleavingOperation::Deinterleave, Real, Imag); in identifyDeinterleave()
1681 PlaceholderNode->ReplacementNode = FinalValue; in identifyDeinterleave()
1682 FinalInstructions.insert(Real); in identifyDeinterleave()
1687 auto *RealShuffle = dyn_cast<ShuffleVectorInst>(Real); in identifyDeinterleave()
1691 LLVM_DEBUG(dbgs() << " - There's a shuffle where there shouldn't be.\n"); in identifyDeinterleave()
1695 Value *RealOp1 = RealShuffle->getOperand(1); in identifyDeinterleave()
1697 LLVM_DEBUG(dbgs() << " - RealOp1 is not undef or zero.\n"); in identifyDeinterleave()
1700 Value *ImagOp1 = ImagShuffle->getOperand(1); in identifyDeinterleave()
1702 LLVM_DEBUG(dbgs() << " - ImagOp1 is not undef or zero.\n"); in identifyDeinterleave()
1706 Value *RealOp0 = RealShuffle->getOperand(0); in identifyDeinterleave()
1707 Value *ImagOp0 = ImagShuffle->getOperand(0); in identifyDeinterleave()
1710 LLVM_DEBUG(dbgs() << " - Shuffle operands are not equal.\n"); in identifyDeinterleave()
1714 ArrayRef<int> RealMask = RealShuffle->getShuffleMask(); in identifyDeinterleave()
1715 ArrayRef<int> ImagMask = ImagShuffle->getShuffleMask(); in identifyDeinterleave()
1717 LLVM_DEBUG(dbgs() << " - Masks are not deinterleaving.\n"); in identifyDeinterleave()
1722 LLVM_DEBUG(dbgs() << " - Masks do not have the correct initial value.\n"); in identifyDeinterleave()
1729 Value *Op = Shuffle->getOperand(0); in identifyDeinterleave()
1730 auto *ShuffleTy = cast<FixedVectorType>(Shuffle->getType()); in identifyDeinterleave()
1731 auto *OpTy = cast<FixedVectorType>(Op->getType()); in identifyDeinterleave()
1733 if (OpTy->getScalarType() != ShuffleTy->getScalarType()) in identifyDeinterleave()
1735 if ((ShuffleTy->getNumElements() * 2) != OpTy->getNumElements()) in identifyDeinterleave()
1741 auto CheckDeinterleavingShuffle = [&](ShuffleVectorInst *Shuffle) -> bool { in identifyDeinterleave()
1745 ArrayRef<int> Mask = Shuffle->getShuffleMask(); in identifyDeinterleave()
1748 Value *Op = Shuffle->getOperand(0); in identifyDeinterleave()
1749 auto *OpTy = cast<FixedVectorType>(Op->getType()); in identifyDeinterleave()
1750 int NumElements = OpTy->getNumElements(); in identifyDeinterleave()
1757 if (RealShuffle->getType() != ImagShuffle->getType()) { in identifyDeinterleave()
1758 LLVM_DEBUG(dbgs() << " - Shuffle types aren't equal.\n"); in identifyDeinterleave()
1762 LLVM_DEBUG(dbgs() << " - RealShuffle is invalid type.\n"); in identifyDeinterleave()
1766 LLVM_DEBUG(dbgs() << " - ImagShuffle is invalid type.\n"); in identifyDeinterleave()
1773 PlaceholderNode->ReplacementNode = RealShuffle->getOperand(0); in identifyDeinterleave()
1781 auto IsSplat = [](Value *V) -> bool { in identifySplat()
1782 // Fixed-width vector with constants in identifySplat()
1791 if (Const->getOpcode() != Instruction::ShuffleVector) in identifySplat()
1793 VTy = cast<VectorType>(Const->getType()); in identifySplat()
1794 Mask = Const->getShuffleMask(); in identifySplat()
1796 VTy = Shuf->getType(); in identifySplat()
1797 Mask = Shuf->getShuffleMask(); in identifySplat()
1805 if (!VTy->isScalableTy() && VTy->getElementCount().getKnownMinValue() == 1) in identifySplat()
1814 auto *Real = dyn_cast<Instruction>(R); in identifySplat() local
1816 if ((!Real && Imag) || (Real && !Imag)) in identifySplat()
1819 if (Real && Imag) { in identifySplat()
1820 // Non-constant splats should be in the same basic block in identifySplat()
1821 if (Real->getParent() != Imag->getParent()) in identifySplat()
1824 FinalInstructions.insert(Real); in identifySplat()
1833 ComplexDeinterleavingGraph::identifyPHINode(Instruction *Real, in identifyPHINode() argument
1835 if (Real != RealPHI || Imag != ImagPHI) in identifyPHINode()
1840 ComplexDeinterleavingOperation::ReductionPHI, Real, Imag); in identifyPHINode()
1845 ComplexDeinterleavingGraph::identifySelectNode(Instruction *Real, in identifySelectNode() argument
1847 auto *SelectReal = dyn_cast<SelectInst>(Real); in identifySelectNode()
1854 if (!match(Real, m_Select(m_Instruction(MaskA), m_Instruction(AR), in identifySelectNode()
1860 if (MaskA != MaskB && !MaskA->isIdenticalTo(MaskB)) in identifySelectNode()
1863 if (!MaskA->getType()->isVectorTy()) in identifySelectNode()
1875 ComplexDeinterleavingOperation::ReductionSelect, Real, Imag); in identifySelectNode()
1876 PlaceholderNode->addOperand(NodeA); in identifySelectNode()
1877 PlaceholderNode->addOperand(NodeB); in identifySelectNode()
1913 cast<Instruction>(I)->setFastMathFlags(*Flags); in replaceSymmetricNode()
1919 if (Node->ReplacementNode) in replaceNode()
1920 return Node->ReplacementNode; in replaceNode()
1922 auto ReplaceOperandIfExist = [&](RawNodePtr &Node, unsigned Idx) -> Value * { in replaceNode()
1923 return Node->Operands.size() > Idx in replaceNode()
1924 ? replaceNode(Builder, Node->Operands[Idx]) in replaceNode()
1929 switch (Node->Operation) { in replaceNode()
1936 assert(!Input1 || (Input0->getType() == Input1->getType() && in replaceNode()
1939 (Input0->getType() == Accumulator->getType() && in replaceNode()
1941 if (Node->Operation == ComplexDeinterleavingOperation::Symmetric) in replaceNode()
1942 ReplacementNode = replaceSymmetricNode(Builder, Node->Opcode, Node->Flags, in replaceNode()
1945 ReplacementNode = TL->createComplexDeinterleavingIR( in replaceNode()
1946 Builder, Node->Operation, Node->Rotation, Input0, Input1, in replaceNode()
1955 cast<VectorType>(Node->Real->getType())); in replaceNode()
1956 auto *R = dyn_cast<Instruction>(Node->Real); in replaceNode()
1957 auto *I = dyn_cast<Instruction>(Node->Imag); in replaceNode()
1960 Instruction *InsertPoint = (I->comesBefore(R) ? R : I)->getNextNode(); in replaceNode()
1963 NewTy, {Node->Real, Node->Imag}); in replaceNode()
1966 Intrinsic::vector_interleave2, NewTy, {Node->Real, Node->Imag}); in replaceNode()
1973 auto *VTy = cast<VectorType>(Node->Real->getType()); in replaceNode()
1975 auto *NewPHI = PHINode::Create(NewVTy, 0, "", BackEdge->getFirstNonPHIIt()); in replaceNode()
1976 OldToNewPHI[dyn_cast<PHINode>(Node->Real)] = NewPHI; in replaceNode()
1981 ReplacementNode = replaceNode(Builder, Node->Operands[0]); in replaceNode()
1985 auto *MaskReal = cast<Instruction>(Node->Real)->getOperand(0); in replaceNode()
1986 auto *MaskImag = cast<Instruction>(Node->Imag)->getOperand(0); in replaceNode()
1987 auto *A = replaceNode(Builder, Node->Operands[0]); in replaceNode()
1988 auto *B = replaceNode(Builder, Node->Operands[1]); in replaceNode()
1990 cast<VectorType>(MaskReal->getType())); in replaceNode()
2000 Node->ReplacementNode = ReplacementNode; in replaceNode()
2006 auto *Real = cast<Instruction>(Node->Real); in processReductionOperation() local
2007 auto *Imag = cast<Instruction>(Node->Imag); in processReductionOperation()
2008 auto *OldPHIReal = ReductionInfo[Real].first; in processReductionOperation()
2012 auto *VTy = cast<VectorType>(Real->getType()); in processReductionOperation()
2016 Value *InitReal = OldPHIReal->getIncomingValueForBlock(Incoming); in processReductionOperation()
2017 Value *InitImag = OldPHIImag->getIncomingValueForBlock(Incoming); in processReductionOperation()
2019 IRBuilder<> Builder(Incoming->getTerminator()); in processReductionOperation()
2023 NewPHI->addIncoming(NewInit, Incoming); in processReductionOperation()
2024 NewPHI->addIncoming(OperationReplacement, BackEdge); in processReductionOperation()
2028 auto *FinalReductionReal = ReductionInfo[Real].second; in processReductionOperation()
2032 &*FinalReductionReal->getParent()->getFirstInsertionPt()); in processReductionOperation()
2034 OperationReplacement->getType(), in processReductionOperation()
2038 FinalReductionReal->replaceUsesOfWith(Real, NewReal); in processReductionOperation()
2042 FinalReductionImag->replaceUsesOfWith(Imag, NewImag); in processReductionOperation()
2057 if (RootNode->Operation == in replaceNodes()
2059 auto *RootReal = cast<Instruction>(RootNode->Real); in replaceNodes()
2060 auto *RootImag = cast<Instruction>(RootNode->Imag); in replaceNodes()
2061 ReductionInfo[RootReal].first->removeIncomingValue(BackEdge); in replaceNodes()
2062 ReductionInfo[RootImag].first->removeIncomingValue(BackEdge); in replaceNodes()
2068 RootInstruction->replaceAllUsesWith(R); in replaceNodes()