Lines Matching refs:MatMul
1332 void lowerDotProduct(CallInst *MatMul, in lowerDotProduct() argument
1335 if (FusedInsts.contains(MatMul) || in lowerDotProduct()
1338 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in lowerDotProduct()
1339 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in lowerDotProduct()
1344 Value *LHS = MatMul->getArgOperand(0); in lowerDotProduct()
1345 Value *RHS = MatMul->getArgOperand(1); in lowerDotProduct()
1456 FusedInsts.insert(MatMul); in lowerDotProduct()
1457 IRBuilder<> Builder(MatMul); in lowerDotProduct()
1491 LHS = MatMul->getArgOperand(0); in lowerDotProduct()
1509 Result = Builder.CreateInsertElement(PoisonValue::get(MatMul->getType()), in lowerDotProduct()
1511 MatMul->replaceAllUsesWith(Result); in lowerDotProduct()
1512 FusedInsts.insert(MatMul); in lowerDotProduct()
1513 ToRemove.push_back(MatMul); in lowerDotProduct()
1608 CallInst *MatMul) { in getNonAliasingPointer() argument
1620 BasicBlock *Check0 = MatMul->getParent(); in getNonAliasingPointer()
1629 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1632 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1635 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1641 IRBuilder<> Builder(MatMul); in getNonAliasingPointer()
1692 bool isFusionProfitable(CallInst *MatMul) { in isFusionProfitable() argument
1696 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in isFusionProfitable()
1697 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in isFusionProfitable()
1702 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in isFusionProfitable()
1736 void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape, in createTiledLoops() argument
1738 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in createTiledLoops()
1743 Instruction *InsertI = cast<Instruction>(MatMul); in createTiledLoops()
1747 IRBuilder<> Builder(MatMul); in createTiledLoops()
1751 FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize); in createTiledLoops()
1776 getFastMathFlags(MatMul)); in createTiledLoops()
1795 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1, in emitSIMDTiling() argument
1800 if (!isFusionProfitable(MatMul)) in emitSIMDTiling()
1803 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in emitSIMDTiling()
1804 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in emitSIMDTiling()
1809 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in emitSIMDTiling()
1811 Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul); in emitSIMDTiling()
1812 Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul); in emitSIMDTiling()
1816 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store); in emitSIMDTiling()
1836 getFastMathFlags(MatMul)); in emitSIMDTiling()
1846 FusedInsts.insert(MatMul); in emitSIMDTiling()
1848 MatMul->eraseFromParent(); in emitSIMDTiling()
1864 LowerMatrixMultiplyFused(CallInst *MatMul, in LowerMatrixMultiplyFused() argument
1872 Value *A = MatMul->getArgOperand(0); in LowerMatrixMultiplyFused()
1873 Value *B = MatMul->getArgOperand(1); in LowerMatrixMultiplyFused()
1880 IRBuilder<> Builder(MatMul); in LowerMatrixMultiplyFused()
1881 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMatrixMultiplyFused()
1882 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMatrixMultiplyFused()
1883 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMatrixMultiplyFused()
1906 getFastMathFlags(MatMul)); in LowerMatrixMultiplyFused()
1908 FusedInsts.insert(MatMul); in LowerMatrixMultiplyFused()
1916 finalizeLowering(MatMul, Result, Builder); in LowerMatrixMultiplyFused()
1920 if (!MatMul->hasOneUse() || MatrixLayout != MatrixLayoutTy::ColumnMajor) in LowerMatrixMultiplyFused()
1927 auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin()); in LowerMatrixMultiplyFused()
1941 if (DT->dominates(CurrI, MatMul)) in LowerMatrixMultiplyFused()
1953 I->moveBefore(MatMul); in LowerMatrixMultiplyFused()
2004 emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts); in LowerMatrixMultiplyFused()
2010 void LowerMultiply(CallInst *MatMul) { in LowerMultiply() argument
2011 IRBuilder<> Builder(MatMul); in LowerMultiply()
2012 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMultiply()
2013 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMultiply()
2014 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMultiply()
2016 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder); in LowerMultiply()
2017 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder); in LowerMultiply()
2031 getFastMathFlags(MatMul)); in LowerMultiply()
2032 finalizeLowering(MatMul, Result, Builder); in LowerMultiply()