Lines Matching refs:PMV
693 raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) { in operator <<() argument
703 PrintObj(PMV.WordType); in operator <<()
705 PrintObj(PMV.ValueType); in operator <<()
707 PrintObj(PMV.AlignedAddr); in operator <<()
708 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n'; in operator <<()
710 PrintObj(PMV.ShiftAmt); in operator <<()
712 PrintObj(PMV.Mask); in operator <<()
714 PrintObj(PMV.Inv_Mask); in operator <<()
741 PartwordMaskValues PMV; in createMaskInstrs() local
748 PMV.ValueType = PMV.IntValueType = ValueType; in createMaskInstrs()
749 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy()) in createMaskInstrs()
750 PMV.IntValueType = in createMaskInstrs()
753 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(Ctx, MinWordSize * 8) in createMaskInstrs()
755 if (PMV.ValueType == PMV.WordType) { in createMaskInstrs()
756 PMV.AlignedAddr = Addr; in createMaskInstrs()
757 PMV.AlignedAddrAlignment = AddrAlign; in createMaskInstrs()
758 PMV.ShiftAmt = ConstantInt::get(PMV.ValueType, 0); in createMaskInstrs()
759 PMV.Mask = ConstantInt::get(PMV.ValueType, ~0, /*isSigned*/ true); in createMaskInstrs()
760 return PMV; in createMaskInstrs()
763 PMV.AlignedAddrAlignment = Align(MinWordSize); in createMaskInstrs()
772 PMV.AlignedAddr = Builder.CreateIntrinsic( in createMaskInstrs()
781 PMV.AlignedAddr = Addr; in createMaskInstrs()
787 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3); in createMaskInstrs()
790 PMV.ShiftAmt = Builder.CreateShl( in createMaskInstrs()
794 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt"); in createMaskInstrs()
795 PMV.Mask = Builder.CreateShl( in createMaskInstrs()
796 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt, in createMaskInstrs()
799 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask"); in createMaskInstrs()
801 return PMV; in createMaskInstrs()
805 const PartwordMaskValues &PMV) { in extractMaskedValue() argument
806 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in extractMaskedValue()
807 if (PMV.WordType == PMV.ValueType) in extractMaskedValue()
810 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted"); in extractMaskedValue()
811 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType, "extracted"); in extractMaskedValue()
812 return Builder.CreateBitCast(Trunc, PMV.ValueType); in extractMaskedValue()
816 Value *Updated, const PartwordMaskValues &PMV) { in insertMaskedValue() argument
817 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in insertMaskedValue()
818 assert(Updated->getType() == PMV.ValueType && "Value type mismatch"); in insertMaskedValue()
819 if (PMV.WordType == PMV.ValueType) in insertMaskedValue()
822 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType); in insertMaskedValue()
824 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended"); in insertMaskedValue()
826 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true); in insertMaskedValue()
827 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked"); in insertMaskedValue()
838 const PartwordMaskValues &PMV) { in performMaskedAtomicOp() argument
844 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
857 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask); in performMaskedAtomicOp()
858 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
875 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV); in performMaskedAtomicOp()
877 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV); in performMaskedAtomicOp()
906 PartwordMaskValues PMV = in expandPartwordAtomicRMW() local
913 Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType); in expandPartwordAtomicRMW()
915 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt, in expandPartwordAtomicRMW()
921 AI->getValOperand(), PMV); in expandPartwordAtomicRMW()
926 OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, in expandPartwordAtomicRMW()
927 PMV.AlignedAddrAlignment, MemOpOrder, SSID, in expandPartwordAtomicRMW()
931 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr, in expandPartwordAtomicRMW()
932 PMV.AlignedAddrAlignment, MemOpOrder, in expandPartwordAtomicRMW()
936 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandPartwordAtomicRMW()
980 PartwordMaskValues PMV = in widenPartwordAtomicRMW() local
985 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in widenPartwordAtomicRMW()
986 PMV.ShiftAmt, "ValOperand_Shifted"); in widenPartwordAtomicRMW()
992 Builder.CreateOr(ValOperand_Shifted, PMV.Inv_Mask, "AndOperand"); in widenPartwordAtomicRMW()
997 Op, PMV.AlignedAddr, NewOperand, PMV.AlignedAddrAlignment, in widenPartwordAtomicRMW()
1002 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV); in widenPartwordAtomicRMW()
1064 PartwordMaskValues PMV = in expandPartwordCmpXchg() local
1070 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
1072 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
1076 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr); in expandPartwordCmpXchg()
1078 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask); in expandPartwordCmpXchg()
1083 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2); in expandPartwordCmpXchg()
1090 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment, in expandPartwordCmpXchg()
1113 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask); in expandPartwordCmpXchg()
1123 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandPartwordCmpXchg()
1148 PartwordMaskValues PMV = in expandAtomicRMWToMaskedIntrinsic() local
1161 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType), in expandAtomicRMWToMaskedIntrinsic()
1162 PMV.ShiftAmt, "ValOperand_Shifted"); in expandAtomicRMWToMaskedIntrinsic()
1164 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt, in expandAtomicRMWToMaskedIntrinsic()
1166 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandAtomicRMWToMaskedIntrinsic()
1175 PartwordMaskValues PMV = createMaskInstrs( in expandAtomicCmpXchgToMaskedIntrinsic() local
1180 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1183 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1186 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask, in expandAtomicCmpXchgToMaskedIntrinsic()
1188 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandAtomicCmpXchgToMaskedIntrinsic()
1192 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success"); in expandAtomicCmpXchgToMaskedIntrinsic()
1392 PartwordMaskValues PMV = in expandAtomicCmpXchg() local
1400 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1402 extractMaskedValue(Builder, UnreleasedLoad, PMV); in expandAtomicCmpXchg()
1417 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore"); in expandAtomicCmpXchg()
1420 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV); in expandAtomicCmpXchg()
1422 PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1433 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1434 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV); in expandAtomicCmpXchg()
1495 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV); in expandAtomicCmpXchg()