197   TargetTransformInfo::UnrollingPreferences UP;  in gatherUnrollingPreferences()  local
200   UP.Threshold =  in gatherUnrollingPreferences()
202   UP.MaxPercentThresholdBoost = 400;  in gatherUnrollingPreferences()
203   UP.OptSizeThreshold = UnrollOptSizeThreshold;  in gatherUnrollingPreferences()
204   UP.PartialThreshold = 150;  in gatherUnrollingPreferences()
205   UP.PartialOptSizeThreshold = UnrollOptSizeThreshold;  in gatherUnrollingPreferences()
206   UP.Count = 0;  in gatherUnrollingPreferences()
207   UP.DefaultUnrollRuntimeCount = 8;  in gatherUnrollingPreferences()
208   UP.MaxCount = std::numeric_limits<unsigned>::max();  in gatherUnrollingPreferences()
209   UP.MaxUpperBound = UnrollMaxUpperBound;  in gatherUnrollingPreferences()
210   UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();  in gatherUnrollingPreferences()
211   UP.BEInsns = 2;  in gatherUnrollingPreferences()
212   UP.Partial = false;  in gatherUnrollingPreferences()
213   UP.Runtime = false;  in gatherUnrollingPreferences()
214   UP.AllowRemainder = true;  in gatherUnrollingPreferences()
215   UP.UnrollRemainder = false;  in gatherUnrollingPreferences()
216   UP.AllowExpensiveTripCount = false;  in gatherUnrollingPreferences()
217   UP.Force = false;  in gatherUnrollingPreferences()
218   UP.UpperBound = false;  in gatherUnrollingPreferences()
219   UP.UnrollAndJam = false;  in gatherUnrollingPreferences()
220   UP.UnrollAndJamInnerLoopThreshold = 60;  in gatherUnrollingPreferences()
221   UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;  in gatherUnrollingPreferences()
224   TTI.getUnrollingPreferences(L, SE, UP, &ORE);  in gatherUnrollingPreferences()
233     UP.Threshold = UP.OptSizeThreshold;  in gatherUnrollingPreferences()
234     UP.PartialThreshold = UP.PartialOptSizeThreshold;  in gatherUnrollingPreferences()
235     UP.MaxPercentThresholdBoost = 100;  in gatherUnrollingPreferences()
240     UP.Threshold = UnrollThreshold;  in gatherUnrollingPreferences()
242     UP.PartialThreshold = UnrollPartialThreshold;  in gatherUnrollingPreferences()
244     UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost;  in gatherUnrollingPreferences()
246     UP.MaxCount = UnrollMaxCount;  in gatherUnrollingPreferences()
248     UP.MaxUpperBound = UnrollMaxUpperBound;  in gatherUnrollingPreferences()
250     UP.FullUnrollMaxCount = UnrollFullMaxCount;  in gatherUnrollingPreferences()
252     UP.Partial = UnrollAllowPartial;  in gatherUnrollingPreferences()
254     UP.AllowRemainder = UnrollAllowRemainder;  in gatherUnrollingPreferences()
256     UP.Runtime = UnrollRuntime;  in gatherUnrollingPreferences()
258     UP.UpperBound = false;  in gatherUnrollingPreferences()
260     UP.UnrollRemainder = UnrollUnrollRemainder;  in gatherUnrollingPreferences()
262     UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;  in gatherUnrollingPreferences()
266     UP.Threshold = *UserThreshold;  in gatherUnrollingPreferences()
267     UP.PartialThreshold = *UserThreshold;  in gatherUnrollingPreferences()
270     UP.Count = *UserCount;  in gatherUnrollingPreferences()
272     UP.Partial = *UserAllowPartial;  in gatherUnrollingPreferences()
274     UP.Runtime = *UserRuntime;  in gatherUnrollingPreferences()
276     UP.UpperBound = *UserUpperBound;  in gatherUnrollingPreferences()
278     UP.FullUnrollMaxCount = *UserFullUnrollMaxCount;  in gatherUnrollingPreferences()
280   return UP;  in gatherUnrollingPreferences()
728     const TargetTransformInfo::UnrollingPreferences &UP,  in getUnrolledLoopSize()  argument
731   assert(LS >= UP.BEInsns && "LoopSize should not be less than BEInsns!");  in getUnrolledLoopSize()
733     return static_cast<uint64_t>(LS - UP.BEInsns) * CountOverwrite + UP.BEInsns;  in getUnrolledLoopSize()
735     return static_cast<uint64_t>(LS - UP.BEInsns) * UP.Count + UP.BEInsns;  in getUnrolledLoopSize()
799                    const TargetTransformInfo::UnrollingPreferences &UP) {  in shouldPragmaUnroll()  argument
805     if (UP.AllowRemainder &&  in shouldPragmaUnroll()
806         UCE.getUnrolledLoopSize(UP, (unsigned)UnrollCount) < UP.Threshold)  in shouldPragmaUnroll()
812     if ((UP.AllowRemainder || (TripMultiple % PInfo.PragmaCount == 0)))  in shouldPragmaUnroll()
829       MaxTripCount <= UP.MaxUpperBound)  in shouldPragmaUnroll()
840     const TargetTransformInfo::UnrollingPreferences &UP) {  in shouldFullUnroll()  argument
843   if (FullUnrollTripCount > UP.FullUnrollMaxCount)  in shouldFullUnroll()
848   if (UCE.getUnrolledLoopSize(UP) < UP.Threshold)  in shouldFullUnroll()
856           UP.Threshold * UP.MaxPercentThresholdBoost / 100,  in shouldFullUnroll()
857           UP.MaxIterationsCountToAnalyze)) {  in shouldFullUnroll()
859       getFullUnrollBoostingFactor(*Cost, UP.MaxPercentThresholdBoost);  in shouldFullUnroll()
860     if (Cost->UnrolledCost < UP.Threshold * Boost / 100)  in shouldFullUnroll()
869                     const TargetTransformInfo::UnrollingPreferences &UP) {  in shouldPartialUnroll()  argument
874   if (!UP.Partial) {  in shouldPartialUnroll()
879   unsigned count = UP.Count;  in shouldPartialUnroll()
882   if (UP.PartialThreshold != NoThreshold) {  in shouldPartialUnroll()
884     if (UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)  in shouldPartialUnroll()
885       count = (std::max(UP.PartialThreshold, UP.BEInsns + 1) - UP.BEInsns) /  in shouldPartialUnroll()
886         (LoopSize - UP.BEInsns);  in shouldPartialUnroll()
887     if (count > UP.MaxCount)  in shouldPartialUnroll()
888       count = UP.MaxCount;  in shouldPartialUnroll()
891     if (UP.AllowRemainder && count <= 1) {  in shouldPartialUnroll()
896       count = UP.DefaultUnrollRuntimeCount;  in shouldPartialUnroll()
898              UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)  in shouldPartialUnroll()
907   if (count > UP.MaxCount)  in shouldPartialUnroll()
908     count = UP.MaxCount;  in shouldPartialUnroll()
928     TargetTransformInfo::UnrollingPreferences &UP,  in computeUnrollCount()  argument
950     UP.Count = 1;  in computeUnrollCount()
951     UP.Runtime = false;  in computeUnrollCount()
958                                              MaxTripCount, UCE, UP)) {  in computeUnrollCount()
959     UP.Count = *UnrollFactor;  in computeUnrollCount()
962       UP.AllowExpensiveTripCount = true;  in computeUnrollCount()
963       UP.Force = true;  in computeUnrollCount()
965     UP.Runtime |= (PragmaCount > 0);  in computeUnrollCount()
972       UP.Threshold = std::max<unsigned>(UP.Threshold, PragmaUnrollThreshold);  in computeUnrollCount()
973       UP.PartialThreshold =  in computeUnrollCount()
974           std::max<unsigned>(UP.PartialThreshold, PragmaUnrollThreshold);  in computeUnrollCount()
980   UP.Count = 0;  in computeUnrollCount()
982     UP.Count = TripCount;  in computeUnrollCount()
984                                              TripCount, UCE, UP)) {  in computeUnrollCount()
985       UP.Count = *UnrollFactor;  in computeUnrollCount()
1003   if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) &&  in computeUnrollCount()
1004       MaxTripCount <= UP.MaxUpperBound) {  in computeUnrollCount()
1005     UP.Count = MaxTripCount;  in computeUnrollCount()
1007                                              MaxTripCount, UCE, UP)) {  in computeUnrollCount()
1008       UP.Count = *UnrollFactor;  in computeUnrollCount()
1015   computePeelCount(L, LoopSize, PP, TripCount, DT, SE, AC, UP.Threshold);  in computeUnrollCount()
1017     UP.Runtime = false;  in computeUnrollCount()
1018     UP.Count = 1;  in computeUnrollCount()
1025     UP.Partial |= ExplicitUnroll;  in computeUnrollCount()
1029   if (auto UnrollFactor = shouldPartialUnroll(LoopSize, TripCount, UCE, UP)) {  in computeUnrollCount()
1030     UP.Count = *UnrollFactor;  in computeUnrollCount()
1033         UP.Count != TripCount)  in computeUnrollCount()
1043     if (UP.PartialThreshold != NoThreshold) {  in computeUnrollCount()
1044       if (UP.Count == 0) {  in computeUnrollCount()
1073     UP.Count = 0;  in computeUnrollCount()
1078   if (MaxTripCount && !UP.Force && MaxTripCount < UP.MaxUpperBound) {  in computeUnrollCount()
1079     UP.Count = 0;  in computeUnrollCount()
1089         UP.AllowExpensiveTripCount = true;  in computeUnrollCount()
1092   UP.Runtime |= PragmaEnableUnroll || PragmaCount > 0 || UserUnrollCount;  in computeUnrollCount()
1093   if (!UP.Runtime) {  in computeUnrollCount()
1097     UP.Count = 0;  in computeUnrollCount()
1100   if (UP.Count == 0)  in computeUnrollCount()
1101     UP.Count = UP.DefaultUnrollRuntimeCount;  in computeUnrollCount()
1105   while (UP.Count != 0 &&  in computeUnrollCount()
1106          UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold)  in computeUnrollCount()
1107     UP.Count >>= 1;  in computeUnrollCount()
1110   unsigned OrigCount = UP.Count;  in computeUnrollCount()
1113   if (!UP.AllowRemainder && UP.Count != 0 && (TripMultiple % UP.Count) != 0) {  in computeUnrollCount()
1114     while (UP.Count != 0 && TripMultiple % UP.Count != 0)  in computeUnrollCount()
1115       UP.Count >>= 1;  in computeUnrollCount()
1122                << " to " << UP.Count << ".\n");  in computeUnrollCount()
1126     if (unrollCountPragmaValue(L) > 0 && !UP.AllowRemainder)  in computeUnrollCount()
1138                << NV("UnrollCount", UP.Count) << " time(s).";  in computeUnrollCount()
1142   if (UP.Count > UP.MaxCount)  in computeUnrollCount()
1143     UP.Count = UP.MaxCount;  in computeUnrollCount()
1145   if (MaxTripCount && UP.Count > MaxTripCount)  in computeUnrollCount()
1146     UP.Count = MaxTripCount;  in computeUnrollCount()
1148   LLVM_DEBUG(dbgs() << "  runtime unrolling with count: " << UP.Count  in computeUnrollCount()
1150   if (UP.Count < 2)  in computeUnrollCount()
1151     UP.Count = 0;  in computeUnrollCount()
1214   TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(  in tryToUnrollLoop()  local
1223   if (UP.Threshold == 0 && (!UP.Partial || UP.PartialThreshold == 0) &&  in tryToUnrollLoop()
1230   UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);  in tryToUnrollLoop()
1242     UP.Threshold = std::max(UP.Threshold, LoopSize + 1);  in tryToUnrollLoop()
1281   UP.AllowRemainder &= UCE.ConvergenceAllowsRuntime;  in tryToUnrollLoop()
1297       MaxOrZero, TripMultiple, UCE, UP, PP, UseUpperBound);  in tryToUnrollLoop()
1298   if (!UP.Count)  in tryToUnrollLoop()
1301   UP.Runtime &= UCE.ConvergenceAllowsRuntime;  in tryToUnrollLoop()
1304     assert(UP.Count == 1 && "Cannot perform peel and unroll in the same step");  in tryToUnrollLoop()
1327   if (OnlyFullUnroll && (UP.Count < TripCount || UP.Count < MaxTripCount)) {  in tryToUnrollLoop()
1338   UP.Runtime &= TripCount == 0 && TripMultiple % UP.Count != 0;  in tryToUnrollLoop()
1346   ULO.Count = UP.Count;  in tryToUnrollLoop()
1347   ULO.Force = UP.Force;  in tryToUnrollLoop()
1348   ULO.AllowExpensiveTripCount = UP.AllowExpensiveTripCount;  in tryToUnrollLoop()
1349   ULO.UnrollRemainder = UP.UnrollRemainder;  in tryToUnrollLoop()
1350   ULO.Runtime = UP.Runtime;  in tryToUnrollLoop()