xref: /freebsd/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp (revision 7ef62cebc2f965b0f640263e179276928885e33d)
1 //===- InstCombineCompares.cpp --------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the visitICmp and visitFCmp functions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "InstCombineInternal.h"
14 #include "llvm/ADT/APSInt.h"
15 #include "llvm/ADT/SetVector.h"
16 #include "llvm/ADT/Statistic.h"
17 #include "llvm/Analysis/CmpInstAnalysis.h"
18 #include "llvm/Analysis/ConstantFolding.h"
19 #include "llvm/Analysis/InstructionSimplify.h"
20 #include "llvm/Analysis/VectorUtils.h"
21 #include "llvm/IR/ConstantRange.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/GetElementPtrTypeIterator.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/PatternMatch.h"
26 #include "llvm/Support/KnownBits.h"
27 #include "llvm/Transforms/InstCombine/InstCombiner.h"
28 
29 using namespace llvm;
30 using namespace PatternMatch;
31 
32 #define DEBUG_TYPE "instcombine"
33 
34 // How many times is a select replaced by one of its operands?
35 STATISTIC(NumSel, "Number of select opts");
36 
37 
38 /// Compute Result = In1+In2, returning true if the result overflowed for this
39 /// type.
40 static bool addWithOverflow(APInt &Result, const APInt &In1,
41                             const APInt &In2, bool IsSigned = false) {
42   bool Overflow;
43   if (IsSigned)
44     Result = In1.sadd_ov(In2, Overflow);
45   else
46     Result = In1.uadd_ov(In2, Overflow);
47 
48   return Overflow;
49 }
50 
51 /// Compute Result = In1-In2, returning true if the result overflowed for this
52 /// type.
53 static bool subWithOverflow(APInt &Result, const APInt &In1,
54                             const APInt &In2, bool IsSigned = false) {
55   bool Overflow;
56   if (IsSigned)
57     Result = In1.ssub_ov(In2, Overflow);
58   else
59     Result = In1.usub_ov(In2, Overflow);
60 
61   return Overflow;
62 }
63 
64 /// Given an icmp instruction, return true if any use of this comparison is a
65 /// branch on sign bit comparison.
66 static bool hasBranchUse(ICmpInst &I) {
67   for (auto *U : I.users())
68     if (isa<BranchInst>(U))
69       return true;
70   return false;
71 }
72 
73 /// Returns true if the exploded icmp can be expressed as a signed comparison
74 /// to zero and updates the predicate accordingly.
75 /// The signedness of the comparison is preserved.
76 /// TODO: Refactor with decomposeBitTestICmp()?
77 static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) {
78   if (!ICmpInst::isSigned(Pred))
79     return false;
80 
81   if (C.isZero())
82     return ICmpInst::isRelational(Pred);
83 
84   if (C.isOne()) {
85     if (Pred == ICmpInst::ICMP_SLT) {
86       Pred = ICmpInst::ICMP_SLE;
87       return true;
88     }
89   } else if (C.isAllOnes()) {
90     if (Pred == ICmpInst::ICMP_SGT) {
91       Pred = ICmpInst::ICMP_SGE;
92       return true;
93     }
94   }
95 
96   return false;
97 }
98 
99 /// This is called when we see this pattern:
100 ///   cmp pred (load (gep GV, ...)), cmpcst
101 /// where GV is a global variable with a constant initializer. Try to simplify
102 /// this into some simple computation that does not need the load. For example
103 /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3".
104 ///
105 /// If AndCst is non-null, then the loaded value is masked with that constant
106 /// before doing the comparison. This handles cases like "A[i]&4 == 0".
107 Instruction *InstCombinerImpl::foldCmpLoadFromIndexedGlobal(
108     LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI,
109     ConstantInt *AndCst) {
110   if (LI->isVolatile() || LI->getType() != GEP->getResultElementType() ||
111       GV->getValueType() != GEP->getSourceElementType() ||
112       !GV->isConstant() || !GV->hasDefinitiveInitializer())
113     return nullptr;
114 
115   Constant *Init = GV->getInitializer();
116   if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init))
117     return nullptr;
118 
119   uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
120   // Don't blow up on huge arrays.
121   if (ArrayElementCount > MaxArraySizeForCombine)
122     return nullptr;
123 
124   // There are many forms of this optimization we can handle, for now, just do
125   // the simple index into a single-dimensional array.
126   //
127   // Require: GEP GV, 0, i {{, constant indices}}
128   if (GEP->getNumOperands() < 3 ||
129       !isa<ConstantInt>(GEP->getOperand(1)) ||
130       !cast<ConstantInt>(GEP->getOperand(1))->isZero() ||
131       isa<Constant>(GEP->getOperand(2)))
132     return nullptr;
133 
134   // Check that indices after the variable are constants and in-range for the
135   // type they index.  Collect the indices.  This is typically for arrays of
136   // structs.
137   SmallVector<unsigned, 4> LaterIndices;
138 
139   Type *EltTy = Init->getType()->getArrayElementType();
140   for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) {
141     ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i));
142     if (!Idx) return nullptr;  // Variable index.
143 
144     uint64_t IdxVal = Idx->getZExtValue();
145     if ((unsigned)IdxVal != IdxVal) return nullptr; // Too large array index.
146 
147     if (StructType *STy = dyn_cast<StructType>(EltTy))
148       EltTy = STy->getElementType(IdxVal);
149     else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) {
150       if (IdxVal >= ATy->getNumElements()) return nullptr;
151       EltTy = ATy->getElementType();
152     } else {
153       return nullptr; // Unknown type.
154     }
155 
156     LaterIndices.push_back(IdxVal);
157   }
158 
159   enum { Overdefined = -3, Undefined = -2 };
160 
161   // Variables for our state machines.
162 
163   // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form
164   // "i == 47 | i == 87", where 47 is the first index the condition is true for,
165   // and 87 is the second (and last) index.  FirstTrueElement is -2 when
166   // undefined, otherwise set to the first true element.  SecondTrueElement is
167   // -2 when undefined, -3 when overdefined and >= 0 when that index is true.
168   int FirstTrueElement = Undefined, SecondTrueElement = Undefined;
169 
170   // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the
171   // form "i != 47 & i != 87".  Same state transitions as for true elements.
172   int FirstFalseElement = Undefined, SecondFalseElement = Undefined;
173 
174   /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these
175   /// define a state machine that triggers for ranges of values that the index
176   /// is true or false for.  This triggers on things like "abbbbc"[i] == 'b'.
177   /// This is -2 when undefined, -3 when overdefined, and otherwise the last
178   /// index in the range (inclusive).  We use -2 for undefined here because we
179   /// use relative comparisons and don't want 0-1 to match -1.
180   int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined;
181 
182   // MagicBitvector - This is a magic bitvector where we set a bit if the
183   // comparison is true for element 'i'.  If there are 64 elements or less in
184   // the array, this will fully represent all the comparison results.
185   uint64_t MagicBitvector = 0;
186 
187   // Scan the array and see if one of our patterns matches.
188   Constant *CompareRHS = cast<Constant>(ICI.getOperand(1));
189   for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) {
190     Constant *Elt = Init->getAggregateElement(i);
191     if (!Elt) return nullptr;
192 
193     // If this is indexing an array of structures, get the structure element.
194     if (!LaterIndices.empty()) {
195       Elt = ConstantFoldExtractValueInstruction(Elt, LaterIndices);
196       if (!Elt)
197         return nullptr;
198     }
199 
200     // If the element is masked, handle it.
201     if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst);
202 
203     // Find out if the comparison would be true or false for the i'th element.
204     Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt,
205                                                   CompareRHS, DL, &TLI);
206     // If the result is undef for this element, ignore it.
207     if (isa<UndefValue>(C)) {
208       // Extend range state machines to cover this element in case there is an
209       // undef in the middle of the range.
210       if (TrueRangeEnd == (int)i-1)
211         TrueRangeEnd = i;
212       if (FalseRangeEnd == (int)i-1)
213         FalseRangeEnd = i;
214       continue;
215     }
216 
217     // If we can't compute the result for any of the elements, we have to give
218     // up evaluating the entire conditional.
219     if (!isa<ConstantInt>(C)) return nullptr;
220 
221     // Otherwise, we know if the comparison is true or false for this element,
222     // update our state machines.
223     bool IsTrueForElt = !cast<ConstantInt>(C)->isZero();
224 
225     // State machine for single/double/range index comparison.
226     if (IsTrueForElt) {
227       // Update the TrueElement state machine.
228       if (FirstTrueElement == Undefined)
229         FirstTrueElement = TrueRangeEnd = i;  // First true element.
230       else {
231         // Update double-compare state machine.
232         if (SecondTrueElement == Undefined)
233           SecondTrueElement = i;
234         else
235           SecondTrueElement = Overdefined;
236 
237         // Update range state machine.
238         if (TrueRangeEnd == (int)i-1)
239           TrueRangeEnd = i;
240         else
241           TrueRangeEnd = Overdefined;
242       }
243     } else {
244       // Update the FalseElement state machine.
245       if (FirstFalseElement == Undefined)
246         FirstFalseElement = FalseRangeEnd = i; // First false element.
247       else {
248         // Update double-compare state machine.
249         if (SecondFalseElement == Undefined)
250           SecondFalseElement = i;
251         else
252           SecondFalseElement = Overdefined;
253 
254         // Update range state machine.
255         if (FalseRangeEnd == (int)i-1)
256           FalseRangeEnd = i;
257         else
258           FalseRangeEnd = Overdefined;
259       }
260     }
261 
262     // If this element is in range, update our magic bitvector.
263     if (i < 64 && IsTrueForElt)
264       MagicBitvector |= 1ULL << i;
265 
266     // If all of our states become overdefined, bail out early.  Since the
267     // predicate is expensive, only check it every 8 elements.  This is only
268     // really useful for really huge arrays.
269     if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined &&
270         SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined &&
271         FalseRangeEnd == Overdefined)
272       return nullptr;
273   }
274 
275   // Now that we've scanned the entire array, emit our new comparison(s).  We
276   // order the state machines in complexity of the generated code.
277   Value *Idx = GEP->getOperand(2);
278 
279   // If the index is larger than the pointer size of the target, truncate the
280   // index down like the GEP would do implicitly.  We don't have to do this for
281   // an inbounds GEP because the index can't be out of range.
282   if (!GEP->isInBounds()) {
283     Type *IntPtrTy = DL.getIntPtrType(GEP->getType());
284     unsigned PtrSize = IntPtrTy->getIntegerBitWidth();
285     if (Idx->getType()->getPrimitiveSizeInBits().getFixedValue() > PtrSize)
286       Idx = Builder.CreateTrunc(Idx, IntPtrTy);
287   }
288 
289   // If inbounds keyword is not present, Idx * ElementSize can overflow.
290   // Let's assume that ElementSize is 2 and the wanted value is at offset 0.
291   // Then, there are two possible values for Idx to match offset 0:
292   // 0x00..00, 0x80..00.
293   // Emitting 'icmp eq Idx, 0' isn't correct in this case because the
294   // comparison is false if Idx was 0x80..00.
295   // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx.
296   unsigned ElementSize =
297       DL.getTypeAllocSize(Init->getType()->getArrayElementType());
298   auto MaskIdx = [&](Value* Idx){
299     if (!GEP->isInBounds() && countTrailingZeros(ElementSize) != 0) {
300       Value *Mask = ConstantInt::get(Idx->getType(), -1);
301       Mask = Builder.CreateLShr(Mask, countTrailingZeros(ElementSize));
302       Idx = Builder.CreateAnd(Idx, Mask);
303     }
304     return Idx;
305   };
306 
307   // If the comparison is only true for one or two elements, emit direct
308   // comparisons.
309   if (SecondTrueElement != Overdefined) {
310     Idx = MaskIdx(Idx);
311     // None true -> false.
312     if (FirstTrueElement == Undefined)
313       return replaceInstUsesWith(ICI, Builder.getFalse());
314 
315     Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement);
316 
317     // True for one element -> 'i == 47'.
318     if (SecondTrueElement == Undefined)
319       return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx);
320 
321     // True for two elements -> 'i == 47 | i == 72'.
322     Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx);
323     Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement);
324     Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx);
325     return BinaryOperator::CreateOr(C1, C2);
326   }
327 
328   // If the comparison is only false for one or two elements, emit direct
329   // comparisons.
330   if (SecondFalseElement != Overdefined) {
331     Idx = MaskIdx(Idx);
332     // None false -> true.
333     if (FirstFalseElement == Undefined)
334       return replaceInstUsesWith(ICI, Builder.getTrue());
335 
336     Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement);
337 
338     // False for one element -> 'i != 47'.
339     if (SecondFalseElement == Undefined)
340       return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx);
341 
342     // False for two elements -> 'i != 47 & i != 72'.
343     Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx);
344     Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement);
345     Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx);
346     return BinaryOperator::CreateAnd(C1, C2);
347   }
348 
349   // If the comparison can be replaced with a range comparison for the elements
350   // where it is true, emit the range check.
351   if (TrueRangeEnd != Overdefined) {
352     assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare");
353     Idx = MaskIdx(Idx);
354 
355     // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1).
356     if (FirstTrueElement) {
357       Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement);
358       Idx = Builder.CreateAdd(Idx, Offs);
359     }
360 
361     Value *End = ConstantInt::get(Idx->getType(),
362                                   TrueRangeEnd-FirstTrueElement+1);
363     return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End);
364   }
365 
366   // False range check.
367   if (FalseRangeEnd != Overdefined) {
368     assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare");
369     Idx = MaskIdx(Idx);
370     // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse).
371     if (FirstFalseElement) {
372       Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement);
373       Idx = Builder.CreateAdd(Idx, Offs);
374     }
375 
376     Value *End = ConstantInt::get(Idx->getType(),
377                                   FalseRangeEnd-FirstFalseElement);
378     return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End);
379   }
380 
381   // If a magic bitvector captures the entire comparison state
382   // of this load, replace it with computation that does:
383   //   ((magic_cst >> i) & 1) != 0
384   {
385     Type *Ty = nullptr;
386 
387     // Look for an appropriate type:
388     // - The type of Idx if the magic fits
389     // - The smallest fitting legal type
390     if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth())
391       Ty = Idx->getType();
392     else
393       Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount);
394 
395     if (Ty) {
396       Idx = MaskIdx(Idx);
397       Value *V = Builder.CreateIntCast(Idx, Ty, false);
398       V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V);
399       V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V);
400       return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0));
401     }
402   }
403 
404   return nullptr;
405 }
406 
407 /// Returns true if we can rewrite Start as a GEP with pointer Base
408 /// and some integer offset. The nodes that need to be re-written
409 /// for this transformation will be added to Explored.
410 static bool canRewriteGEPAsOffset(Type *ElemTy, Value *Start, Value *Base,
411                                   const DataLayout &DL,
412                                   SetVector<Value *> &Explored) {
413   SmallVector<Value *, 16> WorkList(1, Start);
414   Explored.insert(Base);
415 
416   // The following traversal gives us an order which can be used
417   // when doing the final transformation. Since in the final
418   // transformation we create the PHI replacement instructions first,
419   // we don't have to get them in any particular order.
420   //
421   // However, for other instructions we will have to traverse the
422   // operands of an instruction first, which means that we have to
423   // do a post-order traversal.
424   while (!WorkList.empty()) {
425     SetVector<PHINode *> PHIs;
426 
427     while (!WorkList.empty()) {
428       if (Explored.size() >= 100)
429         return false;
430 
431       Value *V = WorkList.back();
432 
433       if (Explored.contains(V)) {
434         WorkList.pop_back();
435         continue;
436       }
437 
438       if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) &&
439           !isa<GetElementPtrInst>(V) && !isa<PHINode>(V))
440         // We've found some value that we can't explore which is different from
441         // the base. Therefore we can't do this transformation.
442         return false;
443 
444       if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) {
445         auto *CI = cast<CastInst>(V);
446         if (!CI->isNoopCast(DL))
447           return false;
448 
449         if (!Explored.contains(CI->getOperand(0)))
450           WorkList.push_back(CI->getOperand(0));
451       }
452 
453       if (auto *GEP = dyn_cast<GEPOperator>(V)) {
454         // We're limiting the GEP to having one index. This will preserve
455         // the original pointer type. We could handle more cases in the
456         // future.
457         if (GEP->getNumIndices() != 1 || !GEP->isInBounds() ||
458             GEP->getSourceElementType() != ElemTy)
459           return false;
460 
461         if (!Explored.contains(GEP->getOperand(0)))
462           WorkList.push_back(GEP->getOperand(0));
463       }
464 
465       if (WorkList.back() == V) {
466         WorkList.pop_back();
467         // We've finished visiting this node, mark it as such.
468         Explored.insert(V);
469       }
470 
471       if (auto *PN = dyn_cast<PHINode>(V)) {
472         // We cannot transform PHIs on unsplittable basic blocks.
473         if (isa<CatchSwitchInst>(PN->getParent()->getTerminator()))
474           return false;
475         Explored.insert(PN);
476         PHIs.insert(PN);
477       }
478     }
479 
480     // Explore the PHI nodes further.
481     for (auto *PN : PHIs)
482       for (Value *Op : PN->incoming_values())
483         if (!Explored.contains(Op))
484           WorkList.push_back(Op);
485   }
486 
487   // Make sure that we can do this. Since we can't insert GEPs in a basic
488   // block before a PHI node, we can't easily do this transformation if
489   // we have PHI node users of transformed instructions.
490   for (Value *Val : Explored) {
491     for (Value *Use : Val->uses()) {
492 
493       auto *PHI = dyn_cast<PHINode>(Use);
494       auto *Inst = dyn_cast<Instruction>(Val);
495 
496       if (Inst == Base || Inst == PHI || !Inst || !PHI ||
497           !Explored.contains(PHI))
498         continue;
499 
500       if (PHI->getParent() == Inst->getParent())
501         return false;
502     }
503   }
504   return true;
505 }
506 
507 // Sets the appropriate insert point on Builder where we can add
508 // a replacement Instruction for V (if that is possible).
509 static void setInsertionPoint(IRBuilder<> &Builder, Value *V,
510                               bool Before = true) {
511   if (auto *PHI = dyn_cast<PHINode>(V)) {
512     Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt());
513     return;
514   }
515   if (auto *I = dyn_cast<Instruction>(V)) {
516     if (!Before)
517       I = &*std::next(I->getIterator());
518     Builder.SetInsertPoint(I);
519     return;
520   }
521   if (auto *A = dyn_cast<Argument>(V)) {
522     // Set the insertion point in the entry block.
523     BasicBlock &Entry = A->getParent()->getEntryBlock();
524     Builder.SetInsertPoint(&*Entry.getFirstInsertionPt());
525     return;
526   }
527   // Otherwise, this is a constant and we don't need to set a new
528   // insertion point.
529   assert(isa<Constant>(V) && "Setting insertion point for unknown value!");
530 }
531 
532 /// Returns a re-written value of Start as an indexed GEP using Base as a
533 /// pointer.
534 static Value *rewriteGEPAsOffset(Type *ElemTy, Value *Start, Value *Base,
535                                  const DataLayout &DL,
536                                  SetVector<Value *> &Explored) {
537   // Perform all the substitutions. This is a bit tricky because we can
538   // have cycles in our use-def chains.
539   // 1. Create the PHI nodes without any incoming values.
540   // 2. Create all the other values.
541   // 3. Add the edges for the PHI nodes.
542   // 4. Emit GEPs to get the original pointers.
543   // 5. Remove the original instructions.
544   Type *IndexType = IntegerType::get(
545       Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType()));
546 
547   DenseMap<Value *, Value *> NewInsts;
548   NewInsts[Base] = ConstantInt::getNullValue(IndexType);
549 
550   // Create the new PHI nodes, without adding any incoming values.
551   for (Value *Val : Explored) {
552     if (Val == Base)
553       continue;
554     // Create empty phi nodes. This avoids cyclic dependencies when creating
555     // the remaining instructions.
556     if (auto *PHI = dyn_cast<PHINode>(Val))
557       NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(),
558                                       PHI->getName() + ".idx", PHI);
559   }
560   IRBuilder<> Builder(Base->getContext());
561 
562   // Create all the other instructions.
563   for (Value *Val : Explored) {
564 
565     if (NewInsts.find(Val) != NewInsts.end())
566       continue;
567 
568     if (auto *CI = dyn_cast<CastInst>(Val)) {
569       // Don't get rid of the intermediate variable here; the store can grow
570       // the map which will invalidate the reference to the input value.
571       Value *V = NewInsts[CI->getOperand(0)];
572       NewInsts[CI] = V;
573       continue;
574     }
575     if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
576       Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)]
577                                                   : GEP->getOperand(1);
578       setInsertionPoint(Builder, GEP);
579       // Indices might need to be sign extended. GEPs will magically do
580       // this, but we need to do it ourselves here.
581       if (Index->getType()->getScalarSizeInBits() !=
582           NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) {
583         Index = Builder.CreateSExtOrTrunc(
584             Index, NewInsts[GEP->getOperand(0)]->getType(),
585             GEP->getOperand(0)->getName() + ".sext");
586       }
587 
588       auto *Op = NewInsts[GEP->getOperand(0)];
589       if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero())
590         NewInsts[GEP] = Index;
591       else
592         NewInsts[GEP] = Builder.CreateNSWAdd(
593             Op, Index, GEP->getOperand(0)->getName() + ".add");
594       continue;
595     }
596     if (isa<PHINode>(Val))
597       continue;
598 
599     llvm_unreachable("Unexpected instruction type");
600   }
601 
602   // Add the incoming values to the PHI nodes.
603   for (Value *Val : Explored) {
604     if (Val == Base)
605       continue;
606     // All the instructions have been created, we can now add edges to the
607     // phi nodes.
608     if (auto *PHI = dyn_cast<PHINode>(Val)) {
609       PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]);
610       for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
611         Value *NewIncoming = PHI->getIncomingValue(I);
612 
613         if (NewInsts.find(NewIncoming) != NewInsts.end())
614           NewIncoming = NewInsts[NewIncoming];
615 
616         NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I));
617       }
618     }
619   }
620 
621   PointerType *PtrTy =
622       ElemTy->getPointerTo(Start->getType()->getPointerAddressSpace());
623   for (Value *Val : Explored) {
624     if (Val == Base)
625       continue;
626 
627     // Depending on the type, for external users we have to emit
628     // a GEP or a GEP + ptrtoint.
629     setInsertionPoint(Builder, Val, false);
630 
631     // Cast base to the expected type.
632     Value *NewVal = Builder.CreateBitOrPointerCast(
633         Base, PtrTy, Start->getName() + "to.ptr");
634     NewVal = Builder.CreateInBoundsGEP(ElemTy, NewVal, ArrayRef(NewInsts[Val]),
635                                        Val->getName() + ".ptr");
636     NewVal = Builder.CreateBitOrPointerCast(
637         NewVal, Val->getType(), Val->getName() + ".conv");
638     Val->replaceAllUsesWith(NewVal);
639   }
640 
641   return NewInsts[Start];
642 }
643 
644 /// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express
645 /// the input Value as a constant indexed GEP. Returns a pair containing
646 /// the GEPs Pointer and Index.
647 static std::pair<Value *, Value *>
648 getAsConstantIndexedAddress(Type *ElemTy, Value *V, const DataLayout &DL) {
649   Type *IndexType = IntegerType::get(V->getContext(),
650                                      DL.getIndexTypeSizeInBits(V->getType()));
651 
652   Constant *Index = ConstantInt::getNullValue(IndexType);
653   while (true) {
654     if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
655       // We accept only inbouds GEPs here to exclude the possibility of
656       // overflow.
657       if (!GEP->isInBounds())
658         break;
659       if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 &&
660           GEP->getSourceElementType() == ElemTy) {
661         V = GEP->getOperand(0);
662         Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1));
663         Index = ConstantExpr::getAdd(
664             Index, ConstantExpr::getSExtOrTrunc(GEPIndex, IndexType));
665         continue;
666       }
667       break;
668     }
669     if (auto *CI = dyn_cast<IntToPtrInst>(V)) {
670       if (!CI->isNoopCast(DL))
671         break;
672       V = CI->getOperand(0);
673       continue;
674     }
675     if (auto *CI = dyn_cast<PtrToIntInst>(V)) {
676       if (!CI->isNoopCast(DL))
677         break;
678       V = CI->getOperand(0);
679       continue;
680     }
681     break;
682   }
683   return {V, Index};
684 }
685 
686 /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
687 /// We can look through PHIs, GEPs and casts in order to determine a common base
688 /// between GEPLHS and RHS.
689 static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS,
690                                               ICmpInst::Predicate Cond,
691                                               const DataLayout &DL) {
692   // FIXME: Support vector of pointers.
693   if (GEPLHS->getType()->isVectorTy())
694     return nullptr;
695 
696   if (!GEPLHS->hasAllConstantIndices())
697     return nullptr;
698 
699   Type *ElemTy = GEPLHS->getSourceElementType();
700   Value *PtrBase, *Index;
701   std::tie(PtrBase, Index) = getAsConstantIndexedAddress(ElemTy, GEPLHS, DL);
702 
703   // The set of nodes that will take part in this transformation.
704   SetVector<Value *> Nodes;
705 
706   if (!canRewriteGEPAsOffset(ElemTy, RHS, PtrBase, DL, Nodes))
707     return nullptr;
708 
709   // We know we can re-write this as
710   //  ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)
711   // Since we've only looked through inbouds GEPs we know that we
712   // can't have overflow on either side. We can therefore re-write
713   // this as:
714   //   OFFSET1 cmp OFFSET2
715   Value *NewRHS = rewriteGEPAsOffset(ElemTy, RHS, PtrBase, DL, Nodes);
716 
717   // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written
718   // GEP having PtrBase as the pointer base, and has returned in NewRHS the
719   // offset. Since Index is the offset of LHS to the base pointer, we will now
720   // compare the offsets instead of comparing the pointers.
721   return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS);
722 }
723 
724 /// Fold comparisons between a GEP instruction and something else. At this point
725 /// we know that the GEP is on the LHS of the comparison.
726 Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
727                                            ICmpInst::Predicate Cond,
728                                            Instruction &I) {
729   // Don't transform signed compares of GEPs into index compares. Even if the
730   // GEP is inbounds, the final add of the base pointer can have signed overflow
731   // and would change the result of the icmp.
732   // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be
733   // the maximum signed value for the pointer type.
734   if (ICmpInst::isSigned(Cond))
735     return nullptr;
736 
737   // Look through bitcasts and addrspacecasts. We do not however want to remove
738   // 0 GEPs.
739   if (!isa<GetElementPtrInst>(RHS))
740     RHS = RHS->stripPointerCasts();
741 
742   Value *PtrBase = GEPLHS->getOperand(0);
743   if (PtrBase == RHS && GEPLHS->isInBounds()) {
744     // ((gep Ptr, OFFSET) cmp Ptr)   ---> (OFFSET cmp 0).
745     Value *Offset = EmitGEPOffset(GEPLHS);
746     return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset,
747                         Constant::getNullValue(Offset->getType()));
748   }
749 
750   if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) &&
751       isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() &&
752       !NullPointerIsDefined(I.getFunction(),
753                             RHS->getType()->getPointerAddressSpace())) {
754     // For most address spaces, an allocation can't be placed at null, but null
755     // itself is treated as a 0 size allocation in the in bounds rules.  Thus,
756     // the only valid inbounds address derived from null, is null itself.
757     // Thus, we have four cases to consider:
758     // 1) Base == nullptr, Offset == 0 -> inbounds, null
759     // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds
760     // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations)
761     // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison)
762     //
763     // (Note if we're indexing a type of size 0, that simply collapses into one
764     //  of the buckets above.)
765     //
766     // In general, we're allowed to make values less poison (i.e. remove
767     //   sources of full UB), so in this case, we just select between the two
768     //   non-poison cases (1 and 4 above).
769     //
770     // For vectors, we apply the same reasoning on a per-lane basis.
771     auto *Base = GEPLHS->getPointerOperand();
772     if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) {
773       auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount();
774       Base = Builder.CreateVectorSplat(EC, Base);
775     }
776     return new ICmpInst(Cond, Base,
777                         ConstantExpr::getPointerBitCastOrAddrSpaceCast(
778                             cast<Constant>(RHS), Base->getType()));
779   } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) {
780     // If the base pointers are different, but the indices are the same, just
781     // compare the base pointer.
782     if (PtrBase != GEPRHS->getOperand(0)) {
783       bool IndicesTheSame =
784           GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
785           GEPLHS->getPointerOperand()->getType() ==
786               GEPRHS->getPointerOperand()->getType() &&
787           GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType();
788       if (IndicesTheSame)
789         for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i)
790           if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
791             IndicesTheSame = false;
792             break;
793           }
794 
795       // If all indices are the same, just compare the base pointers.
796       Type *BaseType = GEPLHS->getOperand(0)->getType();
797       if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType())
798         return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0));
799 
800       // If we're comparing GEPs with two base pointers that only differ in type
801       // and both GEPs have only constant indices or just one use, then fold
802       // the compare with the adjusted indices.
803       // FIXME: Support vector of pointers.
804       if (GEPLHS->isInBounds() && GEPRHS->isInBounds() &&
805           (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) &&
806           (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) &&
807           PtrBase->stripPointerCasts() ==
808               GEPRHS->getOperand(0)->stripPointerCasts() &&
809           !GEPLHS->getType()->isVectorTy()) {
810         Value *LOffset = EmitGEPOffset(GEPLHS);
811         Value *ROffset = EmitGEPOffset(GEPRHS);
812 
813         // If we looked through an addrspacecast between different sized address
814         // spaces, the LHS and RHS pointers are different sized
815         // integers. Truncate to the smaller one.
816         Type *LHSIndexTy = LOffset->getType();
817         Type *RHSIndexTy = ROffset->getType();
818         if (LHSIndexTy != RHSIndexTy) {
819           if (LHSIndexTy->getPrimitiveSizeInBits().getFixedValue() <
820               RHSIndexTy->getPrimitiveSizeInBits().getFixedValue()) {
821             ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy);
822           } else
823             LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy);
824         }
825 
826         Value *Cmp = Builder.CreateICmp(ICmpInst::getSignedPredicate(Cond),
827                                         LOffset, ROffset);
828         return replaceInstUsesWith(I, Cmp);
829       }
830 
831       // Otherwise, the base pointers are different and the indices are
832       // different. Try convert this to an indexed compare by looking through
833       // PHIs/casts.
834       return transformToIndexedCompare(GEPLHS, RHS, Cond, DL);
835     }
836 
837     // If one of the GEPs has all zero indices, recurse.
838     // FIXME: Handle vector of pointers.
839     if (!GEPLHS->getType()->isVectorTy() && GEPLHS->hasAllZeroIndices())
840       return foldGEPICmp(GEPRHS, GEPLHS->getOperand(0),
841                          ICmpInst::getSwappedPredicate(Cond), I);
842 
843     // If the other GEP has all zero indices, recurse.
844     // FIXME: Handle vector of pointers.
845     if (!GEPRHS->getType()->isVectorTy() && GEPRHS->hasAllZeroIndices())
846       return foldGEPICmp(GEPLHS, GEPRHS->getOperand(0), Cond, I);
847 
848     bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds();
849     if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
850         GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
851       // If the GEPs only differ by one index, compare it.
852       unsigned NumDifferences = 0;  // Keep track of # differences.
853       unsigned DiffOperand = 0;     // The operand that differs.
854       for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i)
855         if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) {
856           Type *LHSType = GEPLHS->getOperand(i)->getType();
857           Type *RHSType = GEPRHS->getOperand(i)->getType();
858           // FIXME: Better support for vector of pointers.
859           if (LHSType->getPrimitiveSizeInBits() !=
860                    RHSType->getPrimitiveSizeInBits() ||
861               (GEPLHS->getType()->isVectorTy() &&
862                (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) {
863             // Irreconcilable differences.
864             NumDifferences = 2;
865             break;
866           }
867 
868           if (NumDifferences++) break;
869           DiffOperand = i;
870         }
871 
872       if (NumDifferences == 0)   // SAME GEP?
873         return replaceInstUsesWith(I, // No comparison is needed here.
874           ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond)));
875 
876       else if (NumDifferences == 1 && GEPsInBounds) {
877         Value *LHSV = GEPLHS->getOperand(DiffOperand);
878         Value *RHSV = GEPRHS->getOperand(DiffOperand);
879         // Make sure we do a signed comparison here.
880         return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV);
881       }
882     }
883 
884     // Only lower this if the icmp is the only user of the GEP or if we expect
885     // the result to fold to a constant!
886     if (GEPsInBounds && (isa<ConstantExpr>(GEPLHS) || GEPLHS->hasOneUse()) &&
887         (isa<ConstantExpr>(GEPRHS) || GEPRHS->hasOneUse())) {
888       // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)  --->  (OFFSET1 cmp OFFSET2)
889       Value *L = EmitGEPOffset(GEPLHS);
890       Value *R = EmitGEPOffset(GEPRHS);
891       return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R);
892     }
893   }
894 
895   // Try convert this to an indexed compare by looking through PHIs/casts as a
896   // last resort.
897   return transformToIndexedCompare(GEPLHS, RHS, Cond, DL);
898 }
899 
900 Instruction *InstCombinerImpl::foldAllocaCmp(ICmpInst &ICI,
901                                              const AllocaInst *Alloca) {
902   assert(ICI.isEquality() && "Cannot fold non-equality comparison.");
903 
904   // It would be tempting to fold away comparisons between allocas and any
905   // pointer not based on that alloca (e.g. an argument). However, even
906   // though such pointers cannot alias, they can still compare equal.
907   //
908   // But LLVM doesn't specify where allocas get their memory, so if the alloca
909   // doesn't escape we can argue that it's impossible to guess its value, and we
910   // can therefore act as if any such guesses are wrong.
911   //
912   // The code below checks that the alloca doesn't escape, and that it's only
913   // used in a comparison once (the current instruction). The
914   // single-comparison-use condition ensures that we're trivially folding all
915   // comparisons against the alloca consistently, and avoids the risk of
916   // erroneously folding a comparison of the pointer with itself.
917 
918   unsigned MaxIter = 32; // Break cycles and bound to constant-time.
919 
920   SmallVector<const Use *, 32> Worklist;
921   for (const Use &U : Alloca->uses()) {
922     if (Worklist.size() >= MaxIter)
923       return nullptr;
924     Worklist.push_back(&U);
925   }
926 
927   unsigned NumCmps = 0;
928   while (!Worklist.empty()) {
929     assert(Worklist.size() <= MaxIter);
930     const Use *U = Worklist.pop_back_val();
931     const Value *V = U->getUser();
932     --MaxIter;
933 
934     if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V) || isa<PHINode>(V) ||
935         isa<SelectInst>(V)) {
936       // Track the uses.
937     } else if (isa<LoadInst>(V)) {
938       // Loading from the pointer doesn't escape it.
939       continue;
940     } else if (const auto *SI = dyn_cast<StoreInst>(V)) {
941       // Storing *to* the pointer is fine, but storing the pointer escapes it.
942       if (SI->getValueOperand() == U->get())
943         return nullptr;
944       continue;
945     } else if (isa<ICmpInst>(V)) {
946       if (NumCmps++)
947         return nullptr; // Found more than one cmp.
948       continue;
949     } else if (const auto *Intrin = dyn_cast<IntrinsicInst>(V)) {
950       switch (Intrin->getIntrinsicID()) {
951         // These intrinsics don't escape or compare the pointer. Memset is safe
952         // because we don't allow ptrtoint. Memcpy and memmove are safe because
953         // we don't allow stores, so src cannot point to V.
954         case Intrinsic::lifetime_start: case Intrinsic::lifetime_end:
955         case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset:
956           continue;
957         default:
958           return nullptr;
959       }
960     } else {
961       return nullptr;
962     }
963     for (const Use &U : V->uses()) {
964       if (Worklist.size() >= MaxIter)
965         return nullptr;
966       Worklist.push_back(&U);
967     }
968   }
969 
970   auto *Res = ConstantInt::get(ICI.getType(),
971                                !CmpInst::isTrueWhenEqual(ICI.getPredicate()));
972   return replaceInstUsesWith(ICI, Res);
973 }
974 
975 /// Fold "icmp pred (X+C), X".
976 Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C,
977                                                   ICmpInst::Predicate Pred) {
978   // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
979   // so the values can never be equal.  Similarly for all other "or equals"
980   // operators.
981   assert(!!C && "C should not be zero!");
982 
983   // (X+1) <u X        --> X >u (MAXUINT-1)        --> X == 255
984   // (X+2) <u X        --> X >u (MAXUINT-2)        --> X > 253
985   // (X+MAXUINT) <u X  --> X >u (MAXUINT-MAXUINT)  --> X != 0
986   if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
987     Constant *R = ConstantInt::get(X->getType(),
988                                    APInt::getMaxValue(C.getBitWidth()) - C);
989     return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
990   }
991 
992   // (X+1) >u X        --> X <u (0-1)        --> X != 255
993   // (X+2) >u X        --> X <u (0-2)        --> X <u 254
994   // (X+MAXUINT) >u X  --> X <u (0-MAXUINT)  --> X <u 1  --> X == 0
995   if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
996     return new ICmpInst(ICmpInst::ICMP_ULT, X,
997                         ConstantInt::get(X->getType(), -C));
998 
999   APInt SMax = APInt::getSignedMaxValue(C.getBitWidth());
1000 
1001   // (X+ 1) <s X       --> X >s (MAXSINT-1)          --> X == 127
1002   // (X+ 2) <s X       --> X >s (MAXSINT-2)          --> X >s 125
1003   // (X+MAXSINT) <s X  --> X >s (MAXSINT-MAXSINT)    --> X >s 0
1004   // (X+MINSINT) <s X  --> X >s (MAXSINT-MINSINT)    --> X >s -1
1005   // (X+ -2) <s X      --> X >s (MAXSINT- -2)        --> X >s 126
1006   // (X+ -1) <s X      --> X >s (MAXSINT- -1)        --> X != 127
1007   if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
1008     return new ICmpInst(ICmpInst::ICMP_SGT, X,
1009                         ConstantInt::get(X->getType(), SMax - C));
1010 
1011   // (X+ 1) >s X       --> X <s (MAXSINT-(1-1))       --> X != 127
1012   // (X+ 2) >s X       --> X <s (MAXSINT-(2-1))       --> X <s 126
1013   // (X+MAXSINT) >s X  --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1
1014   // (X+MINSINT) >s X  --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2
1015   // (X+ -2) >s X      --> X <s (MAXSINT-(-2-1))      --> X <s -126
1016   // (X+ -1) >s X      --> X <s (MAXSINT-(-1-1))      --> X == -128
1017 
1018   assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE);
1019   return new ICmpInst(ICmpInst::ICMP_SLT, X,
1020                       ConstantInt::get(X->getType(), SMax - (C - 1)));
1021 }
1022 
1023 /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" ->
1024 /// (icmp eq/ne A, Log2(AP2/AP1)) ->
1025 /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)).
1026 Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A,
1027                                                      const APInt &AP1,
1028                                                      const APInt &AP2) {
1029   assert(I.isEquality() && "Cannot fold icmp gt/lt");
1030 
1031   auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1032     if (I.getPredicate() == I.ICMP_NE)
1033       Pred = CmpInst::getInversePredicate(Pred);
1034     return new ICmpInst(Pred, LHS, RHS);
1035   };
1036 
1037   // Don't bother doing any work for cases which InstSimplify handles.
1038   if (AP2.isZero())
1039     return nullptr;
1040 
1041   bool IsAShr = isa<AShrOperator>(I.getOperand(0));
1042   if (IsAShr) {
1043     if (AP2.isAllOnes())
1044       return nullptr;
1045     if (AP2.isNegative() != AP1.isNegative())
1046       return nullptr;
1047     if (AP2.sgt(AP1))
1048       return nullptr;
1049   }
1050 
1051   if (!AP1)
1052     // 'A' must be large enough to shift out the highest set bit.
1053     return getICmp(I.ICMP_UGT, A,
1054                    ConstantInt::get(A->getType(), AP2.logBase2()));
1055 
1056   if (AP1 == AP2)
1057     return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1058 
1059   int Shift;
1060   if (IsAShr && AP1.isNegative())
1061     Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes();
1062   else
1063     Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros();
1064 
1065   if (Shift > 0) {
1066     if (IsAShr && AP1 == AP2.ashr(Shift)) {
1067       // There are multiple solutions if we are comparing against -1 and the LHS
1068       // of the ashr is not a power of two.
1069       if (AP1.isAllOnes() && !AP2.isPowerOf2())
1070         return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift));
1071       return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1072     } else if (AP1 == AP2.lshr(Shift)) {
1073       return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1074     }
1075   }
1076 
1077   // Shifting const2 will never be equal to const1.
1078   // FIXME: This should always be handled by InstSimplify?
1079   auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1080   return replaceInstUsesWith(I, TorF);
1081 }
1082 
1083 /// Handle "(icmp eq/ne (shl AP2, A), AP1)" ->
1084 /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
1085 Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A,
1086                                                      const APInt &AP1,
1087                                                      const APInt &AP2) {
1088   assert(I.isEquality() && "Cannot fold icmp gt/lt");
1089 
1090   auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1091     if (I.getPredicate() == I.ICMP_NE)
1092       Pred = CmpInst::getInversePredicate(Pred);
1093     return new ICmpInst(Pred, LHS, RHS);
1094   };
1095 
1096   // Don't bother doing any work for cases which InstSimplify handles.
1097   if (AP2.isZero())
1098     return nullptr;
1099 
1100   unsigned AP2TrailingZeros = AP2.countTrailingZeros();
1101 
1102   if (!AP1 && AP2TrailingZeros != 0)
1103     return getICmp(
1104         I.ICMP_UGE, A,
1105         ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros));
1106 
1107   if (AP1 == AP2)
1108     return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType()));
1109 
1110   // Get the distance between the lowest bits that are set.
1111   int Shift = AP1.countTrailingZeros() - AP2TrailingZeros;
1112 
1113   if (Shift > 0 && AP2.shl(Shift) == AP1)
1114     return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift));
1115 
1116   // Shifting const2 will never be equal to const1.
1117   // FIXME: This should always be handled by InstSimplify?
1118   auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE);
1119   return replaceInstUsesWith(I, TorF);
1120 }
1121 
1122 /// The caller has matched a pattern of the form:
1123 ///   I = icmp ugt (add (add A, B), CI2), CI1
1124 /// If this is of the form:
1125 ///   sum = a + b
1126 ///   if (sum+128 >u 255)
1127 /// Then replace it with llvm.sadd.with.overflow.i8.
1128 ///
1129 static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
1130                                           ConstantInt *CI2, ConstantInt *CI1,
1131                                           InstCombinerImpl &IC) {
1132   // The transformation we're trying to do here is to transform this into an
1133   // llvm.sadd.with.overflow.  To do this, we have to replace the original add
1134   // with a narrower add, and discard the add-with-constant that is part of the
1135   // range check (if we can't eliminate it, this isn't profitable).
1136 
1137   // In order to eliminate the add-with-constant, the compare can be its only
1138   // use.
1139   Instruction *AddWithCst = cast<Instruction>(I.getOperand(0));
1140   if (!AddWithCst->hasOneUse())
1141     return nullptr;
1142 
1143   // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow.
1144   if (!CI2->getValue().isPowerOf2())
1145     return nullptr;
1146   unsigned NewWidth = CI2->getValue().countTrailingZeros();
1147   if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31)
1148     return nullptr;
1149 
1150   // The width of the new add formed is 1 more than the bias.
1151   ++NewWidth;
1152 
1153   // Check to see that CI1 is an all-ones value with NewWidth bits.
1154   if (CI1->getBitWidth() == NewWidth ||
1155       CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth))
1156     return nullptr;
1157 
1158   // This is only really a signed overflow check if the inputs have been
1159   // sign-extended; check for that condition. For example, if CI2 is 2^31 and
1160   // the operands of the add are 64 bits wide, we need at least 33 sign bits.
1161   if (IC.ComputeMaxSignificantBits(A, 0, &I) > NewWidth ||
1162       IC.ComputeMaxSignificantBits(B, 0, &I) > NewWidth)
1163     return nullptr;
1164 
1165   // In order to replace the original add with a narrower
1166   // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant
1167   // and truncates that discard the high bits of the add.  Verify that this is
1168   // the case.
1169   Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0));
1170   for (User *U : OrigAdd->users()) {
1171     if (U == AddWithCst)
1172       continue;
1173 
1174     // Only accept truncates for now.  We would really like a nice recursive
1175     // predicate like SimplifyDemandedBits, but which goes downwards the use-def
1176     // chain to see which bits of a value are actually demanded.  If the
1177     // original add had another add which was then immediately truncated, we
1178     // could still do the transformation.
1179     TruncInst *TI = dyn_cast<TruncInst>(U);
1180     if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth)
1181       return nullptr;
1182   }
1183 
1184   // If the pattern matches, truncate the inputs to the narrower type and
1185   // use the sadd_with_overflow intrinsic to efficiently compute both the
1186   // result and the overflow bit.
1187   Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
1188   Function *F = Intrinsic::getDeclaration(
1189       I.getModule(), Intrinsic::sadd_with_overflow, NewType);
1190 
1191   InstCombiner::BuilderTy &Builder = IC.Builder;
1192 
1193   // Put the new code above the original add, in case there are any uses of the
1194   // add between the add and the compare.
1195   Builder.SetInsertPoint(OrigAdd);
1196 
1197   Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc");
1198   Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc");
1199   CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd");
1200   Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result");
1201   Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType());
1202 
1203   // The inner add was the result of the narrow add, zero extended to the
1204   // wider type.  Replace it with the result computed by the intrinsic.
1205   IC.replaceInstUsesWith(*OrigAdd, ZExt);
1206   IC.eraseInstFromFunction(*OrigAdd);
1207 
1208   // The original icmp gets replaced with the overflow value.
1209   return ExtractValueInst::Create(Call, 1, "sadd.overflow");
1210 }
1211 
1212 /// If we have:
1213 ///   icmp eq/ne (urem/srem %x, %y), 0
1214 /// iff %y is a power-of-two, we can replace this with a bit test:
1215 ///   icmp eq/ne (and %x, (add %y, -1)), 0
1216 Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) {
1217   // This fold is only valid for equality predicates.
1218   if (!I.isEquality())
1219     return nullptr;
1220   ICmpInst::Predicate Pred;
1221   Value *X, *Y, *Zero;
1222   if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))),
1223                         m_CombineAnd(m_Zero(), m_Value(Zero)))))
1224     return nullptr;
1225   if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I))
1226     return nullptr;
1227   // This may increase instruction count, we don't enforce that Y is a constant.
1228   Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType()));
1229   Value *Masked = Builder.CreateAnd(X, Mask);
1230   return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero);
1231 }
1232 
1233 /// Fold equality-comparison between zero and any (maybe truncated) right-shift
1234 /// by one-less-than-bitwidth into a sign test on the original value.
1235 Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) {
1236   Instruction *Val;
1237   ICmpInst::Predicate Pred;
1238   if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero())))
1239     return nullptr;
1240 
1241   Value *X;
1242   Type *XTy;
1243 
1244   Constant *C;
1245   if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) {
1246     XTy = X->getType();
1247     unsigned XBitWidth = XTy->getScalarSizeInBits();
1248     if (!match(C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ,
1249                                      APInt(XBitWidth, XBitWidth - 1))))
1250       return nullptr;
1251   } else if (isa<BinaryOperator>(Val) &&
1252              (X = reassociateShiftAmtsOfTwoSameDirectionShifts(
1253                   cast<BinaryOperator>(Val), SQ.getWithInstruction(Val),
1254                   /*AnalyzeForSignBitExtraction=*/true))) {
1255     XTy = X->getType();
1256   } else
1257     return nullptr;
1258 
1259   return ICmpInst::Create(Instruction::ICmp,
1260                           Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE
1261                                                     : ICmpInst::ICMP_SLT,
1262                           X, ConstantInt::getNullValue(XTy));
1263 }
1264 
1265 // Handle  icmp pred X, 0
1266 Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
1267   CmpInst::Predicate Pred = Cmp.getPredicate();
1268   if (!match(Cmp.getOperand(1), m_Zero()))
1269     return nullptr;
1270 
1271   // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0)
1272   if (Pred == ICmpInst::ICMP_SGT) {
1273     Value *A, *B;
1274     if (match(Cmp.getOperand(0), m_SMin(m_Value(A), m_Value(B)))) {
1275       if (isKnownPositive(A, DL, 0, &AC, &Cmp, &DT))
1276         return new ICmpInst(Pred, B, Cmp.getOperand(1));
1277       if (isKnownPositive(B, DL, 0, &AC, &Cmp, &DT))
1278         return new ICmpInst(Pred, A, Cmp.getOperand(1));
1279     }
1280   }
1281 
1282   if (Instruction *New = foldIRemByPowerOfTwoToBitTest(Cmp))
1283     return New;
1284 
1285   // Given:
1286   //   icmp eq/ne (urem %x, %y), 0
1287   // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
1288   //   icmp eq/ne %x, 0
1289   Value *X, *Y;
1290   if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) &&
1291       ICmpInst::isEquality(Pred)) {
1292     KnownBits XKnown = computeKnownBits(X, 0, &Cmp);
1293     KnownBits YKnown = computeKnownBits(Y, 0, &Cmp);
1294     if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
1295       return new ICmpInst(Pred, X, Cmp.getOperand(1));
1296   }
1297 
1298   return nullptr;
1299 }
1300 
1301 /// Fold icmp Pred X, C.
1302 /// TODO: This code structure does not make sense. The saturating add fold
1303 /// should be moved to some other helper and extended as noted below (it is also
1304 /// possible that code has been made unnecessary - do we canonicalize IR to
1305 /// overflow/saturating intrinsics or not?).
1306 Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) {
1307   // Match the following pattern, which is a common idiom when writing
1308   // overflow-safe integer arithmetic functions. The source performs an addition
1309   // in wider type and explicitly checks for overflow using comparisons against
1310   // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic.
1311   //
1312   // TODO: This could probably be generalized to handle other overflow-safe
1313   // operations if we worked out the formulas to compute the appropriate magic
1314   // constants.
1315   //
1316   // sum = a + b
1317   // if (sum+128 >u 255)  ...  -> llvm.sadd.with.overflow.i8
1318   CmpInst::Predicate Pred = Cmp.getPredicate();
1319   Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1);
1320   Value *A, *B;
1321   ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI
1322   if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) &&
1323       match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2))))
1324     if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this))
1325       return Res;
1326 
1327   // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...).
1328   Constant *C = dyn_cast<Constant>(Op1);
1329   if (!C)
1330     return nullptr;
1331 
1332   if (auto *Phi = dyn_cast<PHINode>(Op0))
1333     if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) {
1334       Type *Ty = Cmp.getType();
1335       Builder.SetInsertPoint(Phi);
1336       PHINode *NewPhi =
1337           Builder.CreatePHI(Ty, Phi->getNumOperands());
1338       for (BasicBlock *Predecessor : predecessors(Phi->getParent())) {
1339         auto *Input =
1340             cast<Constant>(Phi->getIncomingValueForBlock(Predecessor));
1341         auto *BoolInput = ConstantExpr::getCompare(Pred, Input, C);
1342         NewPhi->addIncoming(BoolInput, Predecessor);
1343       }
1344       NewPhi->takeName(&Cmp);
1345       return replaceInstUsesWith(Cmp, NewPhi);
1346     }
1347 
1348   return nullptr;
1349 }
1350 
1351 /// Canonicalize icmp instructions based on dominating conditions.
1352 Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
1353   // This is a cheap/incomplete check for dominance - just match a single
1354   // predecessor with a conditional branch.
1355   BasicBlock *CmpBB = Cmp.getParent();
1356   BasicBlock *DomBB = CmpBB->getSinglePredecessor();
1357   if (!DomBB)
1358     return nullptr;
1359 
1360   Value *DomCond;
1361   BasicBlock *TrueBB, *FalseBB;
1362   if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB)))
1363     return nullptr;
1364 
1365   assert((TrueBB == CmpBB || FalseBB == CmpBB) &&
1366          "Predecessor block does not point to successor?");
1367 
1368   // The branch should get simplified. Don't bother simplifying this condition.
1369   if (TrueBB == FalseBB)
1370     return nullptr;
1371 
1372   // Try to simplify this compare to T/F based on the dominating condition.
1373   std::optional<bool> Imp =
1374       isImpliedCondition(DomCond, &Cmp, DL, TrueBB == CmpBB);
1375   if (Imp)
1376     return replaceInstUsesWith(Cmp, ConstantInt::get(Cmp.getType(), *Imp));
1377 
1378   CmpInst::Predicate Pred = Cmp.getPredicate();
1379   Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1);
1380   ICmpInst::Predicate DomPred;
1381   const APInt *C, *DomC;
1382   if (match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))) &&
1383       match(Y, m_APInt(C))) {
1384     // We have 2 compares of a variable with constants. Calculate the constant
1385     // ranges of those compares to see if we can transform the 2nd compare:
1386     // DomBB:
1387     //   DomCond = icmp DomPred X, DomC
1388     //   br DomCond, CmpBB, FalseBB
1389     // CmpBB:
1390     //   Cmp = icmp Pred X, C
1391     ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *C);
1392     ConstantRange DominatingCR =
1393         (CmpBB == TrueBB) ? ConstantRange::makeExactICmpRegion(DomPred, *DomC)
1394                           : ConstantRange::makeExactICmpRegion(
1395                                 CmpInst::getInversePredicate(DomPred), *DomC);
1396     ConstantRange Intersection = DominatingCR.intersectWith(CR);
1397     ConstantRange Difference = DominatingCR.difference(CR);
1398     if (Intersection.isEmptySet())
1399       return replaceInstUsesWith(Cmp, Builder.getFalse());
1400     if (Difference.isEmptySet())
1401       return replaceInstUsesWith(Cmp, Builder.getTrue());
1402 
1403     // Canonicalizing a sign bit comparison that gets used in a branch,
1404     // pessimizes codegen by generating branch on zero instruction instead
1405     // of a test and branch. So we avoid canonicalizing in such situations
1406     // because test and branch instruction has better branch displacement
1407     // than compare and branch instruction.
1408     bool UnusedBit;
1409     bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit);
1410     if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp)))
1411       return nullptr;
1412 
1413     // Avoid an infinite loop with min/max canonicalization.
1414     // TODO: This will be unnecessary if we canonicalize to min/max intrinsics.
1415     if (Cmp.hasOneUse() &&
1416         match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value())))
1417       return nullptr;
1418 
1419     if (const APInt *EqC = Intersection.getSingleElement())
1420       return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC));
1421     if (const APInt *NeC = Difference.getSingleElement())
1422       return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC));
1423   }
1424 
1425   return nullptr;
1426 }
1427 
1428 /// Fold icmp (trunc X), C.
1429 Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
1430                                                      TruncInst *Trunc,
1431                                                      const APInt &C) {
1432   ICmpInst::Predicate Pred = Cmp.getPredicate();
1433   Value *X = Trunc->getOperand(0);
1434   if (C.isOne() && C.getBitWidth() > 1) {
1435     // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
1436     Value *V = nullptr;
1437     if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V))))
1438       return new ICmpInst(ICmpInst::ICMP_SLT, V,
1439                           ConstantInt::get(V->getType(), 1));
1440   }
1441 
1442   Type *SrcTy = X->getType();
1443   unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
1444            SrcBits = SrcTy->getScalarSizeInBits();
1445 
1446   // TODO: Handle any shifted constant by subtracting trailing zeros.
1447   // TODO: Handle non-equality predicates.
1448   Value *Y;
1449   if (Cmp.isEquality() && match(X, m_Shl(m_One(), m_Value(Y)))) {
1450     // (trunc (1 << Y) to iN) == 0 --> Y u>= N
1451     // (trunc (1 << Y) to iN) != 0 --> Y u<  N
1452     if (C.isZero()) {
1453       auto NewPred = (Pred == Cmp.ICMP_EQ) ? Cmp.ICMP_UGE : Cmp.ICMP_ULT;
1454       return new ICmpInst(NewPred, Y, ConstantInt::get(SrcTy, DstBits));
1455     }
1456     // (trunc (1 << Y) to iN) == 2**C --> Y == C
1457     // (trunc (1 << Y) to iN) != 2**C --> Y != C
1458     if (C.isPowerOf2())
1459       return new ICmpInst(Pred, Y, ConstantInt::get(SrcTy, C.logBase2()));
1460   }
1461 
1462   if (Cmp.isEquality() && Trunc->hasOneUse()) {
1463     // Canonicalize to a mask and wider compare if the wide type is suitable:
1464     // (trunc X to i8) == C --> (X & 0xff) == (zext C)
1465     if (!SrcTy->isVectorTy() && shouldChangeType(DstBits, SrcBits)) {
1466       Constant *Mask =
1467           ConstantInt::get(SrcTy, APInt::getLowBitsSet(SrcBits, DstBits));
1468       Value *And = Builder.CreateAnd(X, Mask);
1469       Constant *WideC = ConstantInt::get(SrcTy, C.zext(SrcBits));
1470       return new ICmpInst(Pred, And, WideC);
1471     }
1472 
1473     // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all
1474     // of the high bits truncated out of x are known.
1475     KnownBits Known = computeKnownBits(X, 0, &Cmp);
1476 
1477     // If all the high bits are known, we can do this xform.
1478     if ((Known.Zero | Known.One).countLeadingOnes() >= SrcBits - DstBits) {
1479       // Pull in the high bits from known-ones set.
1480       APInt NewRHS = C.zext(SrcBits);
1481       NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits);
1482       return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, NewRHS));
1483     }
1484   }
1485 
1486   // Look through truncated right-shift of the sign-bit for a sign-bit check:
1487   // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0  --> ShOp <  0
1488   // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1
1489   Value *ShOp;
1490   const APInt *ShAmtC;
1491   bool TrueIfSigned;
1492   if (isSignBitCheck(Pred, C, TrueIfSigned) &&
1493       match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) &&
1494       DstBits == SrcBits - ShAmtC->getZExtValue()) {
1495     return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp,
1496                                        ConstantInt::getNullValue(SrcTy))
1497                         : new ICmpInst(ICmpInst::ICMP_SGT, ShOp,
1498                                        ConstantInt::getAllOnesValue(SrcTy));
1499   }
1500 
1501   return nullptr;
1502 }
1503 
1504 /// Fold icmp (xor X, Y), C.
1505 Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp,
1506                                                    BinaryOperator *Xor,
1507                                                    const APInt &C) {
1508   if (Instruction *I = foldICmpXorShiftConst(Cmp, Xor, C))
1509     return I;
1510 
1511   Value *X = Xor->getOperand(0);
1512   Value *Y = Xor->getOperand(1);
1513   const APInt *XorC;
1514   if (!match(Y, m_APInt(XorC)))
1515     return nullptr;
1516 
1517   // If this is a comparison that tests the signbit (X < 0) or (x > -1),
1518   // fold the xor.
1519   ICmpInst::Predicate Pred = Cmp.getPredicate();
1520   bool TrueIfSigned = false;
1521   if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) {
1522 
1523     // If the sign bit of the XorCst is not set, there is no change to
1524     // the operation, just stop using the Xor.
1525     if (!XorC->isNegative())
1526       return replaceOperand(Cmp, 0, X);
1527 
1528     // Emit the opposite comparison.
1529     if (TrueIfSigned)
1530       return new ICmpInst(ICmpInst::ICMP_SGT, X,
1531                           ConstantInt::getAllOnesValue(X->getType()));
1532     else
1533       return new ICmpInst(ICmpInst::ICMP_SLT, X,
1534                           ConstantInt::getNullValue(X->getType()));
1535   }
1536 
1537   if (Xor->hasOneUse()) {
1538     // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask))
1539     if (!Cmp.isEquality() && XorC->isSignMask()) {
1540       Pred = Cmp.getFlippedSignednessPredicate();
1541       return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1542     }
1543 
1544     // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask))
1545     if (!Cmp.isEquality() && XorC->isMaxSignedValue()) {
1546       Pred = Cmp.getFlippedSignednessPredicate();
1547       Pred = Cmp.getSwappedPredicate(Pred);
1548       return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC));
1549     }
1550   }
1551 
1552   // Mask constant magic can eliminate an 'xor' with unsigned compares.
1553   if (Pred == ICmpInst::ICMP_UGT) {
1554     // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2)
1555     if (*XorC == ~C && (C + 1).isPowerOf2())
1556       return new ICmpInst(ICmpInst::ICMP_ULT, X, Y);
1557     // (xor X, C) >u C --> X >u C (when C+1 is a power of 2)
1558     if (*XorC == C && (C + 1).isPowerOf2())
1559       return new ICmpInst(ICmpInst::ICMP_UGT, X, Y);
1560   }
1561   if (Pred == ICmpInst::ICMP_ULT) {
1562     // (xor X, -C) <u C --> X >u ~C (when C is a power of 2)
1563     if (*XorC == -C && C.isPowerOf2())
1564       return new ICmpInst(ICmpInst::ICMP_UGT, X,
1565                           ConstantInt::get(X->getType(), ~C));
1566     // (xor X, C) <u C --> X >u ~C (when -C is a power of 2)
1567     if (*XorC == C && (-C).isPowerOf2())
1568       return new ICmpInst(ICmpInst::ICMP_UGT, X,
1569                           ConstantInt::get(X->getType(), ~C));
1570   }
1571   return nullptr;
1572 }
1573 
1574 /// For power-of-2 C:
1575 /// ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1)
1576 /// ((X s>> ShiftC) ^ X) u> (C - 1) --> (X + C) u> ((C << 1) - 1)
1577 Instruction *InstCombinerImpl::foldICmpXorShiftConst(ICmpInst &Cmp,
1578                                                      BinaryOperator *Xor,
1579                                                      const APInt &C) {
1580   CmpInst::Predicate Pred = Cmp.getPredicate();
1581   APInt PowerOf2;
1582   if (Pred == ICmpInst::ICMP_ULT)
1583     PowerOf2 = C;
1584   else if (Pred == ICmpInst::ICMP_UGT && !C.isMaxValue())
1585     PowerOf2 = C + 1;
1586   else
1587     return nullptr;
1588   if (!PowerOf2.isPowerOf2())
1589     return nullptr;
1590   Value *X;
1591   const APInt *ShiftC;
1592   if (!match(Xor, m_OneUse(m_c_Xor(m_Value(X),
1593                                    m_AShr(m_Deferred(X), m_APInt(ShiftC))))))
1594     return nullptr;
1595   uint64_t Shift = ShiftC->getLimitedValue();
1596   Type *XType = X->getType();
1597   if (Shift == 0 || PowerOf2.isMinSignedValue())
1598     return nullptr;
1599   Value *Add = Builder.CreateAdd(X, ConstantInt::get(XType, PowerOf2));
1600   APInt Bound =
1601       Pred == ICmpInst::ICMP_ULT ? PowerOf2 << 1 : ((PowerOf2 << 1) - 1);
1602   return new ICmpInst(Pred, Add, ConstantInt::get(XType, Bound));
1603 }
1604 
1605 /// Fold icmp (and (sh X, Y), C2), C1.
1606 Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp,
1607                                                 BinaryOperator *And,
1608                                                 const APInt &C1,
1609                                                 const APInt &C2) {
1610   BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0));
1611   if (!Shift || !Shift->isShift())
1612     return nullptr;
1613 
1614   // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could
1615   // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in
1616   // code produced by the clang front-end, for bitfield access.
1617   // This seemingly simple opportunity to fold away a shift turns out to be
1618   // rather complicated. See PR17827 for details.
1619   unsigned ShiftOpcode = Shift->getOpcode();
1620   bool IsShl = ShiftOpcode == Instruction::Shl;
1621   const APInt *C3;
1622   if (match(Shift->getOperand(1), m_APInt(C3))) {
1623     APInt NewAndCst, NewCmpCst;
1624     bool AnyCmpCstBitsShiftedOut;
1625     if (ShiftOpcode == Instruction::Shl) {
1626       // For a left shift, we can fold if the comparison is not signed. We can
1627       // also fold a signed comparison if the mask value and comparison value
1628       // are not negative. These constraints may not be obvious, but we can
1629       // prove that they are correct using an SMT solver.
1630       if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative()))
1631         return nullptr;
1632 
1633       NewCmpCst = C1.lshr(*C3);
1634       NewAndCst = C2.lshr(*C3);
1635       AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1;
1636     } else if (ShiftOpcode == Instruction::LShr) {
1637       // For a logical right shift, we can fold if the comparison is not signed.
1638       // We can also fold a signed comparison if the shifted mask value and the
1639       // shifted comparison value are not negative. These constraints may not be
1640       // obvious, but we can prove that they are correct using an SMT solver.
1641       NewCmpCst = C1.shl(*C3);
1642       NewAndCst = C2.shl(*C3);
1643       AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1;
1644       if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative()))
1645         return nullptr;
1646     } else {
1647       // For an arithmetic shift, check that both constants don't use (in a
1648       // signed sense) the top bits being shifted out.
1649       assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode");
1650       NewCmpCst = C1.shl(*C3);
1651       NewAndCst = C2.shl(*C3);
1652       AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1;
1653       if (NewAndCst.ashr(*C3) != C2)
1654         return nullptr;
1655     }
1656 
1657     if (AnyCmpCstBitsShiftedOut) {
1658       // If we shifted bits out, the fold is not going to work out. As a
1659       // special case, check to see if this means that the result is always
1660       // true or false now.
1661       if (Cmp.getPredicate() == ICmpInst::ICMP_EQ)
1662         return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType()));
1663       if (Cmp.getPredicate() == ICmpInst::ICMP_NE)
1664         return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType()));
1665     } else {
1666       Value *NewAnd = Builder.CreateAnd(
1667           Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst));
1668       return new ICmpInst(Cmp.getPredicate(),
1669           NewAnd, ConstantInt::get(And->getType(), NewCmpCst));
1670     }
1671   }
1672 
1673   // Turn ((X >> Y) & C2) == 0  into  (X & (C2 << Y)) == 0.  The latter is
1674   // preferable because it allows the C2 << Y expression to be hoisted out of a
1675   // loop if Y is invariant and X is not.
1676   if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() &&
1677       !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) {
1678     // Compute C2 << Y.
1679     Value *NewShift =
1680         IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1))
1681               : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1));
1682 
1683     // Compute X & (C2 << Y).
1684     Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift);
1685     return replaceOperand(Cmp, 0, NewAnd);
1686   }
1687 
1688   return nullptr;
1689 }
1690 
1691 /// Fold icmp (and X, C2), C1.
1692 Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
1693                                                      BinaryOperator *And,
1694                                                      const APInt &C1) {
1695   bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
1696 
1697   // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1
1698   // TODO: We canonicalize to the longer form for scalars because we have
1699   // better analysis/folds for icmp, and codegen may be better with icmp.
1700   if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() &&
1701       match(And->getOperand(1), m_One()))
1702     return new TruncInst(And->getOperand(0), Cmp.getType());
1703 
1704   const APInt *C2;
1705   Value *X;
1706   if (!match(And, m_And(m_Value(X), m_APInt(C2))))
1707     return nullptr;
1708 
1709   // Don't perform the following transforms if the AND has multiple uses
1710   if (!And->hasOneUse())
1711     return nullptr;
1712 
1713   if (Cmp.isEquality() && C1.isZero()) {
1714     // Restrict this fold to single-use 'and' (PR10267).
1715     // Replace (and X, (1 << size(X)-1) != 0) with X s< 0
1716     if (C2->isSignMask()) {
1717       Constant *Zero = Constant::getNullValue(X->getType());
1718       auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
1719       return new ICmpInst(NewPred, X, Zero);
1720     }
1721 
1722     APInt NewC2 = *C2;
1723     KnownBits Know = computeKnownBits(And->getOperand(0), 0, And);
1724     // Set high zeros of C2 to allow matching negated power-of-2.
1725     NewC2 = *C2 | APInt::getHighBitsSet(C2->getBitWidth(),
1726                                         Know.countMinLeadingZeros());
1727 
1728     // Restrict this fold only for single-use 'and' (PR10267).
1729     // ((%x & C) == 0) --> %x u< (-C)  iff (-C) is power of two.
1730     if (NewC2.isNegatedPowerOf2()) {
1731       Constant *NegBOC = ConstantInt::get(And->getType(), -NewC2);
1732       auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
1733       return new ICmpInst(NewPred, X, NegBOC);
1734     }
1735   }
1736 
1737   // If the LHS is an 'and' of a truncate and we can widen the and/compare to
1738   // the input width without changing the value produced, eliminate the cast:
1739   //
1740   // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1'
1741   //
1742   // We can do this transformation if the constants do not have their sign bits
1743   // set or if it is an equality comparison. Extending a relational comparison
1744   // when we're checking the sign bit would not work.
1745   Value *W;
1746   if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) &&
1747       (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) {
1748     // TODO: Is this a good transform for vectors? Wider types may reduce
1749     // throughput. Should this transform be limited (even for scalars) by using
1750     // shouldChangeType()?
1751     if (!Cmp.getType()->isVectorTy()) {
1752       Type *WideType = W->getType();
1753       unsigned WideScalarBits = WideType->getScalarSizeInBits();
1754       Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits));
1755       Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits));
1756       Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName());
1757       return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1);
1758     }
1759   }
1760 
1761   if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2))
1762     return I;
1763 
1764   // (icmp pred (and (or (lshr A, B), A), 1), 0) -->
1765   // (icmp pred (and A, (or (shl 1, B), 1), 0))
1766   //
1767   // iff pred isn't signed
1768   if (!Cmp.isSigned() && C1.isZero() && And->getOperand(0)->hasOneUse() &&
1769       match(And->getOperand(1), m_One())) {
1770     Constant *One = cast<Constant>(And->getOperand(1));
1771     Value *Or = And->getOperand(0);
1772     Value *A, *B, *LShr;
1773     if (match(Or, m_Or(m_Value(LShr), m_Value(A))) &&
1774         match(LShr, m_LShr(m_Specific(A), m_Value(B)))) {
1775       unsigned UsesRemoved = 0;
1776       if (And->hasOneUse())
1777         ++UsesRemoved;
1778       if (Or->hasOneUse())
1779         ++UsesRemoved;
1780       if (LShr->hasOneUse())
1781         ++UsesRemoved;
1782 
1783       // Compute A & ((1 << B) | 1)
1784       Value *NewOr = nullptr;
1785       if (auto *C = dyn_cast<Constant>(B)) {
1786         if (UsesRemoved >= 1)
1787           NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One);
1788       } else {
1789         if (UsesRemoved >= 3)
1790           NewOr = Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(),
1791                                                      /*HasNUW=*/true),
1792                                    One, Or->getName());
1793       }
1794       if (NewOr) {
1795         Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName());
1796         return replaceOperand(Cmp, 0, NewAnd);
1797       }
1798     }
1799   }
1800 
1801   return nullptr;
1802 }
1803 
1804 /// Fold icmp (and X, Y), C.
1805 Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp,
1806                                                    BinaryOperator *And,
1807                                                    const APInt &C) {
1808   if (Instruction *I = foldICmpAndConstConst(Cmp, And, C))
1809     return I;
1810 
1811   const ICmpInst::Predicate Pred = Cmp.getPredicate();
1812   bool TrueIfNeg;
1813   if (isSignBitCheck(Pred, C, TrueIfNeg)) {
1814     // ((X - 1) & ~X) <  0 --> X == 0
1815     // ((X - 1) & ~X) >= 0 --> X != 0
1816     Value *X;
1817     if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) &&
1818         match(And->getOperand(1), m_Not(m_Specific(X)))) {
1819       auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1820       return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType()));
1821     }
1822   }
1823 
1824   // TODO: These all require that Y is constant too, so refactor with the above.
1825 
1826   // Try to optimize things like "A[i] & 42 == 0" to index computations.
1827   Value *X = And->getOperand(0);
1828   Value *Y = And->getOperand(1);
1829   if (auto *C2 = dyn_cast<ConstantInt>(Y))
1830     if (auto *LI = dyn_cast<LoadInst>(X))
1831       if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)))
1832         if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
1833           if (Instruction *Res =
1834                   foldCmpLoadFromIndexedGlobal(LI, GEP, GV, Cmp, C2))
1835             return Res;
1836 
1837   if (!Cmp.isEquality())
1838     return nullptr;
1839 
1840   // X & -C == -C -> X >  u ~C
1841   // X & -C != -C -> X <= u ~C
1842   //   iff C is a power of 2
1843   if (Cmp.getOperand(1) == Y && C.isNegatedPowerOf2()) {
1844     auto NewPred =
1845         Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE;
1846     return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1))));
1847   }
1848 
1849   // ((zext i1 X) & Y) == 0 --> !((trunc Y) & X)
1850   // ((zext i1 X) & Y) != 0 -->  ((trunc Y) & X)
1851   // ((zext i1 X) & Y) == 1 -->  ((trunc Y) & X)
1852   // ((zext i1 X) & Y) != 1 --> !((trunc Y) & X)
1853   if (match(And, m_OneUse(m_c_And(m_OneUse(m_ZExt(m_Value(X))), m_Value(Y)))) &&
1854       X->getType()->isIntOrIntVectorTy(1) && (C.isZero() || C.isOne())) {
1855     Value *TruncY = Builder.CreateTrunc(Y, X->getType());
1856     if (C.isZero() ^ (Pred == CmpInst::ICMP_NE)) {
1857       Value *And = Builder.CreateAnd(TruncY, X);
1858       return BinaryOperator::CreateNot(And);
1859     }
1860     return BinaryOperator::CreateAnd(TruncY, X);
1861   }
1862 
1863   return nullptr;
1864 }
1865 
1866 /// Fold icmp (or X, Y), C.
1867 Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp,
1868                                                   BinaryOperator *Or,
1869                                                   const APInt &C) {
1870   ICmpInst::Predicate Pred = Cmp.getPredicate();
1871   if (C.isOne()) {
1872     // icmp slt signum(V) 1 --> icmp slt V, 1
1873     Value *V = nullptr;
1874     if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V))))
1875       return new ICmpInst(ICmpInst::ICMP_SLT, V,
1876                           ConstantInt::get(V->getType(), 1));
1877   }
1878 
1879   Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1);
1880   const APInt *MaskC;
1881   if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) {
1882     if (*MaskC == C && (C + 1).isPowerOf2()) {
1883       // X | C == C --> X <=u C
1884       // X | C != C --> X  >u C
1885       //   iff C+1 is a power of 2 (C is a bitmask of the low bits)
1886       Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
1887       return new ICmpInst(Pred, OrOp0, OrOp1);
1888     }
1889 
1890     // More general: canonicalize 'equality with set bits mask' to
1891     // 'equality with clear bits mask'.
1892     // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC
1893     // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC
1894     if (Or->hasOneUse()) {
1895       Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC));
1896       Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC));
1897       return new ICmpInst(Pred, And, NewC);
1898     }
1899   }
1900 
1901   // (X | (X-1)) s<  0 --> X s< 1
1902   // (X | (X-1)) s> -1 --> X s> 0
1903   Value *X;
1904   bool TrueIfSigned;
1905   if (isSignBitCheck(Pred, C, TrueIfSigned) &&
1906       match(Or, m_c_Or(m_Add(m_Value(X), m_AllOnes()), m_Deferred(X)))) {
1907     auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT;
1908     Constant *NewC = ConstantInt::get(X->getType(), TrueIfSigned ? 1 : 0);
1909     return new ICmpInst(NewPred, X, NewC);
1910   }
1911 
1912   if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse())
1913     return nullptr;
1914 
1915   Value *P, *Q;
1916   if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) {
1917     // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0
1918     // -> and (icmp eq P, null), (icmp eq Q, null).
1919     Value *CmpP =
1920         Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType()));
1921     Value *CmpQ =
1922         Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType()));
1923     auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
1924     return BinaryOperator::Create(BOpc, CmpP, CmpQ);
1925   }
1926 
1927   // Are we using xors to bitwise check for a pair of (in)equalities? Convert to
1928   // a shorter form that has more potential to be folded even further.
1929   Value *X1, *X2, *X3, *X4;
1930   if (match(OrOp0, m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) &&
1931       match(OrOp1, m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) {
1932     // ((X1 ^ X2) || (X3 ^ X4)) == 0 --> (X1 == X2) && (X3 == X4)
1933     // ((X1 ^ X2) || (X3 ^ X4)) != 0 --> (X1 != X2) || (X3 != X4)
1934     Value *Cmp12 = Builder.CreateICmp(Pred, X1, X2);
1935     Value *Cmp34 = Builder.CreateICmp(Pred, X3, X4);
1936     auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
1937     return BinaryOperator::Create(BOpc, Cmp12, Cmp34);
1938   }
1939 
1940   return nullptr;
1941 }
1942 
1943 /// Fold icmp (mul X, Y), C.
1944 Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp,
1945                                                    BinaryOperator *Mul,
1946                                                    const APInt &C) {
1947   ICmpInst::Predicate Pred = Cmp.getPredicate();
1948   Type *MulTy = Mul->getType();
1949   Value *X = Mul->getOperand(0);
1950 
1951   // If there's no overflow:
1952   // X * X == 0 --> X == 0
1953   // X * X != 0 --> X != 0
1954   if (Cmp.isEquality() && C.isZero() && X == Mul->getOperand(1) &&
1955       (Mul->hasNoUnsignedWrap() || Mul->hasNoSignedWrap()))
1956     return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
1957 
1958   const APInt *MulC;
1959   if (!match(Mul->getOperand(1), m_APInt(MulC)))
1960     return nullptr;
1961 
1962   // If this is a test of the sign bit and the multiply is sign-preserving with
1963   // a constant operand, use the multiply LHS operand instead:
1964   // (X * +MulC) < 0 --> X < 0
1965   // (X * -MulC) < 0 --> X > 0
1966   if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) {
1967     if (MulC->isNegative())
1968       Pred = ICmpInst::getSwappedPredicate(Pred);
1969     return new ICmpInst(Pred, X, ConstantInt::getNullValue(MulTy));
1970   }
1971 
1972   if (MulC->isZero() || (!Mul->hasNoSignedWrap() && !Mul->hasNoUnsignedWrap()))
1973     return nullptr;
1974 
1975   // If the multiply does not wrap, try to divide the compare constant by the
1976   // multiplication factor.
1977   if (Cmp.isEquality()) {
1978     // (mul nsw X, MulC) == C --> X == C /s MulC
1979     if (Mul->hasNoSignedWrap() && C.srem(*MulC).isZero()) {
1980       Constant *NewC = ConstantInt::get(MulTy, C.sdiv(*MulC));
1981       return new ICmpInst(Pred, X, NewC);
1982     }
1983     // (mul nuw X, MulC) == C --> X == C /u MulC
1984     if (Mul->hasNoUnsignedWrap() && C.urem(*MulC).isZero()) {
1985       Constant *NewC = ConstantInt::get(MulTy, C.udiv(*MulC));
1986       return new ICmpInst(Pred, X, NewC);
1987     }
1988   }
1989 
1990   // With a matching no-overflow guarantee, fold the constants:
1991   // (X * MulC) < C --> X < (C / MulC)
1992   // (X * MulC) > C --> X > (C / MulC)
1993   // TODO: Assert that Pred is not equal to SGE, SLE, UGE, ULE?
1994   Constant *NewC = nullptr;
1995   if (Mul->hasNoSignedWrap()) {
1996     // MININT / -1 --> overflow.
1997     if (C.isMinSignedValue() && MulC->isAllOnes())
1998       return nullptr;
1999     if (MulC->isNegative())
2000       Pred = ICmpInst::getSwappedPredicate(Pred);
2001 
2002     if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE)
2003       NewC = ConstantInt::get(
2004           MulTy, APIntOps::RoundingSDiv(C, *MulC, APInt::Rounding::UP));
2005     if (Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_SGT)
2006       NewC = ConstantInt::get(
2007           MulTy, APIntOps::RoundingSDiv(C, *MulC, APInt::Rounding::DOWN));
2008   } else {
2009     assert(Mul->hasNoUnsignedWrap() && "Expected mul nuw");
2010     if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)
2011       NewC = ConstantInt::get(
2012           MulTy, APIntOps::RoundingUDiv(C, *MulC, APInt::Rounding::UP));
2013     if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)
2014       NewC = ConstantInt::get(
2015           MulTy, APIntOps::RoundingUDiv(C, *MulC, APInt::Rounding::DOWN));
2016   }
2017 
2018   return NewC ? new ICmpInst(Pred, X, NewC) : nullptr;
2019 }
2020 
2021 /// Fold icmp (shl 1, Y), C.
2022 static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl,
2023                                    const APInt &C) {
2024   Value *Y;
2025   if (!match(Shl, m_Shl(m_One(), m_Value(Y))))
2026     return nullptr;
2027 
2028   Type *ShiftType = Shl->getType();
2029   unsigned TypeBits = C.getBitWidth();
2030   bool CIsPowerOf2 = C.isPowerOf2();
2031   ICmpInst::Predicate Pred = Cmp.getPredicate();
2032   if (Cmp.isUnsigned()) {
2033     // (1 << Y) pred C -> Y pred Log2(C)
2034     if (!CIsPowerOf2) {
2035       // (1 << Y) <  30 -> Y <= 4
2036       // (1 << Y) <= 30 -> Y <= 4
2037       // (1 << Y) >= 30 -> Y >  4
2038       // (1 << Y) >  30 -> Y >  4
2039       if (Pred == ICmpInst::ICMP_ULT)
2040         Pred = ICmpInst::ICMP_ULE;
2041       else if (Pred == ICmpInst::ICMP_UGE)
2042         Pred = ICmpInst::ICMP_UGT;
2043     }
2044 
2045     unsigned CLog2 = C.logBase2();
2046     return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2));
2047   } else if (Cmp.isSigned()) {
2048     Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1);
2049     // (1 << Y) >  0 -> Y != 31
2050     // (1 << Y) >  C -> Y != 31 if C is negative.
2051     if (Pred == ICmpInst::ICMP_SGT && C.sle(0))
2052       return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne);
2053 
2054     // (1 << Y) <  0 -> Y == 31
2055     // (1 << Y) <  1 -> Y == 31
2056     // (1 << Y) <  C -> Y == 31 if C is negative and not signed min.
2057     // Exclude signed min by subtracting 1 and lower the upper bound to 0.
2058     if (Pred == ICmpInst::ICMP_SLT && (C-1).sle(0))
2059       return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
2060   }
2061 
2062   return nullptr;
2063 }
2064 
2065 /// Fold icmp (shl X, Y), C.
2066 Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp,
2067                                                    BinaryOperator *Shl,
2068                                                    const APInt &C) {
2069   const APInt *ShiftVal;
2070   if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal)))
2071     return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal);
2072 
2073   const APInt *ShiftAmt;
2074   if (!match(Shl->getOperand(1), m_APInt(ShiftAmt)))
2075     return foldICmpShlOne(Cmp, Shl, C);
2076 
2077   // Check that the shift amount is in range. If not, don't perform undefined
2078   // shifts. When the shift is visited, it will be simplified.
2079   unsigned TypeBits = C.getBitWidth();
2080   if (ShiftAmt->uge(TypeBits))
2081     return nullptr;
2082 
2083   ICmpInst::Predicate Pred = Cmp.getPredicate();
2084   Value *X = Shl->getOperand(0);
2085   Type *ShType = Shl->getType();
2086 
2087   // NSW guarantees that we are only shifting out sign bits from the high bits,
2088   // so we can ASHR the compare constant without needing a mask and eliminate
2089   // the shift.
2090   if (Shl->hasNoSignedWrap()) {
2091     if (Pred == ICmpInst::ICMP_SGT) {
2092       // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt)
2093       APInt ShiftedC = C.ashr(*ShiftAmt);
2094       return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2095     }
2096     if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2097         C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) {
2098       APInt ShiftedC = C.ashr(*ShiftAmt);
2099       return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2100     }
2101     if (Pred == ICmpInst::ICMP_SLT) {
2102       // SLE is the same as above, but SLE is canonicalized to SLT, so convert:
2103       // (X << S) <=s C is equiv to X <=s (C >> S) for all C
2104       // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX
2105       // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN
2106       assert(!C.isMinSignedValue() && "Unexpected icmp slt");
2107       APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1;
2108       return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2109     }
2110     // If this is a signed comparison to 0 and the shift is sign preserving,
2111     // use the shift LHS operand instead; isSignTest may change 'Pred', so only
2112     // do that if we're sure to not continue on in this function.
2113     if (isSignTest(Pred, C))
2114       return new ICmpInst(Pred, X, Constant::getNullValue(ShType));
2115   }
2116 
2117   // NUW guarantees that we are only shifting out zero bits from the high bits,
2118   // so we can LSHR the compare constant without needing a mask and eliminate
2119   // the shift.
2120   if (Shl->hasNoUnsignedWrap()) {
2121     if (Pred == ICmpInst::ICMP_UGT) {
2122       // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt)
2123       APInt ShiftedC = C.lshr(*ShiftAmt);
2124       return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2125     }
2126     if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2127         C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) {
2128       APInt ShiftedC = C.lshr(*ShiftAmt);
2129       return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2130     }
2131     if (Pred == ICmpInst::ICMP_ULT) {
2132       // ULE is the same as above, but ULE is canonicalized to ULT, so convert:
2133       // (X << S) <=u C is equiv to X <=u (C >> S) for all C
2134       // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u
2135       // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0
2136       assert(C.ugt(0) && "ult 0 should have been eliminated");
2137       APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1;
2138       return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC));
2139     }
2140   }
2141 
2142   if (Cmp.isEquality() && Shl->hasOneUse()) {
2143     // Strength-reduce the shift into an 'and'.
2144     Constant *Mask = ConstantInt::get(
2145         ShType,
2146         APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue()));
2147     Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2148     Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt));
2149     return new ICmpInst(Pred, And, LShrC);
2150   }
2151 
2152   // Otherwise, if this is a comparison of the sign bit, simplify to and/test.
2153   bool TrueIfSigned = false;
2154   if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) {
2155     // (X << 31) <s 0  --> (X & 1) != 0
2156     Constant *Mask = ConstantInt::get(
2157         ShType,
2158         APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1));
2159     Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask");
2160     return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ,
2161                         And, Constant::getNullValue(ShType));
2162   }
2163 
2164   // Simplify 'shl' inequality test into 'and' equality test.
2165   if (Cmp.isUnsigned() && Shl->hasOneUse()) {
2166     // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0
2167     if ((C + 1).isPowerOf2() &&
2168         (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) {
2169       Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue()));
2170       return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ
2171                                                      : ICmpInst::ICMP_NE,
2172                           And, Constant::getNullValue(ShType));
2173     }
2174     // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0
2175     if (C.isPowerOf2() &&
2176         (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
2177       Value *And =
2178           Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue()));
2179       return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ
2180                                                      : ICmpInst::ICMP_NE,
2181                           And, Constant::getNullValue(ShType));
2182     }
2183   }
2184 
2185   // Transform (icmp pred iM (shl iM %v, N), C)
2186   // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N))
2187   // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N.
2188   // This enables us to get rid of the shift in favor of a trunc that may be
2189   // free on the target. It has the additional benefit of comparing to a
2190   // smaller constant that may be more target-friendly.
2191   unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1);
2192   if (Shl->hasOneUse() && Amt != 0 && C.countTrailingZeros() >= Amt &&
2193       DL.isLegalInteger(TypeBits - Amt)) {
2194     Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt);
2195     if (auto *ShVTy = dyn_cast<VectorType>(ShType))
2196       TruncTy = VectorType::get(TruncTy, ShVTy->getElementCount());
2197     Constant *NewC =
2198         ConstantInt::get(TruncTy, C.ashr(*ShiftAmt).trunc(TypeBits - Amt));
2199     return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC);
2200   }
2201 
2202   return nullptr;
2203 }
2204 
2205 /// Fold icmp ({al}shr X, Y), C.
2206 Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp,
2207                                                    BinaryOperator *Shr,
2208                                                    const APInt &C) {
2209   // An exact shr only shifts out zero bits, so:
2210   // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
2211   Value *X = Shr->getOperand(0);
2212   CmpInst::Predicate Pred = Cmp.getPredicate();
2213   if (Cmp.isEquality() && Shr->isExact() && C.isZero())
2214     return new ICmpInst(Pred, X, Cmp.getOperand(1));
2215 
2216   bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2217   const APInt *ShiftValC;
2218   if (match(X, m_APInt(ShiftValC))) {
2219     if (Cmp.isEquality())
2220       return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftValC);
2221 
2222     // (ShiftValC >> Y) >s -1 --> Y != 0 with ShiftValC < 0
2223     // (ShiftValC >> Y) <s  0 --> Y == 0 with ShiftValC < 0
2224     bool TrueIfSigned;
2225     if (!IsAShr && ShiftValC->isNegative() &&
2226         isSignBitCheck(Pred, C, TrueIfSigned))
2227       return new ICmpInst(TrueIfSigned ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE,
2228                           Shr->getOperand(1),
2229                           ConstantInt::getNullValue(X->getType()));
2230 
2231     // If the shifted constant is a power-of-2, test the shift amount directly:
2232     // (ShiftValC >> Y) >u C --> X <u (LZ(C) - LZ(ShiftValC))
2233     // (ShiftValC >> Y) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC))
2234     if (!IsAShr && ShiftValC->isPowerOf2() &&
2235         (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) {
2236       bool IsUGT = Pred == CmpInst::ICMP_UGT;
2237       assert(ShiftValC->uge(C) && "Expected simplify of compare");
2238       assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify");
2239 
2240       unsigned CmpLZ =
2241           IsUGT ? C.countLeadingZeros() : (C - 1).countLeadingZeros();
2242       unsigned ShiftLZ = ShiftValC->countLeadingZeros();
2243       Constant *NewC = ConstantInt::get(Shr->getType(), CmpLZ - ShiftLZ);
2244       auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
2245       return new ICmpInst(NewPred, Shr->getOperand(1), NewC);
2246     }
2247   }
2248 
2249   const APInt *ShiftAmtC;
2250   if (!match(Shr->getOperand(1), m_APInt(ShiftAmtC)))
2251     return nullptr;
2252 
2253   // Check that the shift amount is in range. If not, don't perform undefined
2254   // shifts. When the shift is visited it will be simplified.
2255   unsigned TypeBits = C.getBitWidth();
2256   unsigned ShAmtVal = ShiftAmtC->getLimitedValue(TypeBits);
2257   if (ShAmtVal >= TypeBits || ShAmtVal == 0)
2258     return nullptr;
2259 
2260   bool IsExact = Shr->isExact();
2261   Type *ShrTy = Shr->getType();
2262   // TODO: If we could guarantee that InstSimplify would handle all of the
2263   // constant-value-based preconditions in the folds below, then we could assert
2264   // those conditions rather than checking them. This is difficult because of
2265   // undef/poison (PR34838).
2266   if (IsAShr) {
2267     if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) {
2268       // When ShAmtC can be shifted losslessly:
2269       // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC)
2270       // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC)
2271       APInt ShiftedC = C.shl(ShAmtVal);
2272       if (ShiftedC.ashr(ShAmtVal) == C)
2273         return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2274     }
2275     if (Pred == CmpInst::ICMP_SGT) {
2276       // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1
2277       APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2278       if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() &&
2279           (ShiftedC + 1).ashr(ShAmtVal) == (C + 1))
2280         return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2281     }
2282     if (Pred == CmpInst::ICMP_UGT) {
2283       // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2284       // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd
2285       // clause accounts for that pattern.
2286       APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2287       if ((ShiftedC + 1).ashr(ShAmtVal) == (C + 1) ||
2288           (C + 1).shl(ShAmtVal).isMinSignedValue())
2289         return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2290     }
2291 
2292     // If the compare constant has significant bits above the lowest sign-bit,
2293     // then convert an unsigned cmp to a test of the sign-bit:
2294     // (ashr X, ShiftC) u> C --> X s< 0
2295     // (ashr X, ShiftC) u< C --> X s> -1
2296     if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) {
2297       if (Pred == CmpInst::ICMP_UGT) {
2298         return new ICmpInst(CmpInst::ICMP_SLT, X,
2299                             ConstantInt::getNullValue(ShrTy));
2300       }
2301       if (Pred == CmpInst::ICMP_ULT) {
2302         return new ICmpInst(CmpInst::ICMP_SGT, X,
2303                             ConstantInt::getAllOnesValue(ShrTy));
2304       }
2305     }
2306   } else {
2307     if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) {
2308       // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC)
2309       // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC)
2310       APInt ShiftedC = C.shl(ShAmtVal);
2311       if (ShiftedC.lshr(ShAmtVal) == C)
2312         return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2313     }
2314     if (Pred == CmpInst::ICMP_UGT) {
2315       // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2316       APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1;
2317       if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1))
2318         return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
2319     }
2320   }
2321 
2322   if (!Cmp.isEquality())
2323     return nullptr;
2324 
2325   // Handle equality comparisons of shift-by-constant.
2326 
2327   // If the comparison constant changes with the shift, the comparison cannot
2328   // succeed (bits of the comparison constant cannot match the shifted value).
2329   // This should be known by InstSimplify and already be folded to true/false.
2330   assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||
2331           (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&
2332          "Expected icmp+shr simplify did not occur.");
2333 
2334   // If the bits shifted out are known zero, compare the unshifted value:
2335   //  (X & 4) >> 1 == 2  --> (X & 4) == 4.
2336   if (Shr->isExact())
2337     return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal));
2338 
2339   if (C.isZero()) {
2340     // == 0 is u< 1.
2341     if (Pred == CmpInst::ICMP_EQ)
2342       return new ICmpInst(CmpInst::ICMP_ULT, X,
2343                           ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal)));
2344     else
2345       return new ICmpInst(CmpInst::ICMP_UGT, X,
2346                           ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1));
2347   }
2348 
2349   if (Shr->hasOneUse()) {
2350     // Canonicalize the shift into an 'and':
2351     // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt)
2352     APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal));
2353     Constant *Mask = ConstantInt::get(ShrTy, Val);
2354     Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask");
2355     return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal));
2356   }
2357 
2358   return nullptr;
2359 }
2360 
2361 Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp,
2362                                                     BinaryOperator *SRem,
2363                                                     const APInt &C) {
2364   // Match an 'is positive' or 'is negative' comparison of remainder by a
2365   // constant power-of-2 value:
2366   // (X % pow2C) sgt/slt 0
2367   const ICmpInst::Predicate Pred = Cmp.getPredicate();
2368   if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT &&
2369       Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
2370     return nullptr;
2371 
2372   // TODO: The one-use check is standard because we do not typically want to
2373   //       create longer instruction sequences, but this might be a special-case
2374   //       because srem is not good for analysis or codegen.
2375   if (!SRem->hasOneUse())
2376     return nullptr;
2377 
2378   const APInt *DivisorC;
2379   if (!match(SRem->getOperand(1), m_Power2(DivisorC)))
2380     return nullptr;
2381 
2382   // For cmp_sgt/cmp_slt only zero valued C is handled.
2383   // For cmp_eq/cmp_ne only positive valued C is handled.
2384   if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) &&
2385        !C.isZero()) ||
2386       ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2387        !C.isStrictlyPositive()))
2388     return nullptr;
2389 
2390   // Mask off the sign bit and the modulo bits (low-bits).
2391   Type *Ty = SRem->getType();
2392   APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits());
2393   Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1));
2394   Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC);
2395 
2396   if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)
2397     return new ICmpInst(Pred, And, ConstantInt::get(Ty, C));
2398 
2399   // For 'is positive?' check that the sign-bit is clear and at least 1 masked
2400   // bit is set. Example:
2401   // (i8 X % 32) s> 0 --> (X & 159) s> 0
2402   if (Pred == ICmpInst::ICMP_SGT)
2403     return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty));
2404 
2405   // For 'is negative?' check that the sign-bit is set and at least 1 masked
2406   // bit is set. Example:
2407   // (i16 X % 4) s< 0 --> (X & 32771) u> 32768
2408   return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask));
2409 }
2410 
2411 /// Fold icmp (udiv X, Y), C.
2412 Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp,
2413                                                     BinaryOperator *UDiv,
2414                                                     const APInt &C) {
2415   ICmpInst::Predicate Pred = Cmp.getPredicate();
2416   Value *X = UDiv->getOperand(0);
2417   Value *Y = UDiv->getOperand(1);
2418   Type *Ty = UDiv->getType();
2419 
2420   const APInt *C2;
2421   if (!match(X, m_APInt(C2)))
2422     return nullptr;
2423 
2424   assert(*C2 != 0 && "udiv 0, X should have been simplified already.");
2425 
2426   // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1))
2427   if (Pred == ICmpInst::ICMP_UGT) {
2428     assert(!C.isMaxValue() &&
2429            "icmp ugt X, UINT_MAX should have been simplified already.");
2430     return new ICmpInst(ICmpInst::ICMP_ULE, Y,
2431                         ConstantInt::get(Ty, C2->udiv(C + 1)));
2432   }
2433 
2434   // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C)
2435   if (Pred == ICmpInst::ICMP_ULT) {
2436     assert(C != 0 && "icmp ult X, 0 should have been simplified already.");
2437     return new ICmpInst(ICmpInst::ICMP_UGT, Y,
2438                         ConstantInt::get(Ty, C2->udiv(C)));
2439   }
2440 
2441   return nullptr;
2442 }
2443 
2444 /// Fold icmp ({su}div X, Y), C.
2445 Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp,
2446                                                    BinaryOperator *Div,
2447                                                    const APInt &C) {
2448   ICmpInst::Predicate Pred = Cmp.getPredicate();
2449   Value *X = Div->getOperand(0);
2450   Value *Y = Div->getOperand(1);
2451   Type *Ty = Div->getType();
2452   bool DivIsSigned = Div->getOpcode() == Instruction::SDiv;
2453 
2454   // If unsigned division and the compare constant is bigger than
2455   // UMAX/2 (negative), there's only one pair of values that satisfies an
2456   // equality check, so eliminate the division:
2457   // (X u/ Y) == C --> (X == C) && (Y == 1)
2458   // (X u/ Y) != C --> (X != C) || (Y != 1)
2459   // Similarly, if signed division and the compare constant is exactly SMIN:
2460   // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1)
2461   // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1)
2462   if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() &&
2463       (!DivIsSigned || C.isMinSignedValue()))   {
2464     Value *XBig = Builder.CreateICmp(Pred, X, ConstantInt::get(Ty, C));
2465     Value *YOne = Builder.CreateICmp(Pred, Y, ConstantInt::get(Ty, 1));
2466     auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2467     return BinaryOperator::Create(Logic, XBig, YOne);
2468   }
2469 
2470   // Fold: icmp pred ([us]div X, C2), C -> range test
2471   // Fold this div into the comparison, producing a range check.
2472   // Determine, based on the divide type, what the range is being
2473   // checked.  If there is an overflow on the low or high side, remember
2474   // it, otherwise compute the range [low, hi) bounding the new value.
2475   // See: InsertRangeTest above for the kinds of replacements possible.
2476   const APInt *C2;
2477   if (!match(Y, m_APInt(C2)))
2478     return nullptr;
2479 
2480   // FIXME: If the operand types don't match the type of the divide
2481   // then don't attempt this transform. The code below doesn't have the
2482   // logic to deal with a signed divide and an unsigned compare (and
2483   // vice versa). This is because (x /s C2) <s C  produces different
2484   // results than (x /s C2) <u C or (x /u C2) <s C or even
2485   // (x /u C2) <u C.  Simply casting the operands and result won't
2486   // work. :(  The if statement below tests that condition and bails
2487   // if it finds it.
2488   if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned())
2489     return nullptr;
2490 
2491   // The ProdOV computation fails on divide by 0 and divide by -1. Cases with
2492   // INT_MIN will also fail if the divisor is 1. Although folds of all these
2493   // division-by-constant cases should be present, we can not assert that they
2494   // have happened before we reach this icmp instruction.
2495   if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes()))
2496     return nullptr;
2497 
2498   // Compute Prod = C * C2. We are essentially solving an equation of
2499   // form X / C2 = C. We solve for X by multiplying C2 and C.
2500   // By solving for X, we can turn this into a range check instead of computing
2501   // a divide.
2502   APInt Prod = C * *C2;
2503 
2504   // Determine if the product overflows by seeing if the product is not equal to
2505   // the divide. Make sure we do the same kind of divide as in the LHS
2506   // instruction that we're folding.
2507   bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C;
2508 
2509   // If the division is known to be exact, then there is no remainder from the
2510   // divide, so the covered range size is unit, otherwise it is the divisor.
2511   APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2;
2512 
2513   // Figure out the interval that is being checked.  For example, a comparison
2514   // like "X /u 5 == 0" is really checking that X is in the interval [0, 5).
2515   // Compute this interval based on the constants involved and the signedness of
2516   // the compare/divide.  This computes a half-open interval, keeping track of
2517   // whether either value in the interval overflows.  After analysis each
2518   // overflow variable is set to 0 if it's corresponding bound variable is valid
2519   // -1 if overflowed off the bottom end, or +1 if overflowed off the top end.
2520   int LoOverflow = 0, HiOverflow = 0;
2521   APInt LoBound, HiBound;
2522 
2523   if (!DivIsSigned) { // udiv
2524     // e.g. X/5 op 3  --> [15, 20)
2525     LoBound = Prod;
2526     HiOverflow = LoOverflow = ProdOV;
2527     if (!HiOverflow) {
2528       // If this is not an exact divide, then many values in the range collapse
2529       // to the same result value.
2530       HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false);
2531     }
2532   } else if (C2->isStrictlyPositive()) { // Divisor is > 0.
2533     if (C.isZero()) {                    // (X / pos) op 0
2534       // Can't overflow.  e.g.  X/2 op 0 --> [-1, 2)
2535       LoBound = -(RangeSize - 1);
2536       HiBound = RangeSize;
2537     } else if (C.isStrictlyPositive()) { // (X / pos) op pos
2538       LoBound = Prod;                    // e.g.   X/5 op 3 --> [15, 20)
2539       HiOverflow = LoOverflow = ProdOV;
2540       if (!HiOverflow)
2541         HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true);
2542     } else { // (X / pos) op neg
2543       // e.g. X/5 op -3  --> [-15-4, -15+1) --> [-19, -14)
2544       HiBound = Prod + 1;
2545       LoOverflow = HiOverflow = ProdOV ? -1 : 0;
2546       if (!LoOverflow) {
2547         APInt DivNeg = -RangeSize;
2548         LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0;
2549       }
2550     }
2551   } else if (C2->isNegative()) { // Divisor is < 0.
2552     if (Div->isExact())
2553       RangeSize.negate();
2554     if (C.isZero()) { // (X / neg) op 0
2555       // e.g. X/-5 op 0  --> [-4, 5)
2556       LoBound = RangeSize + 1;
2557       HiBound = -RangeSize;
2558       if (HiBound == *C2) { // -INTMIN = INTMIN
2559         HiOverflow = 1;     // [INTMIN+1, overflow)
2560         HiBound = APInt();  // e.g. X/INTMIN = 0 --> X > INTMIN
2561       }
2562     } else if (C.isStrictlyPositive()) { // (X / neg) op pos
2563       // e.g. X/-5 op 3  --> [-19, -14)
2564       HiBound = Prod + 1;
2565       HiOverflow = LoOverflow = ProdOV ? -1 : 0;
2566       if (!LoOverflow)
2567         LoOverflow =
2568             addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1 : 0;
2569     } else {          // (X / neg) op neg
2570       LoBound = Prod; // e.g. X/-5 op -3  --> [15, 20)
2571       LoOverflow = HiOverflow = ProdOV;
2572       if (!HiOverflow)
2573         HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true);
2574     }
2575 
2576     // Dividing by a negative swaps the condition.  LT <-> GT
2577     Pred = ICmpInst::getSwappedPredicate(Pred);
2578   }
2579 
2580   switch (Pred) {
2581   default:
2582     llvm_unreachable("Unhandled icmp predicate!");
2583   case ICmpInst::ICMP_EQ:
2584     if (LoOverflow && HiOverflow)
2585       return replaceInstUsesWith(Cmp, Builder.getFalse());
2586     if (HiOverflow)
2587       return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2588                           X, ConstantInt::get(Ty, LoBound));
2589     if (LoOverflow)
2590       return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2591                           X, ConstantInt::get(Ty, HiBound));
2592     return replaceInstUsesWith(
2593         Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true));
2594   case ICmpInst::ICMP_NE:
2595     if (LoOverflow && HiOverflow)
2596       return replaceInstUsesWith(Cmp, Builder.getTrue());
2597     if (HiOverflow)
2598       return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2599                           X, ConstantInt::get(Ty, LoBound));
2600     if (LoOverflow)
2601       return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2602                           X, ConstantInt::get(Ty, HiBound));
2603     return replaceInstUsesWith(
2604         Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, false));
2605   case ICmpInst::ICMP_ULT:
2606   case ICmpInst::ICMP_SLT:
2607     if (LoOverflow == +1) // Low bound is greater than input range.
2608       return replaceInstUsesWith(Cmp, Builder.getTrue());
2609     if (LoOverflow == -1) // Low bound is less than input range.
2610       return replaceInstUsesWith(Cmp, Builder.getFalse());
2611     return new ICmpInst(Pred, X, ConstantInt::get(Ty, LoBound));
2612   case ICmpInst::ICMP_UGT:
2613   case ICmpInst::ICMP_SGT:
2614     if (HiOverflow == +1) // High bound greater than input range.
2615       return replaceInstUsesWith(Cmp, Builder.getFalse());
2616     if (HiOverflow == -1) // High bound less than input range.
2617       return replaceInstUsesWith(Cmp, Builder.getTrue());
2618     if (Pred == ICmpInst::ICMP_UGT)
2619       return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, HiBound));
2620     return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, HiBound));
2621   }
2622 
2623   return nullptr;
2624 }
2625 
2626 /// Fold icmp (sub X, Y), C.
2627 Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp,
2628                                                    BinaryOperator *Sub,
2629                                                    const APInt &C) {
2630   Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1);
2631   ICmpInst::Predicate Pred = Cmp.getPredicate();
2632   Type *Ty = Sub->getType();
2633 
2634   // (SubC - Y) == C) --> Y == (SubC - C)
2635   // (SubC - Y) != C) --> Y != (SubC - C)
2636   Constant *SubC;
2637   if (Cmp.isEquality() && match(X, m_ImmConstant(SubC))) {
2638     return new ICmpInst(Pred, Y,
2639                         ConstantExpr::getSub(SubC, ConstantInt::get(Ty, C)));
2640   }
2641 
2642   // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C)
2643   const APInt *C2;
2644   APInt SubResult;
2645   ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate();
2646   bool HasNSW = Sub->hasNoSignedWrap();
2647   bool HasNUW = Sub->hasNoUnsignedWrap();
2648   if (match(X, m_APInt(C2)) &&
2649       ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) &&
2650       !subWithOverflow(SubResult, *C2, C, Cmp.isSigned()))
2651     return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, SubResult));
2652 
2653   // X - Y == 0 --> X == Y.
2654   // X - Y != 0 --> X != Y.
2655   // TODO: We allow this with multiple uses as long as the other uses are not
2656   //       in phis. The phi use check is guarding against a codegen regression
2657   //       for a loop test. If the backend could undo this (and possibly
2658   //       subsequent transforms), we would not need this hack.
2659   if (Cmp.isEquality() && C.isZero() &&
2660       none_of((Sub->users()), [](const User *U) { return isa<PHINode>(U); }))
2661     return new ICmpInst(Pred, X, Y);
2662 
2663   // The following transforms are only worth it if the only user of the subtract
2664   // is the icmp.
2665   // TODO: This is an artificial restriction for all of the transforms below
2666   //       that only need a single replacement icmp. Can these use the phi test
2667   //       like the transform above here?
2668   if (!Sub->hasOneUse())
2669     return nullptr;
2670 
2671   if (Sub->hasNoSignedWrap()) {
2672     // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y)
2673     if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
2674       return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
2675 
2676     // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y)
2677     if (Pred == ICmpInst::ICMP_SGT && C.isZero())
2678       return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
2679 
2680     // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y)
2681     if (Pred == ICmpInst::ICMP_SLT && C.isZero())
2682       return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
2683 
2684     // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y)
2685     if (Pred == ICmpInst::ICMP_SLT && C.isOne())
2686       return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
2687   }
2688 
2689   if (!match(X, m_APInt(C2)))
2690     return nullptr;
2691 
2692   // C2 - Y <u C -> (Y | (C - 1)) == C2
2693   //   iff (C2 & (C - 1)) == C - 1 and C is a power of 2
2694   if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() &&
2695       (*C2 & (C - 1)) == (C - 1))
2696     return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X);
2697 
2698   // C2 - Y >u C -> (Y | C) != C2
2699   //   iff C2 & C == C and C + 1 is a power of 2
2700   if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C)
2701     return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X);
2702 
2703   // We have handled special cases that reduce.
2704   // Canonicalize any remaining sub to add as:
2705   // (C2 - Y) > C --> (Y + ~C2) < ~C
2706   Value *Add = Builder.CreateAdd(Y, ConstantInt::get(Ty, ~(*C2)), "notsub",
2707                                  HasNUW, HasNSW);
2708   return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, ~C));
2709 }
2710 
2711 /// Fold icmp (add X, Y), C.
2712 Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp,
2713                                                    BinaryOperator *Add,
2714                                                    const APInt &C) {
2715   Value *Y = Add->getOperand(1);
2716   const APInt *C2;
2717   if (Cmp.isEquality() || !match(Y, m_APInt(C2)))
2718     return nullptr;
2719 
2720   // Fold icmp pred (add X, C2), C.
2721   Value *X = Add->getOperand(0);
2722   Type *Ty = Add->getType();
2723   const CmpInst::Predicate Pred = Cmp.getPredicate();
2724 
2725   // If the add does not wrap, we can always adjust the compare by subtracting
2726   // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE
2727   // are canonicalized to SGT/SLT/UGT/ULT.
2728   if ((Add->hasNoSignedWrap() &&
2729        (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) ||
2730       (Add->hasNoUnsignedWrap() &&
2731        (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) {
2732     bool Overflow;
2733     APInt NewC =
2734         Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow);
2735     // If there is overflow, the result must be true or false.
2736     // TODO: Can we assert there is no overflow because InstSimplify always
2737     // handles those cases?
2738     if (!Overflow)
2739       // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2)
2740       return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC));
2741   }
2742 
2743   auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2);
2744   const APInt &Upper = CR.getUpper();
2745   const APInt &Lower = CR.getLower();
2746   if (Cmp.isSigned()) {
2747     if (Lower.isSignMask())
2748       return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper));
2749     if (Upper.isSignMask())
2750       return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower));
2751   } else {
2752     if (Lower.isMinValue())
2753       return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper));
2754     if (Upper.isMinValue())
2755       return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower));
2756   }
2757 
2758   // This set of folds is intentionally placed after folds that use no-wrapping
2759   // flags because those folds are likely better for later analysis/codegen.
2760   const APInt SMax = APInt::getSignedMaxValue(Ty->getScalarSizeInBits());
2761   const APInt SMin = APInt::getSignedMinValue(Ty->getScalarSizeInBits());
2762 
2763   // Fold compare with offset to opposite sign compare if it eliminates offset:
2764   // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX)
2765   if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax)
2766     return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2)));
2767 
2768   // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN)
2769   if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin)
2770     return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2)));
2771 
2772   // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1)
2773   if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1)
2774     return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C));
2775 
2776   // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2)
2777   if (Pred == CmpInst::ICMP_SLT && C == *C2)
2778     return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax));
2779 
2780   // (X + -1) <u C --> X <=u C (if X is never null)
2781   if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
2782     const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
2783     if (llvm::isKnownNonZero(X, DL, 0, Q.AC, Q.CxtI, Q.DT))
2784       return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, C));
2785   }
2786 
2787   if (!Add->hasOneUse())
2788     return nullptr;
2789 
2790   // X+C <u C2 -> (X & -C2) == C
2791   //   iff C & (C2-1) == 0
2792   //       C2 is a power of 2
2793   if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0)
2794     return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -C),
2795                         ConstantExpr::getNeg(cast<Constant>(Y)));
2796 
2797   // X+C >u C2 -> (X & ~C2) != C
2798   //   iff C & C2 == 0
2799   //       C2+1 is a power of 2
2800   if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0)
2801     return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~C),
2802                         ConstantExpr::getNeg(cast<Constant>(Y)));
2803 
2804   // The range test idiom can use either ult or ugt. Arbitrarily canonicalize
2805   // to the ult form.
2806   // X+C2 >u C -> X+(C2-C-1) <u ~C
2807   if (Pred == ICmpInst::ICMP_UGT)
2808     return new ICmpInst(ICmpInst::ICMP_ULT,
2809                         Builder.CreateAdd(X, ConstantInt::get(Ty, *C2 - C - 1)),
2810                         ConstantInt::get(Ty, ~C));
2811 
2812   return nullptr;
2813 }
2814 
2815 bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS,
2816                                                Value *&RHS, ConstantInt *&Less,
2817                                                ConstantInt *&Equal,
2818                                                ConstantInt *&Greater) {
2819   // TODO: Generalize this to work with other comparison idioms or ensure
2820   // they get canonicalized into this form.
2821 
2822   // select i1 (a == b),
2823   //        i32 Equal,
2824   //        i32 (select i1 (a < b), i32 Less, i32 Greater)
2825   // where Equal, Less and Greater are placeholders for any three constants.
2826   ICmpInst::Predicate PredA;
2827   if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) ||
2828       !ICmpInst::isEquality(PredA))
2829     return false;
2830   Value *EqualVal = SI->getTrueValue();
2831   Value *UnequalVal = SI->getFalseValue();
2832   // We still can get non-canonical predicate here, so canonicalize.
2833   if (PredA == ICmpInst::ICMP_NE)
2834     std::swap(EqualVal, UnequalVal);
2835   if (!match(EqualVal, m_ConstantInt(Equal)))
2836     return false;
2837   ICmpInst::Predicate PredB;
2838   Value *LHS2, *RHS2;
2839   if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)),
2840                                   m_ConstantInt(Less), m_ConstantInt(Greater))))
2841     return false;
2842   // We can get predicate mismatch here, so canonicalize if possible:
2843   // First, ensure that 'LHS' match.
2844   if (LHS2 != LHS) {
2845     // x sgt y <--> y slt x
2846     std::swap(LHS2, RHS2);
2847     PredB = ICmpInst::getSwappedPredicate(PredB);
2848   }
2849   if (LHS2 != LHS)
2850     return false;
2851   // We also need to canonicalize 'RHS'.
2852   if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) {
2853     // x sgt C-1  <-->  x sge C  <-->  not(x slt C)
2854     auto FlippedStrictness =
2855         InstCombiner::getFlippedStrictnessPredicateAndConstant(
2856             PredB, cast<Constant>(RHS2));
2857     if (!FlippedStrictness)
2858       return false;
2859     assert(FlippedStrictness->first == ICmpInst::ICMP_SGE &&
2860            "basic correctness failure");
2861     RHS2 = FlippedStrictness->second;
2862     // And kind-of perform the result swap.
2863     std::swap(Less, Greater);
2864     PredB = ICmpInst::ICMP_SLT;
2865   }
2866   return PredB == ICmpInst::ICMP_SLT && RHS == RHS2;
2867 }
2868 
2869 Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp,
2870                                                       SelectInst *Select,
2871                                                       ConstantInt *C) {
2872 
2873   assert(C && "Cmp RHS should be a constant int!");
2874   // If we're testing a constant value against the result of a three way
2875   // comparison, the result can be expressed directly in terms of the
2876   // original values being compared.  Note: We could possibly be more
2877   // aggressive here and remove the hasOneUse test. The original select is
2878   // really likely to simplify or sink when we remove a test of the result.
2879   Value *OrigLHS, *OrigRHS;
2880   ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan;
2881   if (Cmp.hasOneUse() &&
2882       matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal,
2883                               C3GreaterThan)) {
2884     assert(C1LessThan && C2Equal && C3GreaterThan);
2885 
2886     bool TrueWhenLessThan =
2887         ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C)
2888             ->isAllOnesValue();
2889     bool TrueWhenEqual =
2890         ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C)
2891             ->isAllOnesValue();
2892     bool TrueWhenGreaterThan =
2893         ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C)
2894             ->isAllOnesValue();
2895 
2896     // This generates the new instruction that will replace the original Cmp
2897     // Instruction. Instead of enumerating the various combinations when
2898     // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus
2899     // false, we rely on chaining of ORs and future passes of InstCombine to
2900     // simplify the OR further (i.e. a s< b || a == b becomes a s<= b).
2901 
2902     // When none of the three constants satisfy the predicate for the RHS (C),
2903     // the entire original Cmp can be simplified to a false.
2904     Value *Cond = Builder.getFalse();
2905     if (TrueWhenLessThan)
2906       Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT,
2907                                                        OrigLHS, OrigRHS));
2908     if (TrueWhenEqual)
2909       Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ,
2910                                                        OrigLHS, OrigRHS));
2911     if (TrueWhenGreaterThan)
2912       Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT,
2913                                                        OrigLHS, OrigRHS));
2914 
2915     return replaceInstUsesWith(Cmp, Cond);
2916   }
2917   return nullptr;
2918 }
2919 
2920 Instruction *InstCombinerImpl::foldICmpBitCast(ICmpInst &Cmp) {
2921   auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0));
2922   if (!Bitcast)
2923     return nullptr;
2924 
2925   ICmpInst::Predicate Pred = Cmp.getPredicate();
2926   Value *Op1 = Cmp.getOperand(1);
2927   Value *BCSrcOp = Bitcast->getOperand(0);
2928   Type *SrcType = Bitcast->getSrcTy();
2929   Type *DstType = Bitcast->getType();
2930 
2931   // Make sure the bitcast doesn't change between scalar and vector and
2932   // doesn't change the number of vector elements.
2933   if (SrcType->isVectorTy() == DstType->isVectorTy() &&
2934       SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) {
2935     // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
2936     Value *X;
2937     if (match(BCSrcOp, m_SIToFP(m_Value(X)))) {
2938       // icmp  eq (bitcast (sitofp X)), 0 --> icmp  eq X, 0
2939       // icmp  ne (bitcast (sitofp X)), 0 --> icmp  ne X, 0
2940       // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
2941       // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
2942       if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
2943            Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
2944           match(Op1, m_Zero()))
2945         return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
2946 
2947       // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
2948       if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One()))
2949         return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1));
2950 
2951       // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
2952       if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes()))
2953         return new ICmpInst(Pred, X,
2954                             ConstantInt::getAllOnesValue(X->getType()));
2955     }
2956 
2957     // Zero-equality checks are preserved through unsigned floating-point casts:
2958     // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
2959     // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
2960     if (match(BCSrcOp, m_UIToFP(m_Value(X))))
2961       if (Cmp.isEquality() && match(Op1, m_Zero()))
2962         return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
2963 
2964     // If this is a sign-bit test of a bitcast of a casted FP value, eliminate
2965     // the FP extend/truncate because that cast does not change the sign-bit.
2966     // This is true for all standard IEEE-754 types and the X86 80-bit type.
2967     // The sign-bit is always the most significant bit in those types.
2968     const APInt *C;
2969     bool TrueIfSigned;
2970     if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse() &&
2971         isSignBitCheck(Pred, *C, TrueIfSigned)) {
2972       if (match(BCSrcOp, m_FPExt(m_Value(X))) ||
2973           match(BCSrcOp, m_FPTrunc(m_Value(X)))) {
2974         // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
2975         // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1
2976         Type *XType = X->getType();
2977 
2978         // We can't currently handle Power style floating point operations here.
2979         if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) {
2980           Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits());
2981           if (auto *XVTy = dyn_cast<VectorType>(XType))
2982             NewType = VectorType::get(NewType, XVTy->getElementCount());
2983           Value *NewBitcast = Builder.CreateBitCast(X, NewType);
2984           if (TrueIfSigned)
2985             return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast,
2986                                 ConstantInt::getNullValue(NewType));
2987           else
2988             return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast,
2989                                 ConstantInt::getAllOnesValue(NewType));
2990         }
2991       }
2992     }
2993   }
2994 
2995   // Test to see if the operands of the icmp are casted versions of other
2996   // values. If the ptr->ptr cast can be stripped off both arguments, do so.
2997   if (DstType->isPointerTy() && (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) {
2998     // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast
2999     // so eliminate it as well.
3000     if (auto *BC2 = dyn_cast<BitCastInst>(Op1))
3001       Op1 = BC2->getOperand(0);
3002 
3003     Op1 = Builder.CreateBitCast(Op1, SrcType);
3004     return new ICmpInst(Pred, BCSrcOp, Op1);
3005   }
3006 
3007   const APInt *C;
3008   if (!match(Cmp.getOperand(1), m_APInt(C)) || !DstType->isIntegerTy() ||
3009       !SrcType->isIntOrIntVectorTy())
3010     return nullptr;
3011 
3012   // If this is checking if all elements of a vector compare are set or not,
3013   // invert the casted vector equality compare and test if all compare
3014   // elements are clear or not. Compare against zero is generally easier for
3015   // analysis and codegen.
3016   // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
3017   // Example: are all elements equal? --> are zero elements not equal?
3018   // TODO: Try harder to reduce compare of 2 freely invertible operands?
3019   if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse() &&
3020       isFreeToInvert(BCSrcOp, BCSrcOp->hasOneUse())) {
3021     Value *Cast = Builder.CreateBitCast(Builder.CreateNot(BCSrcOp), DstType);
3022     return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(DstType));
3023   }
3024 
3025   // If this is checking if all elements of an extended vector are clear or not,
3026   // compare in a narrow type to eliminate the extend:
3027   // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0
3028   Value *X;
3029   if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() &&
3030       match(BCSrcOp, m_ZExtOrSExt(m_Value(X)))) {
3031     if (auto *VecTy = dyn_cast<FixedVectorType>(X->getType())) {
3032       Type *NewType = Builder.getIntNTy(VecTy->getPrimitiveSizeInBits());
3033       Value *NewCast = Builder.CreateBitCast(X, NewType);
3034       return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(NewType));
3035     }
3036   }
3037 
3038   // Folding: icmp <pred> iN X, C
3039   //  where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
3040   //    and C is a splat of a K-bit pattern
3041   //    and SC is a constant vector = <C', C', C', ..., C'>
3042   // Into:
3043   //   %E = extractelement <M x iK> %vec, i32 C'
3044   //   icmp <pred> iK %E, trunc(C)
3045   Value *Vec;
3046   ArrayRef<int> Mask;
3047   if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
3048     // Check whether every element of Mask is the same constant
3049     if (all_equal(Mask)) {
3050       auto *VecTy = cast<VectorType>(SrcType);
3051       auto *EltTy = cast<IntegerType>(VecTy->getElementType());
3052       if (C->isSplat(EltTy->getBitWidth())) {
3053         // Fold the icmp based on the value of C
3054         // If C is M copies of an iK sized bit pattern,
3055         // then:
3056         //   =>  %E = extractelement <N x iK> %vec, i32 Elem
3057         //       icmp <pred> iK %SplatVal, <pattern>
3058         Value *Elem = Builder.getInt32(Mask[0]);
3059         Value *Extract = Builder.CreateExtractElement(Vec, Elem);
3060         Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
3061         return new ICmpInst(Pred, Extract, NewC);
3062       }
3063     }
3064   }
3065   return nullptr;
3066 }
3067 
3068 /// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3069 /// where X is some kind of instruction.
3070 Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) {
3071   const APInt *C;
3072 
3073   if (match(Cmp.getOperand(1), m_APInt(C))) {
3074     if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0)))
3075       if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, *C))
3076         return I;
3077 
3078     if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0)))
3079       // For now, we only support constant integers while folding the
3080       // ICMP(SELECT)) pattern. We can extend this to support vector of integers
3081       // similar to the cases handled by binary ops above.
3082       if (auto *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1)))
3083         if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS))
3084           return I;
3085 
3086     if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0)))
3087       if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C))
3088         return I;
3089 
3090     if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0)))
3091       if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C))
3092         return I;
3093 
3094     // (extractval ([s/u]subo X, Y), 0) == 0 --> X == Y
3095     // (extractval ([s/u]subo X, Y), 0) != 0 --> X != Y
3096     // TODO: This checks one-use, but that is not strictly necessary.
3097     Value *Cmp0 = Cmp.getOperand(0);
3098     Value *X, *Y;
3099     if (C->isZero() && Cmp.isEquality() && Cmp0->hasOneUse() &&
3100         (match(Cmp0,
3101                m_ExtractValue<0>(m_Intrinsic<Intrinsic::ssub_with_overflow>(
3102                    m_Value(X), m_Value(Y)))) ||
3103          match(Cmp0,
3104                m_ExtractValue<0>(m_Intrinsic<Intrinsic::usub_with_overflow>(
3105                    m_Value(X), m_Value(Y))))))
3106       return new ICmpInst(Cmp.getPredicate(), X, Y);
3107   }
3108 
3109   if (match(Cmp.getOperand(1), m_APIntAllowUndef(C)))
3110     return foldICmpInstWithConstantAllowUndef(Cmp, *C);
3111 
3112   return nullptr;
3113 }
3114 
3115 /// Fold an icmp equality instruction with binary operator LHS and constant RHS:
3116 /// icmp eq/ne BO, C.
3117 Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant(
3118     ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
3119   // TODO: Some of these folds could work with arbitrary constants, but this
3120   // function is limited to scalar and vector splat constants.
3121   if (!Cmp.isEquality())
3122     return nullptr;
3123 
3124   ICmpInst::Predicate Pred = Cmp.getPredicate();
3125   bool isICMP_NE = Pred == ICmpInst::ICMP_NE;
3126   Constant *RHS = cast<Constant>(Cmp.getOperand(1));
3127   Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1);
3128 
3129   switch (BO->getOpcode()) {
3130   case Instruction::SRem:
3131     // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one.
3132     if (C.isZero() && BO->hasOneUse()) {
3133       const APInt *BOC;
3134       if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) {
3135         Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName());
3136         return new ICmpInst(Pred, NewRem,
3137                             Constant::getNullValue(BO->getType()));
3138       }
3139     }
3140     break;
3141   case Instruction::Add: {
3142     // (A + C2) == C --> A == (C - C2)
3143     // (A + C2) != C --> A != (C - C2)
3144     // TODO: Remove the one-use limitation? See discussion in D58633.
3145     if (Constant *C2 = dyn_cast<Constant>(BOp1)) {
3146       if (BO->hasOneUse())
3147         return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, C2));
3148     } else if (C.isZero()) {
3149       // Replace ((add A, B) != 0) with (A != -B) if A or B is
3150       // efficiently invertible, or if the add has just this one use.
3151       if (Value *NegVal = dyn_castNegVal(BOp1))
3152         return new ICmpInst(Pred, BOp0, NegVal);
3153       if (Value *NegVal = dyn_castNegVal(BOp0))
3154         return new ICmpInst(Pred, NegVal, BOp1);
3155       if (BO->hasOneUse()) {
3156         Value *Neg = Builder.CreateNeg(BOp1);
3157         Neg->takeName(BO);
3158         return new ICmpInst(Pred, BOp0, Neg);
3159       }
3160     }
3161     break;
3162   }
3163   case Instruction::Xor:
3164     if (BO->hasOneUse()) {
3165       if (Constant *BOC = dyn_cast<Constant>(BOp1)) {
3166         // For the xor case, we can xor two constants together, eliminating
3167         // the explicit xor.
3168         return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC));
3169       } else if (C.isZero()) {
3170         // Replace ((xor A, B) != 0) with (A != B)
3171         return new ICmpInst(Pred, BOp0, BOp1);
3172       }
3173     }
3174     break;
3175   case Instruction::Or: {
3176     const APInt *BOC;
3177     if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) {
3178       // Comparing if all bits outside of a constant mask are set?
3179       // Replace (X | C) == -1 with (X & ~C) == ~C.
3180       // This removes the -1 constant.
3181       Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1));
3182       Value *And = Builder.CreateAnd(BOp0, NotBOC);
3183       return new ICmpInst(Pred, And, NotBOC);
3184     }
3185     break;
3186   }
3187   case Instruction::UDiv:
3188     if (C.isZero()) {
3189       // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A)
3190       auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3191       return new ICmpInst(NewPred, BOp1, BOp0);
3192     }
3193     break;
3194   default:
3195     break;
3196   }
3197   return nullptr;
3198 }
3199 
3200 /// Fold an equality icmp with LLVM intrinsic and constant operand.
3201 Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant(
3202     ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
3203   Type *Ty = II->getType();
3204   unsigned BitWidth = C.getBitWidth();
3205   const ICmpInst::Predicate Pred = Cmp.getPredicate();
3206 
3207   switch (II->getIntrinsicID()) {
3208   case Intrinsic::abs:
3209     // abs(A) == 0  ->  A == 0
3210     // abs(A) == INT_MIN  ->  A == INT_MIN
3211     if (C.isZero() || C.isMinSignedValue())
3212       return new ICmpInst(Pred, II->getArgOperand(0), ConstantInt::get(Ty, C));
3213     break;
3214 
3215   case Intrinsic::bswap:
3216     // bswap(A) == C  ->  A == bswap(C)
3217     return new ICmpInst(Pred, II->getArgOperand(0),
3218                         ConstantInt::get(Ty, C.byteSwap()));
3219 
3220   case Intrinsic::ctlz:
3221   case Intrinsic::cttz: {
3222     // ctz(A) == bitwidth(A)  ->  A == 0 and likewise for !=
3223     if (C == BitWidth)
3224       return new ICmpInst(Pred, II->getArgOperand(0),
3225                           ConstantInt::getNullValue(Ty));
3226 
3227     // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set
3228     // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits.
3229     // Limit to one use to ensure we don't increase instruction count.
3230     unsigned Num = C.getLimitedValue(BitWidth);
3231     if (Num != BitWidth && II->hasOneUse()) {
3232       bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz;
3233       APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1)
3234                                : APInt::getHighBitsSet(BitWidth, Num + 1);
3235       APInt Mask2 = IsTrailing
3236         ? APInt::getOneBitSet(BitWidth, Num)
3237         : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
3238       return new ICmpInst(Pred, Builder.CreateAnd(II->getArgOperand(0), Mask1),
3239                           ConstantInt::get(Ty, Mask2));
3240     }
3241     break;
3242   }
3243 
3244   case Intrinsic::ctpop: {
3245     // popcount(A) == 0  ->  A == 0 and likewise for !=
3246     // popcount(A) == bitwidth(A)  ->  A == -1 and likewise for !=
3247     bool IsZero = C.isZero();
3248     if (IsZero || C == BitWidth)
3249       return new ICmpInst(Pred, II->getArgOperand(0),
3250                           IsZero ? Constant::getNullValue(Ty)
3251                                  : Constant::getAllOnesValue(Ty));
3252 
3253     break;
3254   }
3255 
3256   case Intrinsic::fshl:
3257   case Intrinsic::fshr:
3258     if (II->getArgOperand(0) == II->getArgOperand(1)) {
3259       const APInt *RotAmtC;
3260       // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC)
3261       // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC)
3262       if (match(II->getArgOperand(2), m_APInt(RotAmtC)))
3263         return new ICmpInst(Pred, II->getArgOperand(0),
3264                             II->getIntrinsicID() == Intrinsic::fshl
3265                                 ? ConstantInt::get(Ty, C.rotr(*RotAmtC))
3266                                 : ConstantInt::get(Ty, C.rotl(*RotAmtC)));
3267     }
3268     break;
3269 
3270   case Intrinsic::uadd_sat: {
3271     // uadd.sat(a, b) == 0  ->  (a | b) == 0
3272     if (C.isZero()) {
3273       Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1));
3274       return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
3275     }
3276     break;
3277   }
3278 
3279   case Intrinsic::usub_sat: {
3280     // usub.sat(a, b) == 0  ->  a <= b
3281     if (C.isZero()) {
3282       ICmpInst::Predicate NewPred =
3283           Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3284       return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1));
3285     }
3286     break;
3287   }
3288   default:
3289     break;
3290   }
3291 
3292   return nullptr;
3293 }
3294 
3295 /// Fold an icmp with LLVM intrinsics
3296 static Instruction *foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp) {
3297   assert(Cmp.isEquality());
3298 
3299   ICmpInst::Predicate Pred = Cmp.getPredicate();
3300   Value *Op0 = Cmp.getOperand(0);
3301   Value *Op1 = Cmp.getOperand(1);
3302   const auto *IIOp0 = dyn_cast<IntrinsicInst>(Op0);
3303   const auto *IIOp1 = dyn_cast<IntrinsicInst>(Op1);
3304   if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID())
3305     return nullptr;
3306 
3307   switch (IIOp0->getIntrinsicID()) {
3308   case Intrinsic::bswap:
3309   case Intrinsic::bitreverse:
3310     // If both operands are byte-swapped or bit-reversed, just compare the
3311     // original values.
3312     return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3313   case Intrinsic::fshl:
3314   case Intrinsic::fshr:
3315     // If both operands are rotated by same amount, just compare the
3316     // original values.
3317     if (IIOp0->getOperand(0) != IIOp0->getOperand(1))
3318       break;
3319     if (IIOp1->getOperand(0) != IIOp1->getOperand(1))
3320       break;
3321     if (IIOp0->getOperand(2) != IIOp1->getOperand(2))
3322       break;
3323     return new ICmpInst(Pred, IIOp0->getOperand(0), IIOp1->getOperand(0));
3324   default:
3325     break;
3326   }
3327 
3328   return nullptr;
3329 }
3330 
3331 /// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3332 /// where X is some kind of instruction and C is AllowUndef.
3333 /// TODO: Move more folds which allow undef to this function.
3334 Instruction *
3335 InstCombinerImpl::foldICmpInstWithConstantAllowUndef(ICmpInst &Cmp,
3336                                                      const APInt &C) {
3337   const ICmpInst::Predicate Pred = Cmp.getPredicate();
3338   if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) {
3339     switch (II->getIntrinsicID()) {
3340     default:
3341       break;
3342     case Intrinsic::fshl:
3343     case Intrinsic::fshr:
3344       if (Cmp.isEquality() && II->getArgOperand(0) == II->getArgOperand(1)) {
3345         // (rot X, ?) == 0/-1 --> X == 0/-1
3346         if (C.isZero() || C.isAllOnes())
3347           return new ICmpInst(Pred, II->getArgOperand(0), Cmp.getOperand(1));
3348       }
3349       break;
3350     }
3351   }
3352 
3353   return nullptr;
3354 }
3355 
3356 /// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
3357 Instruction *InstCombinerImpl::foldICmpBinOpWithConstant(ICmpInst &Cmp,
3358                                                          BinaryOperator *BO,
3359                                                          const APInt &C) {
3360   switch (BO->getOpcode()) {
3361   case Instruction::Xor:
3362     if (Instruction *I = foldICmpXorConstant(Cmp, BO, C))
3363       return I;
3364     break;
3365   case Instruction::And:
3366     if (Instruction *I = foldICmpAndConstant(Cmp, BO, C))
3367       return I;
3368     break;
3369   case Instruction::Or:
3370     if (Instruction *I = foldICmpOrConstant(Cmp, BO, C))
3371       return I;
3372     break;
3373   case Instruction::Mul:
3374     if (Instruction *I = foldICmpMulConstant(Cmp, BO, C))
3375       return I;
3376     break;
3377   case Instruction::Shl:
3378     if (Instruction *I = foldICmpShlConstant(Cmp, BO, C))
3379       return I;
3380     break;
3381   case Instruction::LShr:
3382   case Instruction::AShr:
3383     if (Instruction *I = foldICmpShrConstant(Cmp, BO, C))
3384       return I;
3385     break;
3386   case Instruction::SRem:
3387     if (Instruction *I = foldICmpSRemConstant(Cmp, BO, C))
3388       return I;
3389     break;
3390   case Instruction::UDiv:
3391     if (Instruction *I = foldICmpUDivConstant(Cmp, BO, C))
3392       return I;
3393     [[fallthrough]];
3394   case Instruction::SDiv:
3395     if (Instruction *I = foldICmpDivConstant(Cmp, BO, C))
3396       return I;
3397     break;
3398   case Instruction::Sub:
3399     if (Instruction *I = foldICmpSubConstant(Cmp, BO, C))
3400       return I;
3401     break;
3402   case Instruction::Add:
3403     if (Instruction *I = foldICmpAddConstant(Cmp, BO, C))
3404       return I;
3405     break;
3406   default:
3407     break;
3408   }
3409 
3410   // TODO: These folds could be refactored to be part of the above calls.
3411   return foldICmpBinOpEqualityWithConstant(Cmp, BO, C);
3412 }
3413 
3414 /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
3415 Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
3416                                                              IntrinsicInst *II,
3417                                                              const APInt &C) {
3418   if (Cmp.isEquality())
3419     return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
3420 
3421   Type *Ty = II->getType();
3422   unsigned BitWidth = C.getBitWidth();
3423   ICmpInst::Predicate Pred = Cmp.getPredicate();
3424   switch (II->getIntrinsicID()) {
3425   case Intrinsic::ctpop: {
3426     // (ctpop X > BitWidth - 1) --> X == -1
3427     Value *X = II->getArgOperand(0);
3428     if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT)
3429       return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X,
3430                              ConstantInt::getAllOnesValue(Ty));
3431     // (ctpop X < BitWidth) --> X != -1
3432     if (C == BitWidth && Pred == ICmpInst::ICMP_ULT)
3433       return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X,
3434                              ConstantInt::getAllOnesValue(Ty));
3435     break;
3436   }
3437   case Intrinsic::ctlz: {
3438     // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000
3439     if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
3440       unsigned Num = C.getLimitedValue();
3441       APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
3442       return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT,
3443                              II->getArgOperand(0), ConstantInt::get(Ty, Limit));
3444     }
3445 
3446     // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111
3447     if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
3448       unsigned Num = C.getLimitedValue();
3449       APInt Limit = APInt::getLowBitsSet(BitWidth, BitWidth - Num);
3450       return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT,
3451                              II->getArgOperand(0), ConstantInt::get(Ty, Limit));
3452     }
3453     break;
3454   }
3455   case Intrinsic::cttz: {
3456     // Limit to one use to ensure we don't increase instruction count.
3457     if (!II->hasOneUse())
3458       return nullptr;
3459 
3460     // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0
3461     if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
3462       APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1);
3463       return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ,
3464                              Builder.CreateAnd(II->getArgOperand(0), Mask),
3465                              ConstantInt::getNullValue(Ty));
3466     }
3467 
3468     // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0
3469     if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) {
3470       APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue());
3471       return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE,
3472                              Builder.CreateAnd(II->getArgOperand(0), Mask),
3473                              ConstantInt::getNullValue(Ty));
3474     }
3475     break;
3476   }
3477   default:
3478     break;
3479   }
3480 
3481   return nullptr;
3482 }
3483 
3484 /// Handle icmp with constant (but not simple integer constant) RHS.
3485 Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) {
3486   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
3487   Constant *RHSC = dyn_cast<Constant>(Op1);
3488   Instruction *LHSI = dyn_cast<Instruction>(Op0);
3489   if (!RHSC || !LHSI)
3490     return nullptr;
3491 
3492   switch (LHSI->getOpcode()) {
3493   case Instruction::GetElementPtr:
3494     // icmp pred GEP (P, int 0, int 0, int 0), null -> icmp pred P, null
3495     if (RHSC->isNullValue() &&
3496         cast<GetElementPtrInst>(LHSI)->hasAllZeroIndices())
3497       return new ICmpInst(
3498           I.getPredicate(), LHSI->getOperand(0),
3499           Constant::getNullValue(LHSI->getOperand(0)->getType()));
3500     break;
3501   case Instruction::PHI:
3502     // Only fold icmp into the PHI if the phi and icmp are in the same
3503     // block.  If in the same block, we're encouraging jump threading.  If
3504     // not, we are just pessimizing the code by making an i1 phi.
3505     if (LHSI->getParent() == I.getParent())
3506       if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
3507         return NV;
3508     break;
3509   case Instruction::IntToPtr:
3510     // icmp pred inttoptr(X), null -> icmp pred X, 0
3511     if (RHSC->isNullValue() &&
3512         DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType())
3513       return new ICmpInst(
3514           I.getPredicate(), LHSI->getOperand(0),
3515           Constant::getNullValue(LHSI->getOperand(0)->getType()));
3516     break;
3517 
3518   case Instruction::Load:
3519     // Try to optimize things like "A[i] > 4" to index computations.
3520     if (GetElementPtrInst *GEP =
3521             dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
3522       if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
3523         if (Instruction *Res =
3524                 foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, GV, I))
3525           return Res;
3526     break;
3527   }
3528 
3529   return nullptr;
3530 }
3531 
3532 Instruction *InstCombinerImpl::foldSelectICmp(ICmpInst::Predicate Pred,
3533                                               SelectInst *SI, Value *RHS,
3534                                               const ICmpInst &I) {
3535   // Try to fold the comparison into the select arms, which will cause the
3536   // select to be converted into a logical and/or.
3537   auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * {
3538     if (Value *Res = simplifyICmpInst(Pred, Op, RHS, SQ))
3539       return Res;
3540     if (std::optional<bool> Impl = isImpliedCondition(
3541             SI->getCondition(), Pred, Op, RHS, DL, SelectCondIsTrue))
3542       return ConstantInt::get(I.getType(), *Impl);
3543     return nullptr;
3544   };
3545 
3546   ConstantInt *CI = nullptr;
3547   Value *Op1 = SimplifyOp(SI->getOperand(1), true);
3548   if (Op1)
3549     CI = dyn_cast<ConstantInt>(Op1);
3550 
3551   Value *Op2 = SimplifyOp(SI->getOperand(2), false);
3552   if (Op2)
3553     CI = dyn_cast<ConstantInt>(Op2);
3554 
3555   // We only want to perform this transformation if it will not lead to
3556   // additional code. This is true if either both sides of the select
3557   // fold to a constant (in which case the icmp is replaced with a select
3558   // which will usually simplify) or this is the only user of the
3559   // select (in which case we are trading a select+icmp for a simpler
3560   // select+icmp) or all uses of the select can be replaced based on
3561   // dominance information ("Global cases").
3562   bool Transform = false;
3563   if (Op1 && Op2)
3564     Transform = true;
3565   else if (Op1 || Op2) {
3566     // Local case
3567     if (SI->hasOneUse())
3568       Transform = true;
3569     // Global cases
3570     else if (CI && !CI->isZero())
3571       // When Op1 is constant try replacing select with second operand.
3572       // Otherwise Op2 is constant and try replacing select with first
3573       // operand.
3574       Transform = replacedSelectWithOperand(SI, &I, Op1 ? 2 : 1);
3575   }
3576   if (Transform) {
3577     if (!Op1)
3578       Op1 = Builder.CreateICmp(Pred, SI->getOperand(1), RHS, I.getName());
3579     if (!Op2)
3580       Op2 = Builder.CreateICmp(Pred, SI->getOperand(2), RHS, I.getName());
3581     return SelectInst::Create(SI->getOperand(0), Op1, Op2);
3582   }
3583 
3584   return nullptr;
3585 }
3586 
3587 /// Some comparisons can be simplified.
3588 /// In this case, we are looking for comparisons that look like
3589 /// a check for a lossy truncation.
3590 /// Folds:
3591 ///   icmp SrcPred (x & Mask), x    to    icmp DstPred x, Mask
3592 /// Where Mask is some pattern that produces all-ones in low bits:
3593 ///    (-1 >> y)
3594 ///    ((-1 << y) >> y)     <- non-canonical, has extra uses
3595 ///   ~(-1 << y)
3596 ///    ((1 << y) + (-1))    <- non-canonical, has extra uses
3597 /// The Mask can be a constant, too.
3598 /// For some predicates, the operands are commutative.
3599 /// For others, x can only be on a specific side.
3600 static Value *foldICmpWithLowBitMaskedVal(ICmpInst &I,
3601                                           InstCombiner::BuilderTy &Builder) {
3602   ICmpInst::Predicate SrcPred;
3603   Value *X, *M, *Y;
3604   auto m_VariableMask = m_CombineOr(
3605       m_CombineOr(m_Not(m_Shl(m_AllOnes(), m_Value())),
3606                   m_Add(m_Shl(m_One(), m_Value()), m_AllOnes())),
3607       m_CombineOr(m_LShr(m_AllOnes(), m_Value()),
3608                   m_LShr(m_Shl(m_AllOnes(), m_Value(Y)), m_Deferred(Y))));
3609   auto m_Mask = m_CombineOr(m_VariableMask, m_LowBitMask());
3610   if (!match(&I, m_c_ICmp(SrcPred,
3611                           m_c_And(m_CombineAnd(m_Mask, m_Value(M)), m_Value(X)),
3612                           m_Deferred(X))))
3613     return nullptr;
3614 
3615   ICmpInst::Predicate DstPred;
3616   switch (SrcPred) {
3617   case ICmpInst::Predicate::ICMP_EQ:
3618     //  x & (-1 >> y) == x    ->    x u<= (-1 >> y)
3619     DstPred = ICmpInst::Predicate::ICMP_ULE;
3620     break;
3621   case ICmpInst::Predicate::ICMP_NE:
3622     //  x & (-1 >> y) != x    ->    x u> (-1 >> y)
3623     DstPred = ICmpInst::Predicate::ICMP_UGT;
3624     break;
3625   case ICmpInst::Predicate::ICMP_ULT:
3626     //  x & (-1 >> y) u< x    ->    x u> (-1 >> y)
3627     //  x u> x & (-1 >> y)    ->    x u> (-1 >> y)
3628     DstPred = ICmpInst::Predicate::ICMP_UGT;
3629     break;
3630   case ICmpInst::Predicate::ICMP_UGE:
3631     //  x & (-1 >> y) u>= x    ->    x u<= (-1 >> y)
3632     //  x u<= x & (-1 >> y)    ->    x u<= (-1 >> y)
3633     DstPred = ICmpInst::Predicate::ICMP_ULE;
3634     break;
3635   case ICmpInst::Predicate::ICMP_SLT:
3636     //  x & (-1 >> y) s< x    ->    x s> (-1 >> y)
3637     //  x s> x & (-1 >> y)    ->    x s> (-1 >> y)
3638     if (!match(M, m_Constant())) // Can not do this fold with non-constant.
3639       return nullptr;
3640     if (!match(M, m_NonNegative())) // Must not have any -1 vector elements.
3641       return nullptr;
3642     DstPred = ICmpInst::Predicate::ICMP_SGT;
3643     break;
3644   case ICmpInst::Predicate::ICMP_SGE:
3645     //  x & (-1 >> y) s>= x    ->    x s<= (-1 >> y)
3646     //  x s<= x & (-1 >> y)    ->    x s<= (-1 >> y)
3647     if (!match(M, m_Constant())) // Can not do this fold with non-constant.
3648       return nullptr;
3649     if (!match(M, m_NonNegative())) // Must not have any -1 vector elements.
3650       return nullptr;
3651     DstPred = ICmpInst::Predicate::ICMP_SLE;
3652     break;
3653   case ICmpInst::Predicate::ICMP_SGT:
3654   case ICmpInst::Predicate::ICMP_SLE:
3655     return nullptr;
3656   case ICmpInst::Predicate::ICMP_UGT:
3657   case ICmpInst::Predicate::ICMP_ULE:
3658     llvm_unreachable("Instsimplify took care of commut. variant");
3659     break;
3660   default:
3661     llvm_unreachable("All possible folds are handled.");
3662   }
3663 
3664   // The mask value may be a vector constant that has undefined elements. But it
3665   // may not be safe to propagate those undefs into the new compare, so replace
3666   // those elements by copying an existing, defined, and safe scalar constant.
3667   Type *OpTy = M->getType();
3668   auto *VecC = dyn_cast<Constant>(M);
3669   auto *OpVTy = dyn_cast<FixedVectorType>(OpTy);
3670   if (OpVTy && VecC && VecC->containsUndefOrPoisonElement()) {
3671     Constant *SafeReplacementConstant = nullptr;
3672     for (unsigned i = 0, e = OpVTy->getNumElements(); i != e; ++i) {
3673       if (!isa<UndefValue>(VecC->getAggregateElement(i))) {
3674         SafeReplacementConstant = VecC->getAggregateElement(i);
3675         break;
3676       }
3677     }
3678     assert(SafeReplacementConstant && "Failed to find undef replacement");
3679     M = Constant::replaceUndefsWith(VecC, SafeReplacementConstant);
3680   }
3681 
3682   return Builder.CreateICmp(DstPred, X, M);
3683 }
3684 
3685 /// Some comparisons can be simplified.
3686 /// In this case, we are looking for comparisons that look like
3687 /// a check for a lossy signed truncation.
3688 /// Folds:   (MaskedBits is a constant.)
3689 ///   ((%x << MaskedBits) a>> MaskedBits) SrcPred %x
3690 /// Into:
3691 ///   (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits)
3692 /// Where  KeptBits = bitwidth(%x) - MaskedBits
3693 static Value *
3694 foldICmpWithTruncSignExtendedVal(ICmpInst &I,
3695                                  InstCombiner::BuilderTy &Builder) {
3696   ICmpInst::Predicate SrcPred;
3697   Value *X;
3698   const APInt *C0, *C1; // FIXME: non-splats, potentially with undef.
3699   // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use.
3700   if (!match(&I, m_c_ICmp(SrcPred,
3701                           m_OneUse(m_AShr(m_Shl(m_Value(X), m_APInt(C0)),
3702                                           m_APInt(C1))),
3703                           m_Deferred(X))))
3704     return nullptr;
3705 
3706   // Potential handling of non-splats: for each element:
3707   //  * if both are undef, replace with constant 0.
3708   //    Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0.
3709   //  * if both are not undef, and are different, bailout.
3710   //  * else, only one is undef, then pick the non-undef one.
3711 
3712   // The shift amount must be equal.
3713   if (*C0 != *C1)
3714     return nullptr;
3715   const APInt &MaskedBits = *C0;
3716   assert(MaskedBits != 0 && "shift by zero should be folded away already.");
3717 
3718   ICmpInst::Predicate DstPred;
3719   switch (SrcPred) {
3720   case ICmpInst::Predicate::ICMP_EQ:
3721     // ((%x << MaskedBits) a>> MaskedBits) == %x
3722     //   =>
3723     // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits)
3724     DstPred = ICmpInst::Predicate::ICMP_ULT;
3725     break;
3726   case ICmpInst::Predicate::ICMP_NE:
3727     // ((%x << MaskedBits) a>> MaskedBits) != %x
3728     //   =>
3729     // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits)
3730     DstPred = ICmpInst::Predicate::ICMP_UGE;
3731     break;
3732   // FIXME: are more folds possible?
3733   default:
3734     return nullptr;
3735   }
3736 
3737   auto *XType = X->getType();
3738   const unsigned XBitWidth = XType->getScalarSizeInBits();
3739   const APInt BitWidth = APInt(XBitWidth, XBitWidth);
3740   assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched");
3741 
3742   // KeptBits = bitwidth(%x) - MaskedBits
3743   const APInt KeptBits = BitWidth - MaskedBits;
3744   assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable");
3745   // ICmpCst = (1 << KeptBits)
3746   const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits);
3747   assert(ICmpCst.isPowerOf2());
3748   // AddCst = (1 << (KeptBits-1))
3749   const APInt AddCst = ICmpCst.lshr(1);
3750   assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2());
3751 
3752   // T0 = add %x, AddCst
3753   Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst));
3754   // T1 = T0 DstPred ICmpCst
3755   Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst));
3756 
3757   return T1;
3758 }
3759 
3760 // Given pattern:
3761 //   icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
3762 // we should move shifts to the same hand of 'and', i.e. rewrite as
3763 //   icmp eq/ne (and (x shift (Q+K)), y), 0  iff (Q+K) u< bitwidth(x)
3764 // We are only interested in opposite logical shifts here.
3765 // One of the shifts can be truncated.
3766 // If we can, we want to end up creating 'lshr' shift.
3767 static Value *
3768 foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ,
3769                                            InstCombiner::BuilderTy &Builder) {
3770   if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) ||
3771       !I.getOperand(0)->hasOneUse())
3772     return nullptr;
3773 
3774   auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value());
3775 
3776   // Look for an 'and' of two logical shifts, one of which may be truncated.
3777   // We use m_TruncOrSelf() on the RHS to correctly handle commutative case.
3778   Instruction *XShift, *MaybeTruncation, *YShift;
3779   if (!match(
3780           I.getOperand(0),
3781           m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)),
3782                   m_CombineAnd(m_TruncOrSelf(m_CombineAnd(
3783                                    m_AnyLogicalShift, m_Instruction(YShift))),
3784                                m_Instruction(MaybeTruncation)))))
3785     return nullptr;
3786 
3787   // We potentially looked past 'trunc', but only when matching YShift,
3788   // therefore YShift must have the widest type.
3789   Instruction *WidestShift = YShift;
3790   // Therefore XShift must have the shallowest type.
3791   // Or they both have identical types if there was no truncation.
3792   Instruction *NarrowestShift = XShift;
3793 
3794   Type *WidestTy = WidestShift->getType();
3795   Type *NarrowestTy = NarrowestShift->getType();
3796   assert(NarrowestTy == I.getOperand(0)->getType() &&
3797          "We did not look past any shifts while matching XShift though.");
3798   bool HadTrunc = WidestTy != I.getOperand(0)->getType();
3799 
3800   // If YShift is a 'lshr', swap the shifts around.
3801   if (match(YShift, m_LShr(m_Value(), m_Value())))
3802     std::swap(XShift, YShift);
3803 
3804   // The shifts must be in opposite directions.
3805   auto XShiftOpcode = XShift->getOpcode();
3806   if (XShiftOpcode == YShift->getOpcode())
3807     return nullptr; // Do not care about same-direction shifts here.
3808 
3809   Value *X, *XShAmt, *Y, *YShAmt;
3810   match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt))));
3811   match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt))));
3812 
3813   // If one of the values being shifted is a constant, then we will end with
3814   // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not,
3815   // however, we will need to ensure that we won't increase instruction count.
3816   if (!isa<Constant>(X) && !isa<Constant>(Y)) {
3817     // At least one of the hands of the 'and' should be one-use shift.
3818     if (!match(I.getOperand(0),
3819                m_c_And(m_OneUse(m_AnyLogicalShift), m_Value())))
3820       return nullptr;
3821     if (HadTrunc) {
3822       // Due to the 'trunc', we will need to widen X. For that either the old
3823       // 'trunc' or the shift amt in the non-truncated shift should be one-use.
3824       if (!MaybeTruncation->hasOneUse() &&
3825           !NarrowestShift->getOperand(1)->hasOneUse())
3826         return nullptr;
3827     }
3828   }
3829 
3830   // We have two shift amounts from two different shifts. The types of those
3831   // shift amounts may not match. If that's the case let's bailout now.
3832   if (XShAmt->getType() != YShAmt->getType())
3833     return nullptr;
3834 
3835   // As input, we have the following pattern:
3836   //   icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
3837   // We want to rewrite that as:
3838   //   icmp eq/ne (and (x shift (Q+K)), y), 0  iff (Q+K) u< bitwidth(x)
3839   // While we know that originally (Q+K) would not overflow
3840   // (because  2 * (N-1) u<= iN -1), we have looked past extensions of
3841   // shift amounts. so it may now overflow in smaller bitwidth.
3842   // To ensure that does not happen, we need to ensure that the total maximal
3843   // shift amount is still representable in that smaller bit width.
3844   unsigned MaximalPossibleTotalShiftAmount =
3845       (WidestTy->getScalarSizeInBits() - 1) +
3846       (NarrowestTy->getScalarSizeInBits() - 1);
3847   APInt MaximalRepresentableShiftAmount =
3848       APInt::getAllOnes(XShAmt->getType()->getScalarSizeInBits());
3849   if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount))
3850     return nullptr;
3851 
3852   // Can we fold (XShAmt+YShAmt) ?
3853   auto *NewShAmt = dyn_cast_or_null<Constant>(
3854       simplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false,
3855                       /*isNUW=*/false, SQ.getWithInstruction(&I)));
3856   if (!NewShAmt)
3857     return nullptr;
3858   NewShAmt = ConstantExpr::getZExtOrBitCast(NewShAmt, WidestTy);
3859   unsigned WidestBitWidth = WidestTy->getScalarSizeInBits();
3860 
3861   // Is the new shift amount smaller than the bit width?
3862   // FIXME: could also rely on ConstantRange.
3863   if (!match(NewShAmt,
3864              m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_ULT,
3865                                 APInt(WidestBitWidth, WidestBitWidth))))
3866     return nullptr;
3867 
3868   // An extra legality check is needed if we had trunc-of-lshr.
3869   if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) {
3870     auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ,
3871                     WidestShift]() {
3872       // It isn't obvious whether it's worth it to analyze non-constants here.
3873       // Also, let's basically give up on non-splat cases, pessimizing vectors.
3874       // If *any* of these preconditions matches we can perform the fold.
3875       Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy()
3876                                     ? NewShAmt->getSplatValue()
3877                                     : NewShAmt;
3878       // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold.
3879       if (NewShAmtSplat &&
3880           (NewShAmtSplat->isNullValue() ||
3881            NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1))
3882         return true;
3883       // We consider *min* leading zeros so a single outlier
3884       // blocks the transform as opposed to allowing it.
3885       if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) {
3886         KnownBits Known = computeKnownBits(C, SQ.DL);
3887         unsigned MinLeadZero = Known.countMinLeadingZeros();
3888         // If the value being shifted has at most lowest bit set we can fold.
3889         unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
3890         if (MaxActiveBits <= 1)
3891           return true;
3892         // Precondition:  NewShAmt u<= countLeadingZeros(C)
3893         if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero))
3894           return true;
3895       }
3896       if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) {
3897         KnownBits Known = computeKnownBits(C, SQ.DL);
3898         unsigned MinLeadZero = Known.countMinLeadingZeros();
3899         // If the value being shifted has at most lowest bit set we can fold.
3900         unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
3901         if (MaxActiveBits <= 1)
3902           return true;
3903         // Precondition:  ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C)
3904         if (NewShAmtSplat) {
3905           APInt AdjNewShAmt =
3906               (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger();
3907           if (AdjNewShAmt.ule(MinLeadZero))
3908             return true;
3909         }
3910       }
3911       return false; // Can't tell if it's ok.
3912     };
3913     if (!CanFold())
3914       return nullptr;
3915   }
3916 
3917   // All good, we can do this fold.
3918   X = Builder.CreateZExt(X, WidestTy);
3919   Y = Builder.CreateZExt(Y, WidestTy);
3920   // The shift is the same that was for X.
3921   Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr
3922                   ? Builder.CreateLShr(X, NewShAmt)
3923                   : Builder.CreateShl(X, NewShAmt);
3924   Value *T1 = Builder.CreateAnd(T0, Y);
3925   return Builder.CreateICmp(I.getPredicate(), T1,
3926                             Constant::getNullValue(WidestTy));
3927 }
3928 
3929 /// Fold
3930 ///   (-1 u/ x) u< y
3931 ///   ((x * y) ?/ x) != y
3932 /// to
3933 ///   @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit
3934 /// Note that the comparison is commutative, while inverted (u>=, ==) predicate
3935 /// will mean that we are looking for the opposite answer.
3936 Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) {
3937   ICmpInst::Predicate Pred;
3938   Value *X, *Y;
3939   Instruction *Mul;
3940   Instruction *Div;
3941   bool NeedNegation;
3942   // Look for: (-1 u/ x) u</u>= y
3943   if (!I.isEquality() &&
3944       match(&I, m_c_ICmp(Pred,
3945                          m_CombineAnd(m_OneUse(m_UDiv(m_AllOnes(), m_Value(X))),
3946                                       m_Instruction(Div)),
3947                          m_Value(Y)))) {
3948     Mul = nullptr;
3949 
3950     // Are we checking that overflow does not happen, or does happen?
3951     switch (Pred) {
3952     case ICmpInst::Predicate::ICMP_ULT:
3953       NeedNegation = false;
3954       break; // OK
3955     case ICmpInst::Predicate::ICMP_UGE:
3956       NeedNegation = true;
3957       break; // OK
3958     default:
3959       return nullptr; // Wrong predicate.
3960     }
3961   } else // Look for: ((x * y) / x) !=/== y
3962       if (I.isEquality() &&
3963           match(&I,
3964                 m_c_ICmp(Pred, m_Value(Y),
3965                          m_CombineAnd(
3966                              m_OneUse(m_IDiv(m_CombineAnd(m_c_Mul(m_Deferred(Y),
3967                                                                   m_Value(X)),
3968                                                           m_Instruction(Mul)),
3969                                              m_Deferred(X))),
3970                              m_Instruction(Div))))) {
3971     NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
3972   } else
3973     return nullptr;
3974 
3975   BuilderTy::InsertPointGuard Guard(Builder);
3976   // If the pattern included (x * y), we'll want to insert new instructions
3977   // right before that original multiplication so that we can replace it.
3978   bool MulHadOtherUses = Mul && !Mul->hasOneUse();
3979   if (MulHadOtherUses)
3980     Builder.SetInsertPoint(Mul);
3981 
3982   Function *F = Intrinsic::getDeclaration(I.getModule(),
3983                                           Div->getOpcode() == Instruction::UDiv
3984                                               ? Intrinsic::umul_with_overflow
3985                                               : Intrinsic::smul_with_overflow,
3986                                           X->getType());
3987   CallInst *Call = Builder.CreateCall(F, {X, Y}, "mul");
3988 
3989   // If the multiplication was used elsewhere, to ensure that we don't leave
3990   // "duplicate" instructions, replace uses of that original multiplication
3991   // with the multiplication result from the with.overflow intrinsic.
3992   if (MulHadOtherUses)
3993     replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "mul.val"));
3994 
3995   Value *Res = Builder.CreateExtractValue(Call, 1, "mul.ov");
3996   if (NeedNegation) // This technically increases instruction count.
3997     Res = Builder.CreateNot(Res, "mul.not.ov");
3998 
3999   // If we replaced the mul, erase it. Do this after all uses of Builder,
4000   // as the mul is used as insertion point.
4001   if (MulHadOtherUses)
4002     eraseInstFromFunction(*Mul);
4003 
4004   return Res;
4005 }
4006 
4007 static Instruction *foldICmpXNegX(ICmpInst &I) {
4008   CmpInst::Predicate Pred;
4009   Value *X;
4010   if (!match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X))))
4011     return nullptr;
4012 
4013   if (ICmpInst::isSigned(Pred))
4014     Pred = ICmpInst::getSwappedPredicate(Pred);
4015   else if (ICmpInst::isUnsigned(Pred))
4016     Pred = ICmpInst::getSignedPredicate(Pred);
4017   // else for equality-comparisons just keep the predicate.
4018 
4019   return ICmpInst::Create(Instruction::ICmp, Pred, X,
4020                           Constant::getNullValue(X->getType()), I.getName());
4021 }
4022 
4023 /// Try to fold icmp (binop), X or icmp X, (binop).
4024 /// TODO: A large part of this logic is duplicated in InstSimplify's
4025 /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
4026 /// duplication.
4027 Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
4028                                              const SimplifyQuery &SQ) {
4029   const SimplifyQuery Q = SQ.getWithInstruction(&I);
4030   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
4031 
4032   // Special logic for binary operators.
4033   BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0);
4034   BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1);
4035   if (!BO0 && !BO1)
4036     return nullptr;
4037 
4038   if (Instruction *NewICmp = foldICmpXNegX(I))
4039     return NewICmp;
4040 
4041   const CmpInst::Predicate Pred = I.getPredicate();
4042   Value *X;
4043 
4044   // Convert add-with-unsigned-overflow comparisons into a 'not' with compare.
4045   // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X
4046   if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) &&
4047       (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
4048     return new ICmpInst(Pred, Builder.CreateNot(Op1), X);
4049   // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0
4050   if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) &&
4051       (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
4052     return new ICmpInst(Pred, X, Builder.CreateNot(Op0));
4053 
4054   {
4055     // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1
4056     Constant *C;
4057     if (match(Op0, m_OneUse(m_Add(m_c_Add(m_Specific(Op1), m_Value(X)),
4058                                   m_ImmConstant(C)))) &&
4059         (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
4060       Constant *C2 = ConstantExpr::getNot(C);
4061       return new ICmpInst(Pred, Builder.CreateSub(C2, X), Op1);
4062     }
4063     // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X
4064     if (match(Op1, m_OneUse(m_Add(m_c_Add(m_Specific(Op0), m_Value(X)),
4065                                   m_ImmConstant(C)))) &&
4066         (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) {
4067       Constant *C2 = ConstantExpr::getNot(C);
4068       return new ICmpInst(Pred, Op0, Builder.CreateSub(C2, X));
4069     }
4070   }
4071 
4072   {
4073     // Similar to above: an unsigned overflow comparison may use offset + mask:
4074     // ((Op1 + C) & C) u<  Op1 --> Op1 != 0
4075     // ((Op1 + C) & C) u>= Op1 --> Op1 == 0
4076     // Op0 u>  ((Op0 + C) & C) --> Op0 != 0
4077     // Op0 u<= ((Op0 + C) & C) --> Op0 == 0
4078     BinaryOperator *BO;
4079     const APInt *C;
4080     if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) &&
4081         match(Op0, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
4082         match(BO, m_Add(m_Specific(Op1), m_SpecificIntAllowUndef(*C)))) {
4083       CmpInst::Predicate NewPred =
4084           Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
4085       Constant *Zero = ConstantInt::getNullValue(Op1->getType());
4086       return new ICmpInst(NewPred, Op1, Zero);
4087     }
4088 
4089     if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
4090         match(Op1, m_And(m_BinOp(BO), m_LowBitMask(C))) &&
4091         match(BO, m_Add(m_Specific(Op0), m_SpecificIntAllowUndef(*C)))) {
4092       CmpInst::Predicate NewPred =
4093           Pred == ICmpInst::ICMP_UGT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
4094       Constant *Zero = ConstantInt::getNullValue(Op1->getType());
4095       return new ICmpInst(NewPred, Op0, Zero);
4096     }
4097   }
4098 
4099   bool NoOp0WrapProblem = false, NoOp1WrapProblem = false;
4100   if (BO0 && isa<OverflowingBinaryOperator>(BO0))
4101     NoOp0WrapProblem =
4102         ICmpInst::isEquality(Pred) ||
4103         (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) ||
4104         (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap());
4105   if (BO1 && isa<OverflowingBinaryOperator>(BO1))
4106     NoOp1WrapProblem =
4107         ICmpInst::isEquality(Pred) ||
4108         (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) ||
4109         (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap());
4110 
4111   // Analyze the case when either Op0 or Op1 is an add instruction.
4112   // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null).
4113   Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
4114   if (BO0 && BO0->getOpcode() == Instruction::Add) {
4115     A = BO0->getOperand(0);
4116     B = BO0->getOperand(1);
4117   }
4118   if (BO1 && BO1->getOpcode() == Instruction::Add) {
4119     C = BO1->getOperand(0);
4120     D = BO1->getOperand(1);
4121   }
4122 
4123   // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow.
4124   // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow.
4125   if ((A == Op1 || B == Op1) && NoOp0WrapProblem)
4126     return new ICmpInst(Pred, A == Op1 ? B : A,
4127                         Constant::getNullValue(Op1->getType()));
4128 
4129   // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow.
4130   // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow.
4131   if ((C == Op0 || D == Op0) && NoOp1WrapProblem)
4132     return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()),
4133                         C == Op0 ? D : C);
4134 
4135   // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow.
4136   if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem &&
4137       NoOp1WrapProblem) {
4138     // Determine Y and Z in the form icmp (X+Y), (X+Z).
4139     Value *Y, *Z;
4140     if (A == C) {
4141       // C + B == C + D  ->  B == D
4142       Y = B;
4143       Z = D;
4144     } else if (A == D) {
4145       // D + B == C + D  ->  B == C
4146       Y = B;
4147       Z = C;
4148     } else if (B == C) {
4149       // A + C == C + D  ->  A == D
4150       Y = A;
4151       Z = D;
4152     } else {
4153       assert(B == D);
4154       // A + D == C + D  ->  A == C
4155       Y = A;
4156       Z = C;
4157     }
4158     return new ICmpInst(Pred, Y, Z);
4159   }
4160 
4161   // icmp slt (A + -1), Op1 -> icmp sle A, Op1
4162   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
4163       match(B, m_AllOnes()))
4164     return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
4165 
4166   // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
4167   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
4168       match(B, m_AllOnes()))
4169     return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
4170 
4171   // icmp sle (A + 1), Op1 -> icmp slt A, Op1
4172   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One()))
4173     return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
4174 
4175   // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
4176   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One()))
4177     return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
4178 
4179   // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
4180   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
4181       match(D, m_AllOnes()))
4182     return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
4183 
4184   // icmp sle Op0, (C + -1) -> icmp slt Op0, C
4185   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
4186       match(D, m_AllOnes()))
4187     return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
4188 
4189   // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
4190   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One()))
4191     return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
4192 
4193   // icmp slt Op0, (C + 1) -> icmp sle Op0, C
4194   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One()))
4195     return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
4196 
4197   // TODO: The subtraction-related identities shown below also hold, but
4198   // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
4199   // wouldn't happen even if they were implemented.
4200   //
4201   // icmp ult (A - 1), Op1 -> icmp ule A, Op1
4202   // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
4203   // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
4204   // icmp ule Op0, (C - 1) -> icmp ult Op0, C
4205 
4206   // icmp ule (A + 1), Op0 -> icmp ult A, Op1
4207   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One()))
4208     return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
4209 
4210   // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
4211   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One()))
4212     return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
4213 
4214   // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
4215   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One()))
4216     return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
4217 
4218   // icmp ult Op0, (C + 1) -> icmp ule Op0, C
4219   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One()))
4220     return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
4221 
4222   // if C1 has greater magnitude than C2:
4223   //  icmp (A + C1), (C + C2) -> icmp (A + C3), C
4224   //  s.t. C3 = C1 - C2
4225   //
4226   // if C2 has greater magnitude than C1:
4227   //  icmp (A + C1), (C + C2) -> icmp A, (C + C3)
4228   //  s.t. C3 = C2 - C1
4229   if (A && C && NoOp0WrapProblem && NoOp1WrapProblem &&
4230       (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) {
4231     const APInt *AP1, *AP2;
4232     // TODO: Support non-uniform vectors.
4233     // TODO: Allow undef passthrough if B AND D's element is undef.
4234     if (match(B, m_APIntAllowUndef(AP1)) && match(D, m_APIntAllowUndef(AP2)) &&
4235         AP1->isNegative() == AP2->isNegative()) {
4236       APInt AP1Abs = AP1->abs();
4237       APInt AP2Abs = AP2->abs();
4238       if (AP1Abs.uge(AP2Abs)) {
4239         APInt Diff = *AP1 - *AP2;
4240         bool HasNUW = BO0->hasNoUnsignedWrap() && Diff.ule(*AP1);
4241         bool HasNSW = BO0->hasNoSignedWrap();
4242         Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
4243         Value *NewAdd = Builder.CreateAdd(A, C3, "", HasNUW, HasNSW);
4244         return new ICmpInst(Pred, NewAdd, C);
4245       } else {
4246         APInt Diff = *AP2 - *AP1;
4247         bool HasNUW = BO1->hasNoUnsignedWrap() && Diff.ule(*AP2);
4248         bool HasNSW = BO1->hasNoSignedWrap();
4249         Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff);
4250         Value *NewAdd = Builder.CreateAdd(C, C3, "", HasNUW, HasNSW);
4251         return new ICmpInst(Pred, A, NewAdd);
4252       }
4253     }
4254     Constant *Cst1, *Cst2;
4255     if (match(B, m_ImmConstant(Cst1)) && match(D, m_ImmConstant(Cst2)) &&
4256         ICmpInst::isEquality(Pred)) {
4257       Constant *Diff = ConstantExpr::getSub(Cst2, Cst1);
4258       Value *NewAdd = Builder.CreateAdd(C, Diff);
4259       return new ICmpInst(Pred, A, NewAdd);
4260     }
4261   }
4262 
4263   // Analyze the case when either Op0 or Op1 is a sub instruction.
4264   // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null).
4265   A = nullptr;
4266   B = nullptr;
4267   C = nullptr;
4268   D = nullptr;
4269   if (BO0 && BO0->getOpcode() == Instruction::Sub) {
4270     A = BO0->getOperand(0);
4271     B = BO0->getOperand(1);
4272   }
4273   if (BO1 && BO1->getOpcode() == Instruction::Sub) {
4274     C = BO1->getOperand(0);
4275     D = BO1->getOperand(1);
4276   }
4277 
4278   // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow.
4279   if (A == Op1 && NoOp0WrapProblem)
4280     return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B);
4281   // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow.
4282   if (C == Op0 && NoOp1WrapProblem)
4283     return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType()));
4284 
4285   // Convert sub-with-unsigned-overflow comparisons into a comparison of args.
4286   // (A - B) u>/u<= A --> B u>/u<= A
4287   if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
4288     return new ICmpInst(Pred, B, A);
4289   // C u</u>= (C - D) --> C u</u>= D
4290   if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
4291     return new ICmpInst(Pred, C, D);
4292   // (A - B) u>=/u< A --> B u>/u<= A  iff B != 0
4293   if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
4294       isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
4295     return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A);
4296   // C u<=/u> (C - D) --> C u</u>= D  iff B != 0
4297   if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
4298       isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
4299     return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D);
4300 
4301   // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
4302   if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem)
4303     return new ICmpInst(Pred, A, C);
4304 
4305   // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow.
4306   if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem)
4307     return new ICmpInst(Pred, D, B);
4308 
4309   // icmp (0-X) < cst --> x > -cst
4310   if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) {
4311     Value *X;
4312     if (match(BO0, m_Neg(m_Value(X))))
4313       if (Constant *RHSC = dyn_cast<Constant>(Op1))
4314         if (RHSC->isNotMinSignedValue())
4315           return new ICmpInst(I.getSwappedPredicate(), X,
4316                               ConstantExpr::getNeg(RHSC));
4317   }
4318 
4319   {
4320     // Try to remove shared constant multiplier from equality comparison:
4321     // X * C == Y * C (with no overflowing/aliasing) --> X == Y
4322     Value *X, *Y;
4323     const APInt *C;
4324     if (match(Op0, m_Mul(m_Value(X), m_APInt(C))) && *C != 0 &&
4325         match(Op1, m_Mul(m_Value(Y), m_SpecificInt(*C))) && I.isEquality())
4326       if (!C->countTrailingZeros() ||
4327           (BO0 && BO1 && BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap()) ||
4328           (BO0 && BO1 && BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap()))
4329       return new ICmpInst(Pred, X, Y);
4330   }
4331 
4332   BinaryOperator *SRem = nullptr;
4333   // icmp (srem X, Y), Y
4334   if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1))
4335     SRem = BO0;
4336   // icmp Y, (srem X, Y)
4337   else if (BO1 && BO1->getOpcode() == Instruction::SRem &&
4338            Op0 == BO1->getOperand(1))
4339     SRem = BO1;
4340   if (SRem) {
4341     // We don't check hasOneUse to avoid increasing register pressure because
4342     // the value we use is the same value this instruction was already using.
4343     switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) {
4344     default:
4345       break;
4346     case ICmpInst::ICMP_EQ:
4347       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
4348     case ICmpInst::ICMP_NE:
4349       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
4350     case ICmpInst::ICMP_SGT:
4351     case ICmpInst::ICMP_SGE:
4352       return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1),
4353                           Constant::getAllOnesValue(SRem->getType()));
4354     case ICmpInst::ICMP_SLT:
4355     case ICmpInst::ICMP_SLE:
4356       return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1),
4357                           Constant::getNullValue(SRem->getType()));
4358     }
4359   }
4360 
4361   if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() && BO0->hasOneUse() &&
4362       BO1->hasOneUse() && BO0->getOperand(1) == BO1->getOperand(1)) {
4363     switch (BO0->getOpcode()) {
4364     default:
4365       break;
4366     case Instruction::Add:
4367     case Instruction::Sub:
4368     case Instruction::Xor: {
4369       if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b
4370         return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
4371 
4372       const APInt *C;
4373       if (match(BO0->getOperand(1), m_APInt(C))) {
4374         // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
4375         if (C->isSignMask()) {
4376           ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
4377           return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
4378         }
4379 
4380         // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b
4381         if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) {
4382           ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
4383           NewPred = I.getSwappedPredicate(NewPred);
4384           return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0));
4385         }
4386       }
4387       break;
4388     }
4389     case Instruction::Mul: {
4390       if (!I.isEquality())
4391         break;
4392 
4393       const APInt *C;
4394       if (match(BO0->getOperand(1), m_APInt(C)) && !C->isZero() &&
4395           !C->isOne()) {
4396         // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask)
4397         // Mask = -1 >> count-trailing-zeros(C).
4398         if (unsigned TZs = C->countTrailingZeros()) {
4399           Constant *Mask = ConstantInt::get(
4400               BO0->getType(),
4401               APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs));
4402           Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask);
4403           Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask);
4404           return new ICmpInst(Pred, And1, And2);
4405         }
4406       }
4407       break;
4408     }
4409     case Instruction::UDiv:
4410     case Instruction::LShr:
4411       if (I.isSigned() || !BO0->isExact() || !BO1->isExact())
4412         break;
4413       return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
4414 
4415     case Instruction::SDiv:
4416       if (!I.isEquality() || !BO0->isExact() || !BO1->isExact())
4417         break;
4418       return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
4419 
4420     case Instruction::AShr:
4421       if (!BO0->isExact() || !BO1->isExact())
4422         break;
4423       return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
4424 
4425     case Instruction::Shl: {
4426       bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap();
4427       bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap();
4428       if (!NUW && !NSW)
4429         break;
4430       if (!NSW && I.isSigned())
4431         break;
4432       return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0));
4433     }
4434     }
4435   }
4436 
4437   if (BO0) {
4438     // Transform  A & (L - 1) `ult` L --> L != 0
4439     auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes());
4440     auto BitwiseAnd = m_c_And(m_Value(), LSubOne);
4441 
4442     if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) {
4443       auto *Zero = Constant::getNullValue(BO0->getType());
4444       return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero);
4445     }
4446   }
4447 
4448   // For unsigned predicates / eq / ne:
4449   // icmp pred (x << 1), x --> icmp getSignedPredicate(pred) x, 0
4450   // icmp pred x, (x << 1) --> icmp getSignedPredicate(pred) 0, x
4451   if (!ICmpInst::isSigned(Pred)) {
4452     if (match(Op0, m_Shl(m_Specific(Op1), m_One())))
4453       return new ICmpInst(ICmpInst::getSignedPredicate(Pred), Op1,
4454                           Constant::getNullValue(Op1->getType()));
4455     else if (match(Op1, m_Shl(m_Specific(Op0), m_One())))
4456       return new ICmpInst(ICmpInst::getSignedPredicate(Pred),
4457                           Constant::getNullValue(Op0->getType()), Op0);
4458   }
4459 
4460   if (Value *V = foldMultiplicationOverflowCheck(I))
4461     return replaceInstUsesWith(I, V);
4462 
4463   if (Value *V = foldICmpWithLowBitMaskedVal(I, Builder))
4464     return replaceInstUsesWith(I, V);
4465 
4466   if (Value *V = foldICmpWithTruncSignExtendedVal(I, Builder))
4467     return replaceInstUsesWith(I, V);
4468 
4469   if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder))
4470     return replaceInstUsesWith(I, V);
4471 
4472   return nullptr;
4473 }
4474 
4475 /// Fold icmp Pred min|max(X, Y), X.
4476 static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) {
4477   ICmpInst::Predicate Pred = Cmp.getPredicate();
4478   Value *Op0 = Cmp.getOperand(0);
4479   Value *X = Cmp.getOperand(1);
4480 
4481   // Canonicalize minimum or maximum operand to LHS of the icmp.
4482   if (match(X, m_c_SMin(m_Specific(Op0), m_Value())) ||
4483       match(X, m_c_SMax(m_Specific(Op0), m_Value())) ||
4484       match(X, m_c_UMin(m_Specific(Op0), m_Value())) ||
4485       match(X, m_c_UMax(m_Specific(Op0), m_Value()))) {
4486     std::swap(Op0, X);
4487     Pred = Cmp.getSwappedPredicate();
4488   }
4489 
4490   Value *Y;
4491   if (match(Op0, m_c_SMin(m_Specific(X), m_Value(Y)))) {
4492     // smin(X, Y)  == X --> X s<= Y
4493     // smin(X, Y) s>= X --> X s<= Y
4494     if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SGE)
4495       return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
4496 
4497     // smin(X, Y) != X --> X s> Y
4498     // smin(X, Y) s< X --> X s> Y
4499     if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SLT)
4500       return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
4501 
4502     // These cases should be handled in InstSimplify:
4503     // smin(X, Y) s<= X --> true
4504     // smin(X, Y) s> X --> false
4505     return nullptr;
4506   }
4507 
4508   if (match(Op0, m_c_SMax(m_Specific(X), m_Value(Y)))) {
4509     // smax(X, Y)  == X --> X s>= Y
4510     // smax(X, Y) s<= X --> X s>= Y
4511     if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLE)
4512       return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
4513 
4514     // smax(X, Y) != X --> X s< Y
4515     // smax(X, Y) s> X --> X s< Y
4516     if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SGT)
4517       return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
4518 
4519     // These cases should be handled in InstSimplify:
4520     // smax(X, Y) s>= X --> true
4521     // smax(X, Y) s< X --> false
4522     return nullptr;
4523   }
4524 
4525   if (match(Op0, m_c_UMin(m_Specific(X), m_Value(Y)))) {
4526     // umin(X, Y)  == X --> X u<= Y
4527     // umin(X, Y) u>= X --> X u<= Y
4528     if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_UGE)
4529       return new ICmpInst(ICmpInst::ICMP_ULE, X, Y);
4530 
4531     // umin(X, Y) != X --> X u> Y
4532     // umin(X, Y) u< X --> X u> Y
4533     if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT)
4534       return new ICmpInst(ICmpInst::ICMP_UGT, X, Y);
4535 
4536     // These cases should be handled in InstSimplify:
4537     // umin(X, Y) u<= X --> true
4538     // umin(X, Y) u> X --> false
4539     return nullptr;
4540   }
4541 
4542   if (match(Op0, m_c_UMax(m_Specific(X), m_Value(Y)))) {
4543     // umax(X, Y)  == X --> X u>= Y
4544     // umax(X, Y) u<= X --> X u>= Y
4545     if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_ULE)
4546       return new ICmpInst(ICmpInst::ICMP_UGE, X, Y);
4547 
4548     // umax(X, Y) != X --> X u< Y
4549     // umax(X, Y) u> X --> X u< Y
4550     if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_UGT)
4551       return new ICmpInst(ICmpInst::ICMP_ULT, X, Y);
4552 
4553     // These cases should be handled in InstSimplify:
4554     // umax(X, Y) u>= X --> true
4555     // umax(X, Y) u< X --> false
4556     return nullptr;
4557   }
4558 
4559   return nullptr;
4560 }
4561 
4562 Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
4563   if (!I.isEquality())
4564     return nullptr;
4565 
4566   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
4567   const CmpInst::Predicate Pred = I.getPredicate();
4568   Value *A, *B, *C, *D;
4569   if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) {
4570     if (A == Op1 || B == Op1) { // (A^B) == A  ->  B == 0
4571       Value *OtherVal = A == Op1 ? B : A;
4572       return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
4573     }
4574 
4575     if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) {
4576       // A^c1 == C^c2 --> A == C^(c1^c2)
4577       ConstantInt *C1, *C2;
4578       if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) &&
4579           Op1->hasOneUse()) {
4580         Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue());
4581         Value *Xor = Builder.CreateXor(C, NC);
4582         return new ICmpInst(Pred, A, Xor);
4583       }
4584 
4585       // A^B == A^D -> B == D
4586       if (A == C)
4587         return new ICmpInst(Pred, B, D);
4588       if (A == D)
4589         return new ICmpInst(Pred, B, C);
4590       if (B == C)
4591         return new ICmpInst(Pred, A, D);
4592       if (B == D)
4593         return new ICmpInst(Pred, A, C);
4594     }
4595   }
4596 
4597   if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) {
4598     // A == (A^B)  ->  B == 0
4599     Value *OtherVal = A == Op0 ? B : A;
4600     return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType()));
4601   }
4602 
4603   // (X&Z) == (Y&Z) -> (X^Y) & Z == 0
4604   if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) &&
4605       match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) {
4606     Value *X = nullptr, *Y = nullptr, *Z = nullptr;
4607 
4608     if (A == C) {
4609       X = B;
4610       Y = D;
4611       Z = A;
4612     } else if (A == D) {
4613       X = B;
4614       Y = C;
4615       Z = A;
4616     } else if (B == C) {
4617       X = A;
4618       Y = D;
4619       Z = B;
4620     } else if (B == D) {
4621       X = A;
4622       Y = C;
4623       Z = B;
4624     }
4625 
4626     if (X) { // Build (X^Y) & Z
4627       Op1 = Builder.CreateXor(X, Y);
4628       Op1 = Builder.CreateAnd(Op1, Z);
4629       return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType()));
4630     }
4631   }
4632 
4633   {
4634     // Similar to above, but specialized for constant because invert is needed:
4635     // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0
4636     Value *X, *Y;
4637     Constant *C;
4638     if (match(Op0, m_OneUse(m_Or(m_Value(X), m_Constant(C)))) &&
4639         match(Op1, m_OneUse(m_Or(m_Value(Y), m_Specific(C))))) {
4640       Value *Xor = Builder.CreateXor(X, Y);
4641       Value *And = Builder.CreateAnd(Xor, ConstantExpr::getNot(C));
4642       return new ICmpInst(Pred, And, Constant::getNullValue(And->getType()));
4643     }
4644   }
4645 
4646   if (match(Op1, m_ZExt(m_Value(A))) &&
4647       (Op0->hasOneUse() || Op1->hasOneUse())) {
4648     // (B & (Pow2C-1)) == zext A --> A == trunc B
4649     // (B & (Pow2C-1)) != zext A --> A != trunc B
4650     const APInt *MaskC;
4651     if (match(Op0, m_And(m_Value(B), m_LowBitMask(MaskC))) &&
4652         MaskC->countTrailingOnes() == A->getType()->getScalarSizeInBits())
4653       return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType()));
4654 
4655     // Test if 2 values have different or same signbits:
4656     // (X u>> BitWidth - 1) == zext (Y s> -1) --> (X ^ Y) < 0
4657     // (X u>> BitWidth - 1) != zext (Y s> -1) --> (X ^ Y) > -1
4658     unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
4659     Value *X, *Y;
4660     ICmpInst::Predicate Pred2;
4661     if (match(Op0, m_LShr(m_Value(X), m_SpecificIntAllowUndef(OpWidth - 1))) &&
4662         match(A, m_ICmp(Pred2, m_Value(Y), m_AllOnes())) &&
4663         Pred2 == ICmpInst::ICMP_SGT && X->getType() == Y->getType()) {
4664       Value *Xor = Builder.CreateXor(X, Y, "xor.signbits");
4665       Value *R = (Pred == ICmpInst::ICMP_EQ) ? Builder.CreateIsNeg(Xor) :
4666                                                Builder.CreateIsNotNeg(Xor);
4667       return replaceInstUsesWith(I, R);
4668     }
4669   }
4670 
4671   // (A >> C) == (B >> C) --> (A^B) u< (1 << C)
4672   // For lshr and ashr pairs.
4673   const APInt *AP1, *AP2;
4674   if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_APIntAllowUndef(AP1)))) &&
4675        match(Op1, m_OneUse(m_LShr(m_Value(B), m_APIntAllowUndef(AP2))))) ||
4676       (match(Op0, m_OneUse(m_AShr(m_Value(A), m_APIntAllowUndef(AP1)))) &&
4677        match(Op1, m_OneUse(m_AShr(m_Value(B), m_APIntAllowUndef(AP2)))))) {
4678     if (AP1 != AP2)
4679       return nullptr;
4680     unsigned TypeBits = AP1->getBitWidth();
4681     unsigned ShAmt = AP1->getLimitedValue(TypeBits);
4682     if (ShAmt < TypeBits && ShAmt != 0) {
4683       ICmpInst::Predicate NewPred =
4684           Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
4685       Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
4686       APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt);
4687       return new ICmpInst(NewPred, Xor, ConstantInt::get(A->getType(), CmpVal));
4688     }
4689   }
4690 
4691   // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0
4692   ConstantInt *Cst1;
4693   if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) &&
4694       match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) {
4695     unsigned TypeBits = Cst1->getBitWidth();
4696     unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits);
4697     if (ShAmt < TypeBits && ShAmt != 0) {
4698       Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted");
4699       APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt);
4700       Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal),
4701                                       I.getName() + ".mask");
4702       return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType()));
4703     }
4704   }
4705 
4706   // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to
4707   // "icmp (and X, mask), cst"
4708   uint64_t ShAmt = 0;
4709   if (Op0->hasOneUse() &&
4710       match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) &&
4711       match(Op1, m_ConstantInt(Cst1)) &&
4712       // Only do this when A has multiple uses.  This is most important to do
4713       // when it exposes other optimizations.
4714       !A->hasOneUse()) {
4715     unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits();
4716 
4717     if (ShAmt < ASize) {
4718       APInt MaskV =
4719           APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits());
4720       MaskV <<= ShAmt;
4721 
4722       APInt CmpV = Cst1->getValue().zext(ASize);
4723       CmpV <<= ShAmt;
4724 
4725       Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV));
4726       return new ICmpInst(Pred, Mask, Builder.getInt(CmpV));
4727     }
4728   }
4729 
4730   if (Instruction *ICmp = foldICmpIntrinsicWithIntrinsic(I))
4731     return ICmp;
4732 
4733   // Canonicalize checking for a power-of-2-or-zero value:
4734   // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants)
4735   // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants)
4736   if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()),
4737                                    m_Deferred(A)))) ||
4738       !match(Op1, m_ZeroInt()))
4739     A = nullptr;
4740 
4741   // (A & -A) == A --> ctpop(A) < 2 (four commuted variants)
4742   // (-A & A) != A --> ctpop(A) > 1 (four commuted variants)
4743   if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1)))))
4744     A = Op1;
4745   else if (match(Op1,
4746                  m_OneUse(m_c_And(m_Neg(m_Specific(Op0)), m_Specific(Op0)))))
4747     A = Op0;
4748 
4749   if (A) {
4750     Type *Ty = A->getType();
4751     CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A);
4752     return Pred == ICmpInst::ICMP_EQ
4753         ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop, ConstantInt::get(Ty, 2))
4754         : new ICmpInst(ICmpInst::ICMP_UGT, CtPop, ConstantInt::get(Ty, 1));
4755   }
4756 
4757   // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the
4758   // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX",
4759   // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps
4760   // of instcombine.
4761   unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
4762   if (match(Op0, m_AShr(m_Trunc(m_Value(A)), m_SpecificInt(BitWidth - 1))) &&
4763       match(Op1, m_Trunc(m_LShr(m_Specific(A), m_SpecificInt(BitWidth)))) &&
4764       A->getType()->getScalarSizeInBits() == BitWidth * 2 &&
4765       (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) {
4766     APInt C = APInt::getOneBitSet(BitWidth * 2, BitWidth - 1);
4767     Value *Add = Builder.CreateAdd(A, ConstantInt::get(A->getType(), C));
4768     return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT
4769                                                   : ICmpInst::ICMP_UGE,
4770                         Add, ConstantInt::get(A->getType(), C.shl(1)));
4771   }
4772 
4773   // Canonicalize:
4774   // Assume B_Pow2 != 0
4775   // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0
4776   // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0
4777   if (match(Op0, m_c_And(m_Specific(Op1), m_Value())) &&
4778       isKnownToBeAPowerOfTwo(Op1, /* OrZero */ false, 0, &I))
4779     return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
4780                         ConstantInt::getNullValue(Op0->getType()));
4781 
4782   if (match(Op1, m_c_And(m_Specific(Op0), m_Value())) &&
4783       isKnownToBeAPowerOfTwo(Op0, /* OrZero */ false, 0, &I))
4784     return new ICmpInst(CmpInst::getInversePredicate(Pred), Op1,
4785                         ConstantInt::getNullValue(Op1->getType()));
4786 
4787   return nullptr;
4788 }
4789 
4790 static Instruction *foldICmpWithTrunc(ICmpInst &ICmp,
4791                                       InstCombiner::BuilderTy &Builder) {
4792   ICmpInst::Predicate Pred = ICmp.getPredicate();
4793   Value *Op0 = ICmp.getOperand(0), *Op1 = ICmp.getOperand(1);
4794 
4795   // Try to canonicalize trunc + compare-to-constant into a mask + cmp.
4796   // The trunc masks high bits while the compare may effectively mask low bits.
4797   Value *X;
4798   const APInt *C;
4799   if (!match(Op0, m_OneUse(m_Trunc(m_Value(X)))) || !match(Op1, m_APInt(C)))
4800     return nullptr;
4801 
4802   // This matches patterns corresponding to tests of the signbit as well as:
4803   // (trunc X) u< C --> (X & -C) == 0 (are all masked-high-bits clear?)
4804   // (trunc X) u> C --> (X & ~C) != 0 (are any masked-high-bits set?)
4805   APInt Mask;
4806   if (decomposeBitTestICmp(Op0, Op1, Pred, X, Mask, true /* WithTrunc */)) {
4807     Value *And = Builder.CreateAnd(X, Mask);
4808     Constant *Zero = ConstantInt::getNullValue(X->getType());
4809     return new ICmpInst(Pred, And, Zero);
4810   }
4811 
4812   unsigned SrcBits = X->getType()->getScalarSizeInBits();
4813   if (Pred == ICmpInst::ICMP_ULT && C->isNegatedPowerOf2()) {
4814     // If C is a negative power-of-2 (high-bit mask):
4815     // (trunc X) u< C --> (X & C) != C (are any masked-high-bits clear?)
4816     Constant *MaskC = ConstantInt::get(X->getType(), C->zext(SrcBits));
4817     Value *And = Builder.CreateAnd(X, MaskC);
4818     return new ICmpInst(ICmpInst::ICMP_NE, And, MaskC);
4819   }
4820 
4821   if (Pred == ICmpInst::ICMP_UGT && (~*C).isPowerOf2()) {
4822     // If C is not-of-power-of-2 (one clear bit):
4823     // (trunc X) u> C --> (X & (C+1)) == C+1 (are all masked-high-bits set?)
4824     Constant *MaskC = ConstantInt::get(X->getType(), (*C + 1).zext(SrcBits));
4825     Value *And = Builder.CreateAnd(X, MaskC);
4826     return new ICmpInst(ICmpInst::ICMP_EQ, And, MaskC);
4827   }
4828 
4829   return nullptr;
4830 }
4831 
4832 Instruction *InstCombinerImpl::foldICmpWithZextOrSext(ICmpInst &ICmp) {
4833   assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0");
4834   auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0));
4835   Value *X;
4836   if (!match(CastOp0, m_ZExtOrSExt(m_Value(X))))
4837     return nullptr;
4838 
4839   bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt;
4840   bool IsSignedCmp = ICmp.isSigned();
4841 
4842   // icmp Pred (ext X), (ext Y)
4843   Value *Y;
4844   if (match(ICmp.getOperand(1), m_ZExtOrSExt(m_Value(Y)))) {
4845     bool IsZext0 = isa<ZExtOperator>(ICmp.getOperand(0));
4846     bool IsZext1 = isa<ZExtOperator>(ICmp.getOperand(1));
4847 
4848     // If we have mismatched casts, treat the zext of a non-negative source as
4849     // a sext to simulate matching casts. Otherwise, we are done.
4850     // TODO: Can we handle some predicates (equality) without non-negative?
4851     if (IsZext0 != IsZext1) {
4852       if ((IsZext0 && isKnownNonNegative(X, DL, 0, &AC, &ICmp, &DT)) ||
4853           (IsZext1 && isKnownNonNegative(Y, DL, 0, &AC, &ICmp, &DT)))
4854         IsSignedExt = true;
4855       else
4856         return nullptr;
4857     }
4858 
4859     // Not an extension from the same type?
4860     Type *XTy = X->getType(), *YTy = Y->getType();
4861     if (XTy != YTy) {
4862       // One of the casts must have one use because we are creating a new cast.
4863       if (!ICmp.getOperand(0)->hasOneUse() && !ICmp.getOperand(1)->hasOneUse())
4864         return nullptr;
4865       // Extend the narrower operand to the type of the wider operand.
4866       CastInst::CastOps CastOpcode =
4867           IsSignedExt ? Instruction::SExt : Instruction::ZExt;
4868       if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits())
4869         X = Builder.CreateCast(CastOpcode, X, YTy);
4870       else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits())
4871         Y = Builder.CreateCast(CastOpcode, Y, XTy);
4872       else
4873         return nullptr;
4874     }
4875 
4876     // (zext X) == (zext Y) --> X == Y
4877     // (sext X) == (sext Y) --> X == Y
4878     if (ICmp.isEquality())
4879       return new ICmpInst(ICmp.getPredicate(), X, Y);
4880 
4881     // A signed comparison of sign extended values simplifies into a
4882     // signed comparison.
4883     if (IsSignedCmp && IsSignedExt)
4884       return new ICmpInst(ICmp.getPredicate(), X, Y);
4885 
4886     // The other three cases all fold into an unsigned comparison.
4887     return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y);
4888   }
4889 
4890   // Below here, we are only folding a compare with constant.
4891   auto *C = dyn_cast<Constant>(ICmp.getOperand(1));
4892   if (!C)
4893     return nullptr;
4894 
4895   // Compute the constant that would happen if we truncated to SrcTy then
4896   // re-extended to DestTy.
4897   Type *SrcTy = CastOp0->getSrcTy();
4898   Type *DestTy = CastOp0->getDestTy();
4899   Constant *Res1 = ConstantExpr::getTrunc(C, SrcTy);
4900   Constant *Res2 = ConstantExpr::getCast(CastOp0->getOpcode(), Res1, DestTy);
4901 
4902   // If the re-extended constant didn't change...
4903   if (Res2 == C) {
4904     if (ICmp.isEquality())
4905       return new ICmpInst(ICmp.getPredicate(), X, Res1);
4906 
4907     // A signed comparison of sign extended values simplifies into a
4908     // signed comparison.
4909     if (IsSignedExt && IsSignedCmp)
4910       return new ICmpInst(ICmp.getPredicate(), X, Res1);
4911 
4912     // The other three cases all fold into an unsigned comparison.
4913     return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res1);
4914   }
4915 
4916   // The re-extended constant changed, partly changed (in the case of a vector),
4917   // or could not be determined to be equal (in the case of a constant
4918   // expression), so the constant cannot be represented in the shorter type.
4919   // All the cases that fold to true or false will have already been handled
4920   // by simplifyICmpInst, so only deal with the tricky case.
4921   if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C))
4922     return nullptr;
4923 
4924   // Is source op positive?
4925   // icmp ult (sext X), C --> icmp sgt X, -1
4926   if (ICmp.getPredicate() == ICmpInst::ICMP_ULT)
4927     return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(SrcTy));
4928 
4929   // Is source op negative?
4930   // icmp ugt (sext X), C --> icmp slt X, 0
4931   assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!");
4932   return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(SrcTy));
4933 }
4934 
4935 /// Handle icmp (cast x), (cast or constant).
4936 Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) {
4937   // If any operand of ICmp is a inttoptr roundtrip cast then remove it as
4938   // icmp compares only pointer's value.
4939   // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2.
4940   Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0));
4941   Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1));
4942   if (SimplifiedOp0 || SimplifiedOp1)
4943     return new ICmpInst(ICmp.getPredicate(),
4944                         SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0),
4945                         SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1));
4946 
4947   auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0));
4948   if (!CastOp0)
4949     return nullptr;
4950   if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1)))
4951     return nullptr;
4952 
4953   Value *Op0Src = CastOp0->getOperand(0);
4954   Type *SrcTy = CastOp0->getSrcTy();
4955   Type *DestTy = CastOp0->getDestTy();
4956 
4957   // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the
4958   // integer type is the same size as the pointer type.
4959   auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) {
4960     if (isa<VectorType>(SrcTy)) {
4961       SrcTy = cast<VectorType>(SrcTy)->getElementType();
4962       DestTy = cast<VectorType>(DestTy)->getElementType();
4963     }
4964     return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth();
4965   };
4966   if (CastOp0->getOpcode() == Instruction::PtrToInt &&
4967       CompatibleSizes(SrcTy, DestTy)) {
4968     Value *NewOp1 = nullptr;
4969     if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) {
4970       Value *PtrSrc = PtrToIntOp1->getOperand(0);
4971       if (PtrSrc->getType()->getPointerAddressSpace() ==
4972           Op0Src->getType()->getPointerAddressSpace()) {
4973         NewOp1 = PtrToIntOp1->getOperand(0);
4974         // If the pointer types don't match, insert a bitcast.
4975         if (Op0Src->getType() != NewOp1->getType())
4976           NewOp1 = Builder.CreateBitCast(NewOp1, Op0Src->getType());
4977       }
4978     } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) {
4979       NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy);
4980     }
4981 
4982     if (NewOp1)
4983       return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
4984   }
4985 
4986   if (Instruction *R = foldICmpWithTrunc(ICmp, Builder))
4987     return R;
4988 
4989   return foldICmpWithZextOrSext(ICmp);
4990 }
4991 
4992 static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned) {
4993   switch (BinaryOp) {
4994     default:
4995       llvm_unreachable("Unsupported binary op");
4996     case Instruction::Add:
4997     case Instruction::Sub:
4998       return match(RHS, m_Zero());
4999     case Instruction::Mul:
5000       return !(RHS->getType()->isIntOrIntVectorTy(1) && IsSigned) &&
5001              match(RHS, m_One());
5002   }
5003 }
5004 
5005 OverflowResult
5006 InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp,
5007                                   bool IsSigned, Value *LHS, Value *RHS,
5008                                   Instruction *CxtI) const {
5009   switch (BinaryOp) {
5010     default:
5011       llvm_unreachable("Unsupported binary op");
5012     case Instruction::Add:
5013       if (IsSigned)
5014         return computeOverflowForSignedAdd(LHS, RHS, CxtI);
5015       else
5016         return computeOverflowForUnsignedAdd(LHS, RHS, CxtI);
5017     case Instruction::Sub:
5018       if (IsSigned)
5019         return computeOverflowForSignedSub(LHS, RHS, CxtI);
5020       else
5021         return computeOverflowForUnsignedSub(LHS, RHS, CxtI);
5022     case Instruction::Mul:
5023       if (IsSigned)
5024         return computeOverflowForSignedMul(LHS, RHS, CxtI);
5025       else
5026         return computeOverflowForUnsignedMul(LHS, RHS, CxtI);
5027   }
5028 }
5029 
5030 bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
5031                                              bool IsSigned, Value *LHS,
5032                                              Value *RHS, Instruction &OrigI,
5033                                              Value *&Result,
5034                                              Constant *&Overflow) {
5035   if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS))
5036     std::swap(LHS, RHS);
5037 
5038   // If the overflow check was an add followed by a compare, the insertion point
5039   // may be pointing to the compare.  We want to insert the new instructions
5040   // before the add in case there are uses of the add between the add and the
5041   // compare.
5042   Builder.SetInsertPoint(&OrigI);
5043 
5044   Type *OverflowTy = Type::getInt1Ty(LHS->getContext());
5045   if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType()))
5046     OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount());
5047 
5048   if (isNeutralValue(BinaryOp, RHS, IsSigned)) {
5049     Result = LHS;
5050     Overflow = ConstantInt::getFalse(OverflowTy);
5051     return true;
5052   }
5053 
5054   switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) {
5055     case OverflowResult::MayOverflow:
5056       return false;
5057     case OverflowResult::AlwaysOverflowsLow:
5058     case OverflowResult::AlwaysOverflowsHigh:
5059       Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
5060       Result->takeName(&OrigI);
5061       Overflow = ConstantInt::getTrue(OverflowTy);
5062       return true;
5063     case OverflowResult::NeverOverflows:
5064       Result = Builder.CreateBinOp(BinaryOp, LHS, RHS);
5065       Result->takeName(&OrigI);
5066       Overflow = ConstantInt::getFalse(OverflowTy);
5067       if (auto *Inst = dyn_cast<Instruction>(Result)) {
5068         if (IsSigned)
5069           Inst->setHasNoSignedWrap();
5070         else
5071           Inst->setHasNoUnsignedWrap();
5072       }
5073       return true;
5074   }
5075 
5076   llvm_unreachable("Unexpected overflow result");
5077 }
5078 
5079 /// Recognize and process idiom involving test for multiplication
5080 /// overflow.
5081 ///
5082 /// The caller has matched a pattern of the form:
5083 ///   I = cmp u (mul(zext A, zext B), V
5084 /// The function checks if this is a test for overflow and if so replaces
5085 /// multiplication with call to 'mul.with.overflow' intrinsic.
5086 ///
5087 /// \param I Compare instruction.
5088 /// \param MulVal Result of 'mult' instruction.  It is one of the arguments of
5089 ///               the compare instruction.  Must be of integer type.
5090 /// \param OtherVal The other argument of compare instruction.
5091 /// \returns Instruction which must replace the compare instruction, NULL if no
5092 ///          replacement required.
5093 static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
5094                                          Value *OtherVal,
5095                                          InstCombinerImpl &IC) {
5096   // Don't bother doing this transformation for pointers, don't do it for
5097   // vectors.
5098   if (!isa<IntegerType>(MulVal->getType()))
5099     return nullptr;
5100 
5101   assert(I.getOperand(0) == MulVal || I.getOperand(1) == MulVal);
5102   assert(I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal);
5103   auto *MulInstr = dyn_cast<Instruction>(MulVal);
5104   if (!MulInstr)
5105     return nullptr;
5106   assert(MulInstr->getOpcode() == Instruction::Mul);
5107 
5108   auto *LHS = cast<ZExtOperator>(MulInstr->getOperand(0)),
5109        *RHS = cast<ZExtOperator>(MulInstr->getOperand(1));
5110   assert(LHS->getOpcode() == Instruction::ZExt);
5111   assert(RHS->getOpcode() == Instruction::ZExt);
5112   Value *A = LHS->getOperand(0), *B = RHS->getOperand(0);
5113 
5114   // Calculate type and width of the result produced by mul.with.overflow.
5115   Type *TyA = A->getType(), *TyB = B->getType();
5116   unsigned WidthA = TyA->getPrimitiveSizeInBits(),
5117            WidthB = TyB->getPrimitiveSizeInBits();
5118   unsigned MulWidth;
5119   Type *MulType;
5120   if (WidthB > WidthA) {
5121     MulWidth = WidthB;
5122     MulType = TyB;
5123   } else {
5124     MulWidth = WidthA;
5125     MulType = TyA;
5126   }
5127 
5128   // In order to replace the original mul with a narrower mul.with.overflow,
5129   // all uses must ignore upper bits of the product.  The number of used low
5130   // bits must be not greater than the width of mul.with.overflow.
5131   if (MulVal->hasNUsesOrMore(2))
5132     for (User *U : MulVal->users()) {
5133       if (U == &I)
5134         continue;
5135       if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
5136         // Check if truncation ignores bits above MulWidth.
5137         unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits();
5138         if (TruncWidth > MulWidth)
5139           return nullptr;
5140       } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
5141         // Check if AND ignores bits above MulWidth.
5142         if (BO->getOpcode() != Instruction::And)
5143           return nullptr;
5144         if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) {
5145           const APInt &CVal = CI->getValue();
5146           if (CVal.getBitWidth() - CVal.countLeadingZeros() > MulWidth)
5147             return nullptr;
5148         } else {
5149           // In this case we could have the operand of the binary operation
5150           // being defined in another block, and performing the replacement
5151           // could break the dominance relation.
5152           return nullptr;
5153         }
5154       } else {
5155         // Other uses prohibit this transformation.
5156         return nullptr;
5157       }
5158     }
5159 
5160   // Recognize patterns
5161   switch (I.getPredicate()) {
5162   case ICmpInst::ICMP_EQ:
5163   case ICmpInst::ICMP_NE:
5164     // Recognize pattern:
5165     //   mulval = mul(zext A, zext B)
5166     //   cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits.
5167     ConstantInt *CI;
5168     Value *ValToMask;
5169     if (match(OtherVal, m_And(m_Value(ValToMask), m_ConstantInt(CI)))) {
5170       if (ValToMask != MulVal)
5171         return nullptr;
5172       const APInt &CVal = CI->getValue() + 1;
5173       if (CVal.isPowerOf2()) {
5174         unsigned MaskWidth = CVal.logBase2();
5175         if (MaskWidth == MulWidth)
5176           break; // Recognized
5177       }
5178     }
5179     return nullptr;
5180 
5181   case ICmpInst::ICMP_UGT:
5182     // Recognize pattern:
5183     //   mulval = mul(zext A, zext B)
5184     //   cmp ugt mulval, max
5185     if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) {
5186       APInt MaxVal = APInt::getMaxValue(MulWidth);
5187       MaxVal = MaxVal.zext(CI->getBitWidth());
5188       if (MaxVal.eq(CI->getValue()))
5189         break; // Recognized
5190     }
5191     return nullptr;
5192 
5193   case ICmpInst::ICMP_UGE:
5194     // Recognize pattern:
5195     //   mulval = mul(zext A, zext B)
5196     //   cmp uge mulval, max+1
5197     if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) {
5198       APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth);
5199       if (MaxVal.eq(CI->getValue()))
5200         break; // Recognized
5201     }
5202     return nullptr;
5203 
5204   case ICmpInst::ICMP_ULE:
5205     // Recognize pattern:
5206     //   mulval = mul(zext A, zext B)
5207     //   cmp ule mulval, max
5208     if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) {
5209       APInt MaxVal = APInt::getMaxValue(MulWidth);
5210       MaxVal = MaxVal.zext(CI->getBitWidth());
5211       if (MaxVal.eq(CI->getValue()))
5212         break; // Recognized
5213     }
5214     return nullptr;
5215 
5216   case ICmpInst::ICMP_ULT:
5217     // Recognize pattern:
5218     //   mulval = mul(zext A, zext B)
5219     //   cmp ule mulval, max + 1
5220     if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) {
5221       APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth);
5222       if (MaxVal.eq(CI->getValue()))
5223         break; // Recognized
5224     }
5225     return nullptr;
5226 
5227   default:
5228     return nullptr;
5229   }
5230 
5231   InstCombiner::BuilderTy &Builder = IC.Builder;
5232   Builder.SetInsertPoint(MulInstr);
5233 
5234   // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B)
5235   Value *MulA = A, *MulB = B;
5236   if (WidthA < MulWidth)
5237     MulA = Builder.CreateZExt(A, MulType);
5238   if (WidthB < MulWidth)
5239     MulB = Builder.CreateZExt(B, MulType);
5240   Function *F = Intrinsic::getDeclaration(
5241       I.getModule(), Intrinsic::umul_with_overflow, MulType);
5242   CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul");
5243   IC.addToWorklist(MulInstr);
5244 
5245   // If there are uses of mul result other than the comparison, we know that
5246   // they are truncation or binary AND. Change them to use result of
5247   // mul.with.overflow and adjust properly mask/size.
5248   if (MulVal->hasNUsesOrMore(2)) {
5249     Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value");
5250     for (User *U : make_early_inc_range(MulVal->users())) {
5251       if (U == &I || U == OtherVal)
5252         continue;
5253       if (TruncInst *TI = dyn_cast<TruncInst>(U)) {
5254         if (TI->getType()->getPrimitiveSizeInBits() == MulWidth)
5255           IC.replaceInstUsesWith(*TI, Mul);
5256         else
5257           TI->setOperand(0, Mul);
5258       } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) {
5259         assert(BO->getOpcode() == Instruction::And);
5260         // Replace (mul & mask) --> zext (mul.with.overflow & short_mask)
5261         ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1));
5262         APInt ShortMask = CI->getValue().trunc(MulWidth);
5263         Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask);
5264         Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType());
5265         IC.replaceInstUsesWith(*BO, Zext);
5266       } else {
5267         llvm_unreachable("Unexpected Binary operation");
5268       }
5269       IC.addToWorklist(cast<Instruction>(U));
5270     }
5271   }
5272   if (isa<Instruction>(OtherVal))
5273     IC.addToWorklist(cast<Instruction>(OtherVal));
5274 
5275   // The original icmp gets replaced with the overflow value, maybe inverted
5276   // depending on predicate.
5277   bool Inverse = false;
5278   switch (I.getPredicate()) {
5279   case ICmpInst::ICMP_NE:
5280     break;
5281   case ICmpInst::ICMP_EQ:
5282     Inverse = true;
5283     break;
5284   case ICmpInst::ICMP_UGT:
5285   case ICmpInst::ICMP_UGE:
5286     if (I.getOperand(0) == MulVal)
5287       break;
5288     Inverse = true;
5289     break;
5290   case ICmpInst::ICMP_ULT:
5291   case ICmpInst::ICMP_ULE:
5292     if (I.getOperand(1) == MulVal)
5293       break;
5294     Inverse = true;
5295     break;
5296   default:
5297     llvm_unreachable("Unexpected predicate");
5298   }
5299   if (Inverse) {
5300     Value *Res = Builder.CreateExtractValue(Call, 1);
5301     return BinaryOperator::CreateNot(Res);
5302   }
5303 
5304   return ExtractValueInst::Create(Call, 1);
5305 }
5306 
5307 /// When performing a comparison against a constant, it is possible that not all
5308 /// the bits in the LHS are demanded. This helper method computes the mask that
5309 /// IS demanded.
5310 static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) {
5311   const APInt *RHS;
5312   if (!match(I.getOperand(1), m_APInt(RHS)))
5313     return APInt::getAllOnes(BitWidth);
5314 
5315   // If this is a normal comparison, it demands all bits. If it is a sign bit
5316   // comparison, it only demands the sign bit.
5317   bool UnusedBit;
5318   if (InstCombiner::isSignBitCheck(I.getPredicate(), *RHS, UnusedBit))
5319     return APInt::getSignMask(BitWidth);
5320 
5321   switch (I.getPredicate()) {
5322   // For a UGT comparison, we don't care about any bits that
5323   // correspond to the trailing ones of the comparand.  The value of these
5324   // bits doesn't impact the outcome of the comparison, because any value
5325   // greater than the RHS must differ in a bit higher than these due to carry.
5326   case ICmpInst::ICMP_UGT:
5327     return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingOnes());
5328 
5329   // Similarly, for a ULT comparison, we don't care about the trailing zeros.
5330   // Any value less than the RHS must differ in a higher bit because of carries.
5331   case ICmpInst::ICMP_ULT:
5332     return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingZeros());
5333 
5334   default:
5335     return APInt::getAllOnes(BitWidth);
5336   }
5337 }
5338 
5339 /// Check if the order of \p Op0 and \p Op1 as operands in an ICmpInst
5340 /// should be swapped.
5341 /// The decision is based on how many times these two operands are reused
5342 /// as subtract operands and their positions in those instructions.
5343 /// The rationale is that several architectures use the same instruction for
5344 /// both subtract and cmp. Thus, it is better if the order of those operands
5345 /// match.
5346 /// \return true if Op0 and Op1 should be swapped.
5347 static bool swapMayExposeCSEOpportunities(const Value *Op0, const Value *Op1) {
5348   // Filter out pointer values as those cannot appear directly in subtract.
5349   // FIXME: we may want to go through inttoptrs or bitcasts.
5350   if (Op0->getType()->isPointerTy())
5351     return false;
5352   // If a subtract already has the same operands as a compare, swapping would be
5353   // bad. If a subtract has the same operands as a compare but in reverse order,
5354   // then swapping is good.
5355   int GoodToSwap = 0;
5356   for (const User *U : Op0->users()) {
5357     if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0))))
5358       GoodToSwap++;
5359     else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1))))
5360       GoodToSwap--;
5361   }
5362   return GoodToSwap > 0;
5363 }
5364 
5365 /// Check that one use is in the same block as the definition and all
5366 /// other uses are in blocks dominated by a given block.
5367 ///
5368 /// \param DI Definition
5369 /// \param UI Use
5370 /// \param DB Block that must dominate all uses of \p DI outside
5371 ///           the parent block
5372 /// \return true when \p UI is the only use of \p DI in the parent block
5373 /// and all other uses of \p DI are in blocks dominated by \p DB.
5374 ///
5375 bool InstCombinerImpl::dominatesAllUses(const Instruction *DI,
5376                                         const Instruction *UI,
5377                                         const BasicBlock *DB) const {
5378   assert(DI && UI && "Instruction not defined\n");
5379   // Ignore incomplete definitions.
5380   if (!DI->getParent())
5381     return false;
5382   // DI and UI must be in the same block.
5383   if (DI->getParent() != UI->getParent())
5384     return false;
5385   // Protect from self-referencing blocks.
5386   if (DI->getParent() == DB)
5387     return false;
5388   for (const User *U : DI->users()) {
5389     auto *Usr = cast<Instruction>(U);
5390     if (Usr != UI && !DT.dominates(DB, Usr->getParent()))
5391       return false;
5392   }
5393   return true;
5394 }
5395 
5396 /// Return true when the instruction sequence within a block is select-cmp-br.
5397 static bool isChainSelectCmpBranch(const SelectInst *SI) {
5398   const BasicBlock *BB = SI->getParent();
5399   if (!BB)
5400     return false;
5401   auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator());
5402   if (!BI || BI->getNumSuccessors() != 2)
5403     return false;
5404   auto *IC = dyn_cast<ICmpInst>(BI->getCondition());
5405   if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI))
5406     return false;
5407   return true;
5408 }
5409 
5410 /// True when a select result is replaced by one of its operands
5411 /// in select-icmp sequence. This will eventually result in the elimination
5412 /// of the select.
5413 ///
5414 /// \param SI    Select instruction
5415 /// \param Icmp  Compare instruction
5416 /// \param SIOpd Operand that replaces the select
5417 ///
5418 /// Notes:
5419 /// - The replacement is global and requires dominator information
5420 /// - The caller is responsible for the actual replacement
5421 ///
5422 /// Example:
5423 ///
5424 /// entry:
5425 ///  %4 = select i1 %3, %C* %0, %C* null
5426 ///  %5 = icmp eq %C* %4, null
5427 ///  br i1 %5, label %9, label %7
5428 ///  ...
5429 ///  ; <label>:7                                       ; preds = %entry
5430 ///  %8 = getelementptr inbounds %C* %4, i64 0, i32 0
5431 ///  ...
5432 ///
5433 /// can be transformed to
5434 ///
5435 ///  %5 = icmp eq %C* %0, null
5436 ///  %6 = select i1 %3, i1 %5, i1 true
5437 ///  br i1 %6, label %9, label %7
5438 ///  ...
5439 ///  ; <label>:7                                       ; preds = %entry
5440 ///  %8 = getelementptr inbounds %C* %0, i64 0, i32 0  // replace by %0!
5441 ///
5442 /// Similar when the first operand of the select is a constant or/and
5443 /// the compare is for not equal rather than equal.
5444 ///
5445 /// NOTE: The function is only called when the select and compare constants
5446 /// are equal, the optimization can work only for EQ predicates. This is not a
5447 /// major restriction since a NE compare should be 'normalized' to an equal
5448 /// compare, which usually happens in the combiner and test case
5449 /// select-cmp-br.ll checks for it.
5450 bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI,
5451                                                  const ICmpInst *Icmp,
5452                                                  const unsigned SIOpd) {
5453   assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!");
5454   if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) {
5455     BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1);
5456     // The check for the single predecessor is not the best that can be
5457     // done. But it protects efficiently against cases like when SI's
5458     // home block has two successors, Succ and Succ1, and Succ1 predecessor
5459     // of Succ. Then SI can't be replaced by SIOpd because the use that gets
5460     // replaced can be reached on either path. So the uniqueness check
5461     // guarantees that the path all uses of SI (outside SI's parent) are on
5462     // is disjoint from all other paths out of SI. But that information
5463     // is more expensive to compute, and the trade-off here is in favor
5464     // of compile-time. It should also be noticed that we check for a single
5465     // predecessor and not only uniqueness. This to handle the situation when
5466     // Succ and Succ1 points to the same basic block.
5467     if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) {
5468       NumSel++;
5469       SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent());
5470       return true;
5471     }
5472   }
5473   return false;
5474 }
5475 
5476 /// Try to fold the comparison based on range information we can get by checking
5477 /// whether bits are known to be zero or one in the inputs.
5478 Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) {
5479   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
5480   Type *Ty = Op0->getType();
5481   ICmpInst::Predicate Pred = I.getPredicate();
5482 
5483   // Get scalar or pointer size.
5484   unsigned BitWidth = Ty->isIntOrIntVectorTy()
5485                           ? Ty->getScalarSizeInBits()
5486                           : DL.getPointerTypeSizeInBits(Ty->getScalarType());
5487 
5488   if (!BitWidth)
5489     return nullptr;
5490 
5491   KnownBits Op0Known(BitWidth);
5492   KnownBits Op1Known(BitWidth);
5493 
5494   if (SimplifyDemandedBits(&I, 0,
5495                            getDemandedBitsLHSMask(I, BitWidth),
5496                            Op0Known, 0))
5497     return &I;
5498 
5499   if (SimplifyDemandedBits(&I, 1, APInt::getAllOnes(BitWidth), Op1Known, 0))
5500     return &I;
5501 
5502   // Given the known and unknown bits, compute a range that the LHS could be
5503   // in.  Compute the Min, Max and RHS values based on the known bits. For the
5504   // EQ and NE we use unsigned values.
5505   APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0);
5506   APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0);
5507   if (I.isSigned()) {
5508     Op0Min = Op0Known.getSignedMinValue();
5509     Op0Max = Op0Known.getSignedMaxValue();
5510     Op1Min = Op1Known.getSignedMinValue();
5511     Op1Max = Op1Known.getSignedMaxValue();
5512   } else {
5513     Op0Min = Op0Known.getMinValue();
5514     Op0Max = Op0Known.getMaxValue();
5515     Op1Min = Op1Known.getMinValue();
5516     Op1Max = Op1Known.getMaxValue();
5517   }
5518 
5519   // If Min and Max are known to be the same, then SimplifyDemandedBits figured
5520   // out that the LHS or RHS is a constant. Constant fold this now, so that
5521   // code below can assume that Min != Max.
5522   if (!isa<Constant>(Op0) && Op0Min == Op0Max)
5523     return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, Op0Min), Op1);
5524   if (!isa<Constant>(Op1) && Op1Min == Op1Max)
5525     return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Min));
5526 
5527   // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a
5528   // min/max canonical compare with some other compare. That could lead to
5529   // conflict with select canonicalization and infinite looping.
5530   // FIXME: This constraint may go away if min/max intrinsics are canonical.
5531   auto isMinMaxCmp = [&](Instruction &Cmp) {
5532     if (!Cmp.hasOneUse())
5533       return false;
5534     Value *A, *B;
5535     SelectPatternFlavor SPF = matchSelectPattern(Cmp.user_back(), A, B).Flavor;
5536     if (!SelectPatternResult::isMinOrMax(SPF))
5537       return false;
5538     return match(Op0, m_MaxOrMin(m_Value(), m_Value())) ||
5539            match(Op1, m_MaxOrMin(m_Value(), m_Value()));
5540   };
5541   if (!isMinMaxCmp(I)) {
5542     switch (Pred) {
5543     default:
5544       break;
5545     case ICmpInst::ICMP_ULT: {
5546       if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B)
5547         return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
5548       const APInt *CmpC;
5549       if (match(Op1, m_APInt(CmpC))) {
5550         // A <u C -> A == C-1 if min(A)+1 == C
5551         if (*CmpC == Op0Min + 1)
5552           return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
5553                               ConstantInt::get(Op1->getType(), *CmpC - 1));
5554         // X <u C --> X == 0, if the number of zero bits in the bottom of X
5555         // exceeds the log2 of C.
5556         if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2())
5557           return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
5558                               Constant::getNullValue(Op1->getType()));
5559       }
5560       break;
5561     }
5562     case ICmpInst::ICMP_UGT: {
5563       if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B)
5564         return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
5565       const APInt *CmpC;
5566       if (match(Op1, m_APInt(CmpC))) {
5567         // A >u C -> A == C+1 if max(a)-1 == C
5568         if (*CmpC == Op0Max - 1)
5569           return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
5570                               ConstantInt::get(Op1->getType(), *CmpC + 1));
5571         // X >u C --> X != 0, if the number of zero bits in the bottom of X
5572         // exceeds the log2 of C.
5573         if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits())
5574           return new ICmpInst(ICmpInst::ICMP_NE, Op0,
5575                               Constant::getNullValue(Op1->getType()));
5576       }
5577       break;
5578     }
5579     case ICmpInst::ICMP_SLT: {
5580       if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B)
5581         return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
5582       const APInt *CmpC;
5583       if (match(Op1, m_APInt(CmpC))) {
5584         if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C
5585           return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
5586                               ConstantInt::get(Op1->getType(), *CmpC - 1));
5587       }
5588       break;
5589     }
5590     case ICmpInst::ICMP_SGT: {
5591       if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B)
5592         return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
5593       const APInt *CmpC;
5594       if (match(Op1, m_APInt(CmpC))) {
5595         if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C
5596           return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
5597                               ConstantInt::get(Op1->getType(), *CmpC + 1));
5598       }
5599       break;
5600     }
5601     }
5602   }
5603 
5604   // Based on the range information we know about the LHS, see if we can
5605   // simplify this comparison.  For example, (x&4) < 8 is always true.
5606   switch (Pred) {
5607   default:
5608     llvm_unreachable("Unknown icmp opcode!");
5609   case ICmpInst::ICMP_EQ:
5610   case ICmpInst::ICMP_NE: {
5611     if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max))
5612       return replaceInstUsesWith(
5613           I, ConstantInt::getBool(I.getType(), Pred == CmpInst::ICMP_NE));
5614 
5615     // If all bits are known zero except for one, then we know at most one bit
5616     // is set. If the comparison is against zero, then this is a check to see if
5617     // *that* bit is set.
5618     APInt Op0KnownZeroInverted = ~Op0Known.Zero;
5619     if (Op1Known.isZero()) {
5620       // If the LHS is an AND with the same constant, look through it.
5621       Value *LHS = nullptr;
5622       const APInt *LHSC;
5623       if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) ||
5624           *LHSC != Op0KnownZeroInverted)
5625         LHS = Op0;
5626 
5627       Value *X;
5628       const APInt *C1;
5629       if (match(LHS, m_Shl(m_Power2(C1), m_Value(X)))) {
5630         Type *XTy = X->getType();
5631         unsigned Log2C1 = C1->countTrailingZeros();
5632         APInt C2 = Op0KnownZeroInverted;
5633         APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1;
5634         if (C2Pow2.isPowerOf2()) {
5635           // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2):
5636           // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1))
5637           // ((C1 << X) & C2) != 0 -> X  < (Log2(C2+C1) - Log2(C1))
5638           unsigned Log2C2 = C2Pow2.countTrailingZeros();
5639           auto *CmpC = ConstantInt::get(XTy, Log2C2 - Log2C1);
5640           auto NewPred =
5641               Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT;
5642           return new ICmpInst(NewPred, X, CmpC);
5643         }
5644       }
5645     }
5646 
5647     // Op0 eq C_Pow2 -> Op0 ne 0 if Op0 is known to be C_Pow2 or zero.
5648     if (Op1Known.isConstant() && Op1Known.getConstant().isPowerOf2() &&
5649         (Op0Known & Op1Known) == Op0Known)
5650       return new ICmpInst(CmpInst::getInversePredicate(Pred), Op0,
5651                           ConstantInt::getNullValue(Op1->getType()));
5652     break;
5653   }
5654   case ICmpInst::ICMP_ULT: {
5655     if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B)
5656       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5657     if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B)
5658       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5659     break;
5660   }
5661   case ICmpInst::ICMP_UGT: {
5662     if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B)
5663       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5664     if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B)
5665       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5666     break;
5667   }
5668   case ICmpInst::ICMP_SLT: {
5669     if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C)
5670       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5671     if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C)
5672       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5673     break;
5674   }
5675   case ICmpInst::ICMP_SGT: {
5676     if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B)
5677       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5678     if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B)
5679       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5680     break;
5681   }
5682   case ICmpInst::ICMP_SGE:
5683     assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!");
5684     if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B)
5685       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5686     if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B)
5687       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5688     if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B)
5689       return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
5690     break;
5691   case ICmpInst::ICMP_SLE:
5692     assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!");
5693     if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B)
5694       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5695     if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B)
5696       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5697     if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B)
5698       return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
5699     break;
5700   case ICmpInst::ICMP_UGE:
5701     assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!");
5702     if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B)
5703       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5704     if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B)
5705       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5706     if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B)
5707       return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
5708     break;
5709   case ICmpInst::ICMP_ULE:
5710     assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!");
5711     if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B)
5712       return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
5713     if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B)
5714       return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
5715     if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B)
5716       return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
5717     break;
5718   }
5719 
5720   // Turn a signed comparison into an unsigned one if both operands are known to
5721   // have the same sign.
5722   if (I.isSigned() &&
5723       ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) ||
5724        (Op0Known.One.isNegative() && Op1Known.One.isNegative())))
5725     return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1);
5726 
5727   return nullptr;
5728 }
5729 
5730 /// If one operand of an icmp is effectively a bool (value range of {0,1}),
5731 /// then try to reduce patterns based on that limit.
5732 static Instruction *foldICmpUsingBoolRange(ICmpInst &I,
5733                                            InstCombiner::BuilderTy &Builder) {
5734   Value *X, *Y;
5735   ICmpInst::Predicate Pred;
5736 
5737   // X must be 0 and bool must be true for "ULT":
5738   // X <u (zext i1 Y) --> (X == 0) & Y
5739   if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_ZExt(m_Value(Y))))) &&
5740       Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULT)
5741     return BinaryOperator::CreateAnd(Builder.CreateIsNull(X), Y);
5742 
5743   // X must be 0 or bool must be true for "ULE":
5744   // X <=u (sext i1 Y) --> (X == 0) | Y
5745   if (match(&I, m_c_ICmp(Pred, m_Value(X), m_OneUse(m_SExt(m_Value(Y))))) &&
5746       Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULE)
5747     return BinaryOperator::CreateOr(Builder.CreateIsNull(X), Y);
5748 
5749   return nullptr;
5750 }
5751 
5752 std::optional<std::pair<CmpInst::Predicate, Constant *>>
5753 InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
5754                                                        Constant *C) {
5755   assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
5756          "Only for relational integer predicates.");
5757 
5758   Type *Type = C->getType();
5759   bool IsSigned = ICmpInst::isSigned(Pred);
5760 
5761   CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
5762   bool WillIncrement =
5763       UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
5764 
5765   // Check if the constant operand can be safely incremented/decremented
5766   // without overflowing/underflowing.
5767   auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
5768     return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
5769   };
5770 
5771   Constant *SafeReplacementConstant = nullptr;
5772   if (auto *CI = dyn_cast<ConstantInt>(C)) {
5773     // Bail out if the constant can't be safely incremented/decremented.
5774     if (!ConstantIsOk(CI))
5775       return std::nullopt;
5776   } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) {
5777     unsigned NumElts = FVTy->getNumElements();
5778     for (unsigned i = 0; i != NumElts; ++i) {
5779       Constant *Elt = C->getAggregateElement(i);
5780       if (!Elt)
5781         return std::nullopt;
5782 
5783       if (isa<UndefValue>(Elt))
5784         continue;
5785 
5786       // Bail out if we can't determine if this constant is min/max or if we
5787       // know that this constant is min/max.
5788       auto *CI = dyn_cast<ConstantInt>(Elt);
5789       if (!CI || !ConstantIsOk(CI))
5790         return std::nullopt;
5791 
5792       if (!SafeReplacementConstant)
5793         SafeReplacementConstant = CI;
5794     }
5795   } else {
5796     // ConstantExpr?
5797     return std::nullopt;
5798   }
5799 
5800   // It may not be safe to change a compare predicate in the presence of
5801   // undefined elements, so replace those elements with the first safe constant
5802   // that we found.
5803   // TODO: in case of poison, it is safe; let's replace undefs only.
5804   if (C->containsUndefOrPoisonElement()) {
5805     assert(SafeReplacementConstant && "Replacement constant not set");
5806     C = Constant::replaceUndefsWith(C, SafeReplacementConstant);
5807   }
5808 
5809   CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(Pred);
5810 
5811   // Increment or decrement the constant.
5812   Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true);
5813   Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne);
5814 
5815   return std::make_pair(NewPred, NewC);
5816 }
5817 
5818 /// If we have an icmp le or icmp ge instruction with a constant operand, turn
5819 /// it into the appropriate icmp lt or icmp gt instruction. This transform
5820 /// allows them to be folded in visitICmpInst.
5821 static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) {
5822   ICmpInst::Predicate Pred = I.getPredicate();
5823   if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) ||
5824       InstCombiner::isCanonicalPredicate(Pred))
5825     return nullptr;
5826 
5827   Value *Op0 = I.getOperand(0);
5828   Value *Op1 = I.getOperand(1);
5829   auto *Op1C = dyn_cast<Constant>(Op1);
5830   if (!Op1C)
5831     return nullptr;
5832 
5833   auto FlippedStrictness =
5834       InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, Op1C);
5835   if (!FlippedStrictness)
5836     return nullptr;
5837 
5838   return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second);
5839 }
5840 
5841 /// If we have a comparison with a non-canonical predicate, if we can update
5842 /// all the users, invert the predicate and adjust all the users.
5843 CmpInst *InstCombinerImpl::canonicalizeICmpPredicate(CmpInst &I) {
5844   // Is the predicate already canonical?
5845   CmpInst::Predicate Pred = I.getPredicate();
5846   if (InstCombiner::isCanonicalPredicate(Pred))
5847     return nullptr;
5848 
5849   // Can all users be adjusted to predicate inversion?
5850   if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
5851     return nullptr;
5852 
5853   // Ok, we can canonicalize comparison!
5854   // Let's first invert the comparison's predicate.
5855   I.setPredicate(CmpInst::getInversePredicate(Pred));
5856   I.setName(I.getName() + ".not");
5857 
5858   // And, adapt users.
5859   freelyInvertAllUsersOf(&I);
5860 
5861   return &I;
5862 }
5863 
5864 /// Integer compare with boolean values can always be turned into bitwise ops.
5865 static Instruction *canonicalizeICmpBool(ICmpInst &I,
5866                                          InstCombiner::BuilderTy &Builder) {
5867   Value *A = I.getOperand(0), *B = I.getOperand(1);
5868   assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only");
5869 
5870   // A boolean compared to true/false can be simplified to Op0/true/false in
5871   // 14 out of the 20 (10 predicates * 2 constants) possible combinations.
5872   // Cases not handled by InstSimplify are always 'not' of Op0.
5873   if (match(B, m_Zero())) {
5874     switch (I.getPredicate()) {
5875       case CmpInst::ICMP_EQ:  // A ==   0 -> !A
5876       case CmpInst::ICMP_ULE: // A <=u  0 -> !A
5877       case CmpInst::ICMP_SGE: // A >=s  0 -> !A
5878         return BinaryOperator::CreateNot(A);
5879       default:
5880         llvm_unreachable("ICmp i1 X, C not simplified as expected.");
5881     }
5882   } else if (match(B, m_One())) {
5883     switch (I.getPredicate()) {
5884       case CmpInst::ICMP_NE:  // A !=  1 -> !A
5885       case CmpInst::ICMP_ULT: // A <u  1 -> !A
5886       case CmpInst::ICMP_SGT: // A >s -1 -> !A
5887         return BinaryOperator::CreateNot(A);
5888       default:
5889         llvm_unreachable("ICmp i1 X, C not simplified as expected.");
5890     }
5891   }
5892 
5893   switch (I.getPredicate()) {
5894   default:
5895     llvm_unreachable("Invalid icmp instruction!");
5896   case ICmpInst::ICMP_EQ:
5897     // icmp eq i1 A, B -> ~(A ^ B)
5898     return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
5899 
5900   case ICmpInst::ICMP_NE:
5901     // icmp ne i1 A, B -> A ^ B
5902     return BinaryOperator::CreateXor(A, B);
5903 
5904   case ICmpInst::ICMP_UGT:
5905     // icmp ugt -> icmp ult
5906     std::swap(A, B);
5907     [[fallthrough]];
5908   case ICmpInst::ICMP_ULT:
5909     // icmp ult i1 A, B -> ~A & B
5910     return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
5911 
5912   case ICmpInst::ICMP_SGT:
5913     // icmp sgt -> icmp slt
5914     std::swap(A, B);
5915     [[fallthrough]];
5916   case ICmpInst::ICMP_SLT:
5917     // icmp slt i1 A, B -> A & ~B
5918     return BinaryOperator::CreateAnd(Builder.CreateNot(B), A);
5919 
5920   case ICmpInst::ICMP_UGE:
5921     // icmp uge -> icmp ule
5922     std::swap(A, B);
5923     [[fallthrough]];
5924   case ICmpInst::ICMP_ULE:
5925     // icmp ule i1 A, B -> ~A | B
5926     return BinaryOperator::CreateOr(Builder.CreateNot(A), B);
5927 
5928   case ICmpInst::ICMP_SGE:
5929     // icmp sge -> icmp sle
5930     std::swap(A, B);
5931     [[fallthrough]];
5932   case ICmpInst::ICMP_SLE:
5933     // icmp sle i1 A, B -> A | ~B
5934     return BinaryOperator::CreateOr(Builder.CreateNot(B), A);
5935   }
5936 }
5937 
5938 // Transform pattern like:
5939 //   (1 << Y) u<= X  or  ~(-1 << Y) u<  X  or  ((1 << Y)+(-1)) u<  X
5940 //   (1 << Y) u>  X  or  ~(-1 << Y) u>= X  or  ((1 << Y)+(-1)) u>= X
5941 // Into:
5942 //   (X l>> Y) != 0
5943 //   (X l>> Y) == 0
5944 static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp,
5945                                             InstCombiner::BuilderTy &Builder) {
5946   ICmpInst::Predicate Pred, NewPred;
5947   Value *X, *Y;
5948   if (match(&Cmp,
5949             m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) {
5950     switch (Pred) {
5951     case ICmpInst::ICMP_ULE:
5952       NewPred = ICmpInst::ICMP_NE;
5953       break;
5954     case ICmpInst::ICMP_UGT:
5955       NewPred = ICmpInst::ICMP_EQ;
5956       break;
5957     default:
5958       return nullptr;
5959     }
5960   } else if (match(&Cmp, m_c_ICmp(Pred,
5961                                   m_OneUse(m_CombineOr(
5962                                       m_Not(m_Shl(m_AllOnes(), m_Value(Y))),
5963                                       m_Add(m_Shl(m_One(), m_Value(Y)),
5964                                             m_AllOnes()))),
5965                                   m_Value(X)))) {
5966     // The variant with 'add' is not canonical, (the variant with 'not' is)
5967     // we only get it because it has extra uses, and can't be canonicalized,
5968 
5969     switch (Pred) {
5970     case ICmpInst::ICMP_ULT:
5971       NewPred = ICmpInst::ICMP_NE;
5972       break;
5973     case ICmpInst::ICMP_UGE:
5974       NewPred = ICmpInst::ICMP_EQ;
5975       break;
5976     default:
5977       return nullptr;
5978     }
5979   } else
5980     return nullptr;
5981 
5982   Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits");
5983   Constant *Zero = Constant::getNullValue(NewX->getType());
5984   return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero);
5985 }
5986 
5987 static Instruction *foldVectorCmp(CmpInst &Cmp,
5988                                   InstCombiner::BuilderTy &Builder) {
5989   const CmpInst::Predicate Pred = Cmp.getPredicate();
5990   Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1);
5991   Value *V1, *V2;
5992 
5993   auto createCmpReverse = [&](CmpInst::Predicate Pred, Value *X, Value *Y) {
5994     Value *V = Builder.CreateCmp(Pred, X, Y, Cmp.getName());
5995     if (auto *I = dyn_cast<Instruction>(V))
5996       I->copyIRFlags(&Cmp);
5997     Module *M = Cmp.getModule();
5998     Function *F = Intrinsic::getDeclaration(
5999         M, Intrinsic::experimental_vector_reverse, V->getType());
6000     return CallInst::Create(F, V);
6001   };
6002 
6003   if (match(LHS, m_VecReverse(m_Value(V1)))) {
6004     // cmp Pred, rev(V1), rev(V2) --> rev(cmp Pred, V1, V2)
6005     if (match(RHS, m_VecReverse(m_Value(V2))) &&
6006         (LHS->hasOneUse() || RHS->hasOneUse()))
6007       return createCmpReverse(Pred, V1, V2);
6008 
6009     // cmp Pred, rev(V1), RHSSplat --> rev(cmp Pred, V1, RHSSplat)
6010     if (LHS->hasOneUse() && isSplatValue(RHS))
6011       return createCmpReverse(Pred, V1, RHS);
6012   }
6013   // cmp Pred, LHSSplat, rev(V2) --> rev(cmp Pred, LHSSplat, V2)
6014   else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
6015     return createCmpReverse(Pred, LHS, V2);
6016 
6017   ArrayRef<int> M;
6018   if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M))))
6019     return nullptr;
6020 
6021   // If both arguments of the cmp are shuffles that use the same mask and
6022   // shuffle within a single vector, move the shuffle after the cmp:
6023   // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
6024   Type *V1Ty = V1->getType();
6025   if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
6026       V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
6027     Value *NewCmp = Builder.CreateCmp(Pred, V1, V2);
6028     return new ShuffleVectorInst(NewCmp, M);
6029   }
6030 
6031   // Try to canonicalize compare with splatted operand and splat constant.
6032   // TODO: We could generalize this for more than splats. See/use the code in
6033   //       InstCombiner::foldVectorBinop().
6034   Constant *C;
6035   if (!LHS->hasOneUse() || !match(RHS, m_Constant(C)))
6036     return nullptr;
6037 
6038   // Length-changing splats are ok, so adjust the constants as needed:
6039   // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
6040   Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true);
6041   int MaskSplatIndex;
6042   if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) {
6043     // We allow undefs in matching, but this transform removes those for safety.
6044     // Demanded elements analysis should be able to recover some/all of that.
6045     C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(),
6046                                  ScalarC);
6047     SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
6048     Value *NewCmp = Builder.CreateCmp(Pred, V1, C);
6049     return new ShuffleVectorInst(NewCmp, NewM);
6050   }
6051 
6052   return nullptr;
6053 }
6054 
6055 // extract(uadd.with.overflow(A, B), 0) ult A
6056 //  -> extract(uadd.with.overflow(A, B), 1)
6057 static Instruction *foldICmpOfUAddOv(ICmpInst &I) {
6058   CmpInst::Predicate Pred = I.getPredicate();
6059   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6060 
6061   Value *UAddOv;
6062   Value *A, *B;
6063   auto UAddOvResultPat = m_ExtractValue<0>(
6064       m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B)));
6065   if (match(Op0, UAddOvResultPat) &&
6066       ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) ||
6067        (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) &&
6068         (match(A, m_One()) || match(B, m_One()))) ||
6069        (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) &&
6070         (match(A, m_AllOnes()) || match(B, m_AllOnes())))))
6071     // extract(uadd.with.overflow(A, B), 0) < A
6072     // extract(uadd.with.overflow(A, 1), 0) == 0
6073     // extract(uadd.with.overflow(A, -1), 0) != -1
6074     UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand();
6075   else if (match(Op1, UAddOvResultPat) &&
6076            Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B))
6077     // A > extract(uadd.with.overflow(A, B), 0)
6078     UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand();
6079   else
6080     return nullptr;
6081 
6082   return ExtractValueInst::Create(UAddOv, 1);
6083 }
6084 
6085 static Instruction *foldICmpInvariantGroup(ICmpInst &I) {
6086   if (!I.getOperand(0)->getType()->isPointerTy() ||
6087       NullPointerIsDefined(
6088           I.getParent()->getParent(),
6089           I.getOperand(0)->getType()->getPointerAddressSpace())) {
6090     return nullptr;
6091   }
6092   Instruction *Op;
6093   if (match(I.getOperand(0), m_Instruction(Op)) &&
6094       match(I.getOperand(1), m_Zero()) &&
6095       Op->isLaunderOrStripInvariantGroup()) {
6096     return ICmpInst::Create(Instruction::ICmp, I.getPredicate(),
6097                             Op->getOperand(0), I.getOperand(1));
6098   }
6099   return nullptr;
6100 }
6101 
6102 /// This function folds patterns produced by lowering of reduce idioms, such as
6103 /// llvm.vector.reduce.and which are lowered into instruction chains. This code
6104 /// attempts to generate fewer number of scalar comparisons instead of vector
6105 /// comparisons when possible.
6106 static Instruction *foldReductionIdiom(ICmpInst &I,
6107                                        InstCombiner::BuilderTy &Builder,
6108                                        const DataLayout &DL) {
6109   if (I.getType()->isVectorTy())
6110     return nullptr;
6111   ICmpInst::Predicate OuterPred, InnerPred;
6112   Value *LHS, *RHS;
6113 
6114   // Match lowering of @llvm.vector.reduce.and. Turn
6115   ///   %vec_ne = icmp ne <8 x i8> %lhs, %rhs
6116   ///   %scalar_ne = bitcast <8 x i1> %vec_ne to i8
6117   ///   %res = icmp <pred> i8 %scalar_ne, 0
6118   ///
6119   /// into
6120   ///
6121   ///   %lhs.scalar = bitcast <8 x i8> %lhs to i64
6122   ///   %rhs.scalar = bitcast <8 x i8> %rhs to i64
6123   ///   %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar
6124   ///
6125   /// for <pred> in {ne, eq}.
6126   if (!match(&I, m_ICmp(OuterPred,
6127                         m_OneUse(m_BitCast(m_OneUse(
6128                             m_ICmp(InnerPred, m_Value(LHS), m_Value(RHS))))),
6129                         m_Zero())))
6130     return nullptr;
6131   auto *LHSTy = dyn_cast<FixedVectorType>(LHS->getType());
6132   if (!LHSTy || !LHSTy->getElementType()->isIntegerTy())
6133     return nullptr;
6134   unsigned NumBits =
6135       LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth();
6136   // TODO: Relax this to "not wider than max legal integer type"?
6137   if (!DL.isLegalInteger(NumBits))
6138     return nullptr;
6139 
6140   if (ICmpInst::isEquality(OuterPred) && InnerPred == ICmpInst::ICMP_NE) {
6141     auto *ScalarTy = Builder.getIntNTy(NumBits);
6142     LHS = Builder.CreateBitCast(LHS, ScalarTy, LHS->getName() + ".scalar");
6143     RHS = Builder.CreateBitCast(RHS, ScalarTy, RHS->getName() + ".scalar");
6144     return ICmpInst::Create(Instruction::ICmp, OuterPred, LHS, RHS,
6145                             I.getName());
6146   }
6147 
6148   return nullptr;
6149 }
6150 
6151 Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
6152   bool Changed = false;
6153   const SimplifyQuery Q = SQ.getWithInstruction(&I);
6154   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6155   unsigned Op0Cplxity = getComplexity(Op0);
6156   unsigned Op1Cplxity = getComplexity(Op1);
6157 
6158   /// Orders the operands of the compare so that they are listed from most
6159   /// complex to least complex.  This puts constants before unary operators,
6160   /// before binary operators.
6161   if (Op0Cplxity < Op1Cplxity ||
6162       (Op0Cplxity == Op1Cplxity && swapMayExposeCSEOpportunities(Op0, Op1))) {
6163     I.swapOperands();
6164     std::swap(Op0, Op1);
6165     Changed = true;
6166   }
6167 
6168   if (Value *V = simplifyICmpInst(I.getPredicate(), Op0, Op1, Q))
6169     return replaceInstUsesWith(I, V);
6170 
6171   // Comparing -val or val with non-zero is the same as just comparing val
6172   // ie, abs(val) != 0 -> val != 0
6173   if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) {
6174     Value *Cond, *SelectTrue, *SelectFalse;
6175     if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue),
6176                             m_Value(SelectFalse)))) {
6177       if (Value *V = dyn_castNegVal(SelectTrue)) {
6178         if (V == SelectFalse)
6179           return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
6180       }
6181       else if (Value *V = dyn_castNegVal(SelectFalse)) {
6182         if (V == SelectTrue)
6183           return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1);
6184       }
6185     }
6186   }
6187 
6188   if (Op0->getType()->isIntOrIntVectorTy(1))
6189     if (Instruction *Res = canonicalizeICmpBool(I, Builder))
6190       return Res;
6191 
6192   if (Instruction *Res = canonicalizeCmpWithConstant(I))
6193     return Res;
6194 
6195   if (Instruction *Res = canonicalizeICmpPredicate(I))
6196     return Res;
6197 
6198   if (Instruction *Res = foldICmpWithConstant(I))
6199     return Res;
6200 
6201   if (Instruction *Res = foldICmpWithDominatingICmp(I))
6202     return Res;
6203 
6204   if (Instruction *Res = foldICmpUsingBoolRange(I, Builder))
6205     return Res;
6206 
6207   if (Instruction *Res = foldICmpUsingKnownBits(I))
6208     return Res;
6209 
6210   // Test if the ICmpInst instruction is used exclusively by a select as
6211   // part of a minimum or maximum operation. If so, refrain from doing
6212   // any other folding. This helps out other analyses which understand
6213   // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
6214   // and CodeGen. And in this case, at least one of the comparison
6215   // operands has at least one user besides the compare (the select),
6216   // which would often largely negate the benefit of folding anyway.
6217   //
6218   // Do the same for the other patterns recognized by matchSelectPattern.
6219   if (I.hasOneUse())
6220     if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
6221       Value *A, *B;
6222       SelectPatternResult SPR = matchSelectPattern(SI, A, B);
6223       if (SPR.Flavor != SPF_UNKNOWN)
6224         return nullptr;
6225     }
6226 
6227   // Do this after checking for min/max to prevent infinite looping.
6228   if (Instruction *Res = foldICmpWithZero(I))
6229     return Res;
6230 
6231   // FIXME: We only do this after checking for min/max to prevent infinite
6232   // looping caused by a reverse canonicalization of these patterns for min/max.
6233   // FIXME: The organization of folds is a mess. These would naturally go into
6234   // canonicalizeCmpWithConstant(), but we can't move all of the above folds
6235   // down here after the min/max restriction.
6236   ICmpInst::Predicate Pred = I.getPredicate();
6237   const APInt *C;
6238   if (match(Op1, m_APInt(C))) {
6239     // For i32: x >u 2147483647 -> x <s 0  -> true if sign bit set
6240     if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) {
6241       Constant *Zero = Constant::getNullValue(Op0->getType());
6242       return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero);
6243     }
6244 
6245     // For i32: x <u 2147483648 -> x >s -1  -> true if sign bit clear
6246     if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) {
6247       Constant *AllOnes = Constant::getAllOnesValue(Op0->getType());
6248       return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes);
6249     }
6250   }
6251 
6252   // The folds in here may rely on wrapping flags and special constants, so
6253   // they can break up min/max idioms in some cases but not seemingly similar
6254   // patterns.
6255   // FIXME: It may be possible to enhance select folding to make this
6256   //        unnecessary. It may also be moot if we canonicalize to min/max
6257   //        intrinsics.
6258   if (Instruction *Res = foldICmpBinOp(I, Q))
6259     return Res;
6260 
6261   if (Instruction *Res = foldICmpInstWithConstant(I))
6262     return Res;
6263 
6264   // Try to match comparison as a sign bit test. Intentionally do this after
6265   // foldICmpInstWithConstant() to potentially let other folds to happen first.
6266   if (Instruction *New = foldSignBitTest(I))
6267     return New;
6268 
6269   if (Instruction *Res = foldICmpInstWithConstantNotInt(I))
6270     return Res;
6271 
6272   // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'.
6273   if (auto *GEP = dyn_cast<GEPOperator>(Op0))
6274     if (Instruction *NI = foldGEPICmp(GEP, Op1, I.getPredicate(), I))
6275       return NI;
6276   if (auto *GEP = dyn_cast<GEPOperator>(Op1))
6277     if (Instruction *NI = foldGEPICmp(GEP, Op0, I.getSwappedPredicate(), I))
6278       return NI;
6279 
6280   if (auto *SI = dyn_cast<SelectInst>(Op0))
6281     if (Instruction *NI = foldSelectICmp(I.getPredicate(), SI, Op1, I))
6282       return NI;
6283   if (auto *SI = dyn_cast<SelectInst>(Op1))
6284     if (Instruction *NI = foldSelectICmp(I.getSwappedPredicate(), SI, Op0, I))
6285       return NI;
6286 
6287   // Try to optimize equality comparisons against alloca-based pointers.
6288   if (Op0->getType()->isPointerTy() && I.isEquality()) {
6289     assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
6290     if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
6291       if (Instruction *New = foldAllocaCmp(I, Alloca))
6292         return New;
6293     if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1)))
6294       if (Instruction *New = foldAllocaCmp(I, Alloca))
6295         return New;
6296   }
6297 
6298   if (Instruction *Res = foldICmpBitCast(I))
6299     return Res;
6300 
6301   // TODO: Hoist this above the min/max bailout.
6302   if (Instruction *R = foldICmpWithCastOp(I))
6303     return R;
6304 
6305   if (Instruction *Res = foldICmpWithMinMax(I))
6306     return Res;
6307 
6308   {
6309     Value *A, *B;
6310     // Transform (A & ~B) == 0 --> (A & B) != 0
6311     // and       (A & ~B) != 0 --> (A & B) == 0
6312     // if A is a power of 2.
6313     if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) &&
6314         match(Op1, m_Zero()) &&
6315         isKnownToBeAPowerOfTwo(A, false, 0, &I) && I.isEquality())
6316       return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(A, B),
6317                           Op1);
6318 
6319     // ~X < ~Y --> Y < X
6320     // ~X < C -->  X > ~C
6321     if (match(Op0, m_Not(m_Value(A)))) {
6322       if (match(Op1, m_Not(m_Value(B))))
6323         return new ICmpInst(I.getPredicate(), B, A);
6324 
6325       const APInt *C;
6326       if (match(Op1, m_APInt(C)))
6327         return new ICmpInst(I.getSwappedPredicate(), A,
6328                             ConstantInt::get(Op1->getType(), ~(*C)));
6329     }
6330 
6331     Instruction *AddI = nullptr;
6332     if (match(&I, m_UAddWithOverflow(m_Value(A), m_Value(B),
6333                                      m_Instruction(AddI))) &&
6334         isa<IntegerType>(A->getType())) {
6335       Value *Result;
6336       Constant *Overflow;
6337       // m_UAddWithOverflow can match patterns that do not include  an explicit
6338       // "add" instruction, so check the opcode of the matched op.
6339       if (AddI->getOpcode() == Instruction::Add &&
6340           OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, A, B, *AddI,
6341                                 Result, Overflow)) {
6342         replaceInstUsesWith(*AddI, Result);
6343         eraseInstFromFunction(*AddI);
6344         return replaceInstUsesWith(I, Overflow);
6345       }
6346     }
6347 
6348     // (zext a) * (zext b)  --> llvm.umul.with.overflow.
6349     if (match(Op0, m_NUWMul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) {
6350       if (Instruction *R = processUMulZExtIdiom(I, Op0, Op1, *this))
6351         return R;
6352     }
6353     if (match(Op1, m_NUWMul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) {
6354       if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this))
6355         return R;
6356     }
6357   }
6358 
6359   if (Instruction *Res = foldICmpEquality(I))
6360     return Res;
6361 
6362   if (Instruction *Res = foldICmpOfUAddOv(I))
6363     return Res;
6364 
6365   // The 'cmpxchg' instruction returns an aggregate containing the old value and
6366   // an i1 which indicates whether or not we successfully did the swap.
6367   //
6368   // Replace comparisons between the old value and the expected value with the
6369   // indicator that 'cmpxchg' returns.
6370   //
6371   // N.B.  This transform is only valid when the 'cmpxchg' is not permitted to
6372   // spuriously fail.  In those cases, the old value may equal the expected
6373   // value but it is possible for the swap to not occur.
6374   if (I.getPredicate() == ICmpInst::ICMP_EQ)
6375     if (auto *EVI = dyn_cast<ExtractValueInst>(Op0))
6376       if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand()))
6377         if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 &&
6378             !ACXI->isWeak())
6379           return ExtractValueInst::Create(ACXI, 1);
6380 
6381   {
6382     Value *X;
6383     const APInt *C;
6384     // icmp X+Cst, X
6385     if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X)
6386       return foldICmpAddOpConst(X, *C, I.getPredicate());
6387 
6388     // icmp X, X+Cst
6389     if (match(Op1, m_Add(m_Value(X), m_APInt(C))) && Op0 == X)
6390       return foldICmpAddOpConst(X, *C, I.getSwappedPredicate());
6391   }
6392 
6393   if (Instruction *Res = foldICmpWithHighBitMask(I, Builder))
6394     return Res;
6395 
6396   if (I.getType()->isVectorTy())
6397     if (Instruction *Res = foldVectorCmp(I, Builder))
6398       return Res;
6399 
6400   if (Instruction *Res = foldICmpInvariantGroup(I))
6401     return Res;
6402 
6403   if (Instruction *Res = foldReductionIdiom(I, Builder, DL))
6404     return Res;
6405 
6406   return Changed ? &I : nullptr;
6407 }
6408 
6409 /// Fold fcmp ([us]itofp x, cst) if possible.
6410 Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
6411                                                     Instruction *LHSI,
6412                                                     Constant *RHSC) {
6413   if (!isa<ConstantFP>(RHSC)) return nullptr;
6414   const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF();
6415 
6416   // Get the width of the mantissa.  We don't want to hack on conversions that
6417   // might lose information from the integer, e.g. "i64 -> float"
6418   int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
6419   if (MantissaWidth == -1) return nullptr;  // Unknown.
6420 
6421   IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType());
6422 
6423   bool LHSUnsigned = isa<UIToFPInst>(LHSI);
6424 
6425   if (I.isEquality()) {
6426     FCmpInst::Predicate P = I.getPredicate();
6427     bool IsExact = false;
6428     APSInt RHSCvt(IntTy->getBitWidth(), LHSUnsigned);
6429     RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact);
6430 
6431     // If the floating point constant isn't an integer value, we know if we will
6432     // ever compare equal / not equal to it.
6433     if (!IsExact) {
6434       // TODO: Can never be -0.0 and other non-representable values
6435       APFloat RHSRoundInt(RHS);
6436       RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven);
6437       if (RHS != RHSRoundInt) {
6438         if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ)
6439           return replaceInstUsesWith(I, Builder.getFalse());
6440 
6441         assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE);
6442         return replaceInstUsesWith(I, Builder.getTrue());
6443       }
6444     }
6445 
6446     // TODO: If the constant is exactly representable, is it always OK to do
6447     // equality compares as integer?
6448   }
6449 
6450   // Check to see that the input is converted from an integer type that is small
6451   // enough that preserves all bits.  TODO: check here for "known" sign bits.
6452   // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e.
6453   unsigned InputSize = IntTy->getScalarSizeInBits();
6454 
6455   // Following test does NOT adjust InputSize downwards for signed inputs,
6456   // because the most negative value still requires all the mantissa bits
6457   // to distinguish it from one less than that value.
6458   if ((int)InputSize > MantissaWidth) {
6459     // Conversion would lose accuracy. Check if loss can impact comparison.
6460     int Exp = ilogb(RHS);
6461     if (Exp == APFloat::IEK_Inf) {
6462       int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics()));
6463       if (MaxExponent < (int)InputSize - !LHSUnsigned)
6464         // Conversion could create infinity.
6465         return nullptr;
6466     } else {
6467       // Note that if RHS is zero or NaN, then Exp is negative
6468       // and first condition is trivially false.
6469       if (MantissaWidth <= Exp && Exp <= (int)InputSize - !LHSUnsigned)
6470         // Conversion could affect comparison.
6471         return nullptr;
6472     }
6473   }
6474 
6475   // Otherwise, we can potentially simplify the comparison.  We know that it
6476   // will always come through as an integer value and we know the constant is
6477   // not a NAN (it would have been previously simplified).
6478   assert(!RHS.isNaN() && "NaN comparison not already folded!");
6479 
6480   ICmpInst::Predicate Pred;
6481   switch (I.getPredicate()) {
6482   default: llvm_unreachable("Unexpected predicate!");
6483   case FCmpInst::FCMP_UEQ:
6484   case FCmpInst::FCMP_OEQ:
6485     Pred = ICmpInst::ICMP_EQ;
6486     break;
6487   case FCmpInst::FCMP_UGT:
6488   case FCmpInst::FCMP_OGT:
6489     Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT;
6490     break;
6491   case FCmpInst::FCMP_UGE:
6492   case FCmpInst::FCMP_OGE:
6493     Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE;
6494     break;
6495   case FCmpInst::FCMP_ULT:
6496   case FCmpInst::FCMP_OLT:
6497     Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT;
6498     break;
6499   case FCmpInst::FCMP_ULE:
6500   case FCmpInst::FCMP_OLE:
6501     Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE;
6502     break;
6503   case FCmpInst::FCMP_UNE:
6504   case FCmpInst::FCMP_ONE:
6505     Pred = ICmpInst::ICMP_NE;
6506     break;
6507   case FCmpInst::FCMP_ORD:
6508     return replaceInstUsesWith(I, Builder.getTrue());
6509   case FCmpInst::FCMP_UNO:
6510     return replaceInstUsesWith(I, Builder.getFalse());
6511   }
6512 
6513   // Now we know that the APFloat is a normal number, zero or inf.
6514 
6515   // See if the FP constant is too large for the integer.  For example,
6516   // comparing an i8 to 300.0.
6517   unsigned IntWidth = IntTy->getScalarSizeInBits();
6518 
6519   if (!LHSUnsigned) {
6520     // If the RHS value is > SignedMax, fold the comparison.  This handles +INF
6521     // and large values.
6522     APFloat SMax(RHS.getSemantics());
6523     SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true,
6524                           APFloat::rmNearestTiesToEven);
6525     if (SMax < RHS) { // smax < 13123.0
6526       if (Pred == ICmpInst::ICMP_NE  || Pred == ICmpInst::ICMP_SLT ||
6527           Pred == ICmpInst::ICMP_SLE)
6528         return replaceInstUsesWith(I, Builder.getTrue());
6529       return replaceInstUsesWith(I, Builder.getFalse());
6530     }
6531   } else {
6532     // If the RHS value is > UnsignedMax, fold the comparison. This handles
6533     // +INF and large values.
6534     APFloat UMax(RHS.getSemantics());
6535     UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false,
6536                           APFloat::rmNearestTiesToEven);
6537     if (UMax < RHS) { // umax < 13123.0
6538       if (Pred == ICmpInst::ICMP_NE  || Pred == ICmpInst::ICMP_ULT ||
6539           Pred == ICmpInst::ICMP_ULE)
6540         return replaceInstUsesWith(I, Builder.getTrue());
6541       return replaceInstUsesWith(I, Builder.getFalse());
6542     }
6543   }
6544 
6545   if (!LHSUnsigned) {
6546     // See if the RHS value is < SignedMin.
6547     APFloat SMin(RHS.getSemantics());
6548     SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true,
6549                           APFloat::rmNearestTiesToEven);
6550     if (SMin > RHS) { // smin > 12312.0
6551       if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT ||
6552           Pred == ICmpInst::ICMP_SGE)
6553         return replaceInstUsesWith(I, Builder.getTrue());
6554       return replaceInstUsesWith(I, Builder.getFalse());
6555     }
6556   } else {
6557     // See if the RHS value is < UnsignedMin.
6558     APFloat UMin(RHS.getSemantics());
6559     UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false,
6560                           APFloat::rmNearestTiesToEven);
6561     if (UMin > RHS) { // umin > 12312.0
6562       if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT ||
6563           Pred == ICmpInst::ICMP_UGE)
6564         return replaceInstUsesWith(I, Builder.getTrue());
6565       return replaceInstUsesWith(I, Builder.getFalse());
6566     }
6567   }
6568 
6569   // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or
6570   // [0, UMAX], but it may still be fractional.  See if it is fractional by
6571   // casting the FP value to the integer value and back, checking for equality.
6572   // Don't do this for zero, because -0.0 is not fractional.
6573   Constant *RHSInt = LHSUnsigned
6574     ? ConstantExpr::getFPToUI(RHSC, IntTy)
6575     : ConstantExpr::getFPToSI(RHSC, IntTy);
6576   if (!RHS.isZero()) {
6577     bool Equal = LHSUnsigned
6578       ? ConstantExpr::getUIToFP(RHSInt, RHSC->getType()) == RHSC
6579       : ConstantExpr::getSIToFP(RHSInt, RHSC->getType()) == RHSC;
6580     if (!Equal) {
6581       // If we had a comparison against a fractional value, we have to adjust
6582       // the compare predicate and sometimes the value.  RHSC is rounded towards
6583       // zero at this point.
6584       switch (Pred) {
6585       default: llvm_unreachable("Unexpected integer comparison!");
6586       case ICmpInst::ICMP_NE:  // (float)int != 4.4   --> true
6587         return replaceInstUsesWith(I, Builder.getTrue());
6588       case ICmpInst::ICMP_EQ:  // (float)int == 4.4   --> false
6589         return replaceInstUsesWith(I, Builder.getFalse());
6590       case ICmpInst::ICMP_ULE:
6591         // (float)int <= 4.4   --> int <= 4
6592         // (float)int <= -4.4  --> false
6593         if (RHS.isNegative())
6594           return replaceInstUsesWith(I, Builder.getFalse());
6595         break;
6596       case ICmpInst::ICMP_SLE:
6597         // (float)int <= 4.4   --> int <= 4
6598         // (float)int <= -4.4  --> int < -4
6599         if (RHS.isNegative())
6600           Pred = ICmpInst::ICMP_SLT;
6601         break;
6602       case ICmpInst::ICMP_ULT:
6603         // (float)int < -4.4   --> false
6604         // (float)int < 4.4    --> int <= 4
6605         if (RHS.isNegative())
6606           return replaceInstUsesWith(I, Builder.getFalse());
6607         Pred = ICmpInst::ICMP_ULE;
6608         break;
6609       case ICmpInst::ICMP_SLT:
6610         // (float)int < -4.4   --> int < -4
6611         // (float)int < 4.4    --> int <= 4
6612         if (!RHS.isNegative())
6613           Pred = ICmpInst::ICMP_SLE;
6614         break;
6615       case ICmpInst::ICMP_UGT:
6616         // (float)int > 4.4    --> int > 4
6617         // (float)int > -4.4   --> true
6618         if (RHS.isNegative())
6619           return replaceInstUsesWith(I, Builder.getTrue());
6620         break;
6621       case ICmpInst::ICMP_SGT:
6622         // (float)int > 4.4    --> int > 4
6623         // (float)int > -4.4   --> int >= -4
6624         if (RHS.isNegative())
6625           Pred = ICmpInst::ICMP_SGE;
6626         break;
6627       case ICmpInst::ICMP_UGE:
6628         // (float)int >= -4.4   --> true
6629         // (float)int >= 4.4    --> int > 4
6630         if (RHS.isNegative())
6631           return replaceInstUsesWith(I, Builder.getTrue());
6632         Pred = ICmpInst::ICMP_UGT;
6633         break;
6634       case ICmpInst::ICMP_SGE:
6635         // (float)int >= -4.4   --> int >= -4
6636         // (float)int >= 4.4    --> int > 4
6637         if (!RHS.isNegative())
6638           Pred = ICmpInst::ICMP_SGT;
6639         break;
6640       }
6641     }
6642   }
6643 
6644   // Lower this FP comparison into an appropriate integer version of the
6645   // comparison.
6646   return new ICmpInst(Pred, LHSI->getOperand(0), RHSInt);
6647 }
6648 
6649 /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
6650 static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI,
6651                                               Constant *RHSC) {
6652   // When C is not 0.0 and infinities are not allowed:
6653   // (C / X) < 0.0 is a sign-bit test of X
6654   // (C / X) < 0.0 --> X < 0.0 (if C is positive)
6655   // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate)
6656   //
6657   // Proof:
6658   // Multiply (C / X) < 0.0 by X * X / C.
6659   // - X is non zero, if it is the flag 'ninf' is violated.
6660   // - C defines the sign of X * X * C. Thus it also defines whether to swap
6661   //   the predicate. C is also non zero by definition.
6662   //
6663   // Thus X * X / C is non zero and the transformation is valid. [qed]
6664 
6665   FCmpInst::Predicate Pred = I.getPredicate();
6666 
6667   // Check that predicates are valid.
6668   if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) &&
6669       (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE))
6670     return nullptr;
6671 
6672   // Check that RHS operand is zero.
6673   if (!match(RHSC, m_AnyZeroFP()))
6674     return nullptr;
6675 
6676   // Check fastmath flags ('ninf').
6677   if (!LHSI->hasNoInfs() || !I.hasNoInfs())
6678     return nullptr;
6679 
6680   // Check the properties of the dividend. It must not be zero to avoid a
6681   // division by zero (see Proof).
6682   const APFloat *C;
6683   if (!match(LHSI->getOperand(0), m_APFloat(C)))
6684     return nullptr;
6685 
6686   if (C->isZero())
6687     return nullptr;
6688 
6689   // Get swapped predicate if necessary.
6690   if (C->isNegative())
6691     Pred = I.getSwappedPredicate();
6692 
6693   return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I);
6694 }
6695 
6696 /// Optimize fabs(X) compared with zero.
6697 static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) {
6698   Value *X;
6699   if (!match(I.getOperand(0), m_FAbs(m_Value(X))))
6700     return nullptr;
6701 
6702   const APFloat *C;
6703   if (!match(I.getOperand(1), m_APFloat(C)))
6704     return nullptr;
6705 
6706   if (!C->isPosZero()) {
6707     if (!C->isSmallestNormalized())
6708       return nullptr;
6709 
6710     const Function *F = I.getFunction();
6711     DenormalMode Mode = F->getDenormalMode(C->getSemantics());
6712     if (Mode.Input == DenormalMode::PreserveSign ||
6713         Mode.Input == DenormalMode::PositiveZero) {
6714 
6715       auto replaceFCmp = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
6716         Constant *Zero = ConstantFP::getNullValue(X->getType());
6717         return new FCmpInst(P, X, Zero, "", I);
6718       };
6719 
6720       switch (I.getPredicate()) {
6721       case FCmpInst::FCMP_OLT:
6722         // fcmp olt fabs(x), smallest_normalized_number -> fcmp oeq x, 0.0
6723         return replaceFCmp(&I, FCmpInst::FCMP_OEQ, X);
6724       case FCmpInst::FCMP_UGE:
6725         // fcmp uge fabs(x), smallest_normalized_number -> fcmp une x, 0.0
6726         return replaceFCmp(&I, FCmpInst::FCMP_UNE, X);
6727       case FCmpInst::FCMP_OGE:
6728         // fcmp oge fabs(x), smallest_normalized_number -> fcmp one x, 0.0
6729         return replaceFCmp(&I, FCmpInst::FCMP_ONE, X);
6730       case FCmpInst::FCMP_ULT:
6731         // fcmp ult fabs(x), smallest_normalized_number -> fcmp ueq x, 0.0
6732         return replaceFCmp(&I, FCmpInst::FCMP_UEQ, X);
6733       default:
6734         break;
6735       }
6736     }
6737 
6738     return nullptr;
6739   }
6740 
6741   auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
6742     I->setPredicate(P);
6743     return IC.replaceOperand(*I, 0, X);
6744   };
6745 
6746   switch (I.getPredicate()) {
6747   case FCmpInst::FCMP_UGE:
6748   case FCmpInst::FCMP_OLT:
6749     // fabs(X) >= 0.0 --> true
6750     // fabs(X) <  0.0 --> false
6751     llvm_unreachable("fcmp should have simplified");
6752 
6753   case FCmpInst::FCMP_OGT:
6754     // fabs(X) > 0.0 --> X != 0.0
6755     return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X);
6756 
6757   case FCmpInst::FCMP_UGT:
6758     // fabs(X) u> 0.0 --> X u!= 0.0
6759     return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X);
6760 
6761   case FCmpInst::FCMP_OLE:
6762     // fabs(X) <= 0.0 --> X == 0.0
6763     return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X);
6764 
6765   case FCmpInst::FCMP_ULE:
6766     // fabs(X) u<= 0.0 --> X u== 0.0
6767     return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X);
6768 
6769   case FCmpInst::FCMP_OGE:
6770     // fabs(X) >= 0.0 --> !isnan(X)
6771     assert(!I.hasNoNaNs() && "fcmp should have simplified");
6772     return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X);
6773 
6774   case FCmpInst::FCMP_ULT:
6775     // fabs(X) u< 0.0 --> isnan(X)
6776     assert(!I.hasNoNaNs() && "fcmp should have simplified");
6777     return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X);
6778 
6779   case FCmpInst::FCMP_OEQ:
6780   case FCmpInst::FCMP_UEQ:
6781   case FCmpInst::FCMP_ONE:
6782   case FCmpInst::FCMP_UNE:
6783   case FCmpInst::FCMP_ORD:
6784   case FCmpInst::FCMP_UNO:
6785     // Look through the fabs() because it doesn't change anything but the sign.
6786     // fabs(X) == 0.0 --> X == 0.0,
6787     // fabs(X) != 0.0 --> X != 0.0
6788     // isnan(fabs(X)) --> isnan(X)
6789     // !isnan(fabs(X) --> !isnan(X)
6790     return replacePredAndOp0(&I, I.getPredicate(), X);
6791 
6792   default:
6793     return nullptr;
6794   }
6795 }
6796 
6797 static Instruction *foldFCmpFNegCommonOp(FCmpInst &I) {
6798   CmpInst::Predicate Pred = I.getPredicate();
6799   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6800 
6801   // Canonicalize fneg as Op1.
6802   if (match(Op0, m_FNeg(m_Value())) && !match(Op1, m_FNeg(m_Value()))) {
6803     std::swap(Op0, Op1);
6804     Pred = I.getSwappedPredicate();
6805   }
6806 
6807   if (!match(Op1, m_FNeg(m_Specific(Op0))))
6808     return nullptr;
6809 
6810   // Replace the negated operand with 0.0:
6811   // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0
6812   Constant *Zero = ConstantFP::getNullValue(Op0->getType());
6813   return new FCmpInst(Pred, Op0, Zero, "", &I);
6814 }
6815 
6816 Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) {
6817   bool Changed = false;
6818 
6819   /// Orders the operands of the compare so that they are listed from most
6820   /// complex to least complex.  This puts constants before unary operators,
6821   /// before binary operators.
6822   if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) {
6823     I.swapOperands();
6824     Changed = true;
6825   }
6826 
6827   const CmpInst::Predicate Pred = I.getPredicate();
6828   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
6829   if (Value *V = simplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(),
6830                                   SQ.getWithInstruction(&I)))
6831     return replaceInstUsesWith(I, V);
6832 
6833   // Simplify 'fcmp pred X, X'
6834   Type *OpType = Op0->getType();
6835   assert(OpType == Op1->getType() && "fcmp with different-typed operands?");
6836   if (Op0 == Op1) {
6837     switch (Pred) {
6838       default: break;
6839     case FCmpInst::FCMP_UNO:    // True if unordered: isnan(X) | isnan(Y)
6840     case FCmpInst::FCMP_ULT:    // True if unordered or less than
6841     case FCmpInst::FCMP_UGT:    // True if unordered or greater than
6842     case FCmpInst::FCMP_UNE:    // True if unordered or not equal
6843       // Canonicalize these to be 'fcmp uno %X, 0.0'.
6844       I.setPredicate(FCmpInst::FCMP_UNO);
6845       I.setOperand(1, Constant::getNullValue(OpType));
6846       return &I;
6847 
6848     case FCmpInst::FCMP_ORD:    // True if ordered (no nans)
6849     case FCmpInst::FCMP_OEQ:    // True if ordered and equal
6850     case FCmpInst::FCMP_OGE:    // True if ordered and greater than or equal
6851     case FCmpInst::FCMP_OLE:    // True if ordered and less than or equal
6852       // Canonicalize these to be 'fcmp ord %X, 0.0'.
6853       I.setPredicate(FCmpInst::FCMP_ORD);
6854       I.setOperand(1, Constant::getNullValue(OpType));
6855       return &I;
6856     }
6857   }
6858 
6859   // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand,
6860   // then canonicalize the operand to 0.0.
6861   if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
6862     if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI))
6863       return replaceOperand(I, 0, ConstantFP::getNullValue(OpType));
6864 
6865     if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI))
6866       return replaceOperand(I, 1, ConstantFP::getNullValue(OpType));
6867   }
6868 
6869   // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y
6870   Value *X, *Y;
6871   if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y))))
6872     return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I);
6873 
6874   if (Instruction *R = foldFCmpFNegCommonOp(I))
6875     return R;
6876 
6877   // Test if the FCmpInst instruction is used exclusively by a select as
6878   // part of a minimum or maximum operation. If so, refrain from doing
6879   // any other folding. This helps out other analyses which understand
6880   // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
6881   // and CodeGen. And in this case, at least one of the comparison
6882   // operands has at least one user besides the compare (the select),
6883   // which would often largely negate the benefit of folding anyway.
6884   if (I.hasOneUse())
6885     if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) {
6886       Value *A, *B;
6887       SelectPatternResult SPR = matchSelectPattern(SI, A, B);
6888       if (SPR.Flavor != SPF_UNKNOWN)
6889         return nullptr;
6890     }
6891 
6892   // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0:
6893   // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0
6894   if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP()))
6895     return replaceOperand(I, 1, ConstantFP::getNullValue(OpType));
6896 
6897   // Ignore signbit of bitcasted int when comparing equality to FP 0.0:
6898   // fcmp oeq/une (bitcast X), 0.0 --> (and X, SignMaskC) ==/!= 0
6899   if (match(Op1, m_PosZeroFP()) &&
6900       match(Op0, m_OneUse(m_BitCast(m_Value(X)))) &&
6901       X->getType()->isVectorTy() == OpType->isVectorTy() &&
6902       X->getType()->getScalarSizeInBits() == OpType->getScalarSizeInBits()) {
6903     ICmpInst::Predicate IntPred = ICmpInst::BAD_ICMP_PREDICATE;
6904     if (Pred == FCmpInst::FCMP_OEQ)
6905       IntPred = ICmpInst::ICMP_EQ;
6906     else if (Pred == FCmpInst::FCMP_UNE)
6907       IntPred = ICmpInst::ICMP_NE;
6908 
6909     if (IntPred != ICmpInst::BAD_ICMP_PREDICATE) {
6910       Type *IntTy = X->getType();
6911       const APInt &SignMask = ~APInt::getSignMask(IntTy->getScalarSizeInBits());
6912       Value *MaskX = Builder.CreateAnd(X, ConstantInt::get(IntTy, SignMask));
6913       return new ICmpInst(IntPred, MaskX, ConstantInt::getNullValue(IntTy));
6914     }
6915   }
6916 
6917   // Handle fcmp with instruction LHS and constant RHS.
6918   Instruction *LHSI;
6919   Constant *RHSC;
6920   if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) {
6921     switch (LHSI->getOpcode()) {
6922     case Instruction::PHI:
6923       // Only fold fcmp into the PHI if the phi and fcmp are in the same
6924       // block.  If in the same block, we're encouraging jump threading.  If
6925       // not, we are just pessimizing the code by making an i1 phi.
6926       if (LHSI->getParent() == I.getParent())
6927         if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI)))
6928           return NV;
6929       break;
6930     case Instruction::SIToFP:
6931     case Instruction::UIToFP:
6932       if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC))
6933         return NV;
6934       break;
6935     case Instruction::FDiv:
6936       if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC))
6937         return NV;
6938       break;
6939     case Instruction::Load:
6940       if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0)))
6941         if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)))
6942           if (Instruction *Res = foldCmpLoadFromIndexedGlobal(
6943                   cast<LoadInst>(LHSI), GEP, GV, I))
6944             return Res;
6945       break;
6946   }
6947   }
6948 
6949   if (Instruction *R = foldFabsWithFcmpZero(I, *this))
6950     return R;
6951 
6952   if (match(Op0, m_FNeg(m_Value(X)))) {
6953     // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C
6954     Constant *C;
6955     if (match(Op1, m_Constant(C)))
6956       if (Constant *NegC = ConstantFoldUnaryOpOperand(Instruction::FNeg, C, DL))
6957         return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I);
6958   }
6959 
6960   if (match(Op0, m_FPExt(m_Value(X)))) {
6961     // fcmp (fpext X), (fpext Y) -> fcmp X, Y
6962     if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType())
6963       return new FCmpInst(Pred, X, Y, "", &I);
6964 
6965     const APFloat *C;
6966     if (match(Op1, m_APFloat(C))) {
6967       const fltSemantics &FPSem =
6968           X->getType()->getScalarType()->getFltSemantics();
6969       bool Lossy;
6970       APFloat TruncC = *C;
6971       TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy);
6972 
6973       if (Lossy) {
6974         // X can't possibly equal the higher-precision constant, so reduce any
6975         // equality comparison.
6976         // TODO: Other predicates can be handled via getFCmpCode().
6977         switch (Pred) {
6978         case FCmpInst::FCMP_OEQ:
6979           // X is ordered and equal to an impossible constant --> false
6980           return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType()));
6981         case FCmpInst::FCMP_ONE:
6982           // X is ordered and not equal to an impossible constant --> ordered
6983           return new FCmpInst(FCmpInst::FCMP_ORD, X,
6984                               ConstantFP::getNullValue(X->getType()));
6985         case FCmpInst::FCMP_UEQ:
6986           // X is unordered or equal to an impossible constant --> unordered
6987           return new FCmpInst(FCmpInst::FCMP_UNO, X,
6988                               ConstantFP::getNullValue(X->getType()));
6989         case FCmpInst::FCMP_UNE:
6990           // X is unordered or not equal to an impossible constant --> true
6991           return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType()));
6992         default:
6993           break;
6994         }
6995       }
6996 
6997       // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless
6998       // Avoid lossy conversions and denormals.
6999       // Zero is a special case that's OK to convert.
7000       APFloat Fabs = TruncC;
7001       Fabs.clearSign();
7002       if (!Lossy &&
7003           (Fabs.isZero() || !(Fabs < APFloat::getSmallestNormalized(FPSem)))) {
7004         Constant *NewC = ConstantFP::get(X->getType(), TruncC);
7005         return new FCmpInst(Pred, X, NewC, "", &I);
7006       }
7007     }
7008   }
7009 
7010   // Convert a sign-bit test of an FP value into a cast and integer compare.
7011   // TODO: Simplify if the copysign constant is 0.0 or NaN.
7012   // TODO: Handle non-zero compare constants.
7013   // TODO: Handle other predicates.
7014   const APFloat *C;
7015   if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C),
7016                                                            m_Value(X)))) &&
7017       match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) {
7018     Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits());
7019     if (auto *VecTy = dyn_cast<VectorType>(OpType))
7020       IntType = VectorType::get(IntType, VecTy->getElementCount());
7021 
7022     // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0
7023     if (Pred == FCmpInst::FCMP_OLT) {
7024       Value *IntX = Builder.CreateBitCast(X, IntType);
7025       return new ICmpInst(ICmpInst::ICMP_SLT, IntX,
7026                           ConstantInt::getNullValue(IntType));
7027     }
7028   }
7029 
7030   {
7031     Value *CanonLHS = nullptr, *CanonRHS = nullptr;
7032     match(Op0, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonLHS)));
7033     match(Op1, m_Intrinsic<Intrinsic::canonicalize>(m_Value(CanonRHS)));
7034 
7035     // (canonicalize(x) == x) => (x == x)
7036     if (CanonLHS == Op1)
7037       return new FCmpInst(Pred, Op1, Op1, "", &I);
7038 
7039     // (x == canonicalize(x)) => (x == x)
7040     if (CanonRHS == Op0)
7041       return new FCmpInst(Pred, Op0, Op0, "", &I);
7042 
7043     // (canonicalize(x) == canonicalize(y)) => (x == y)
7044     if (CanonLHS && CanonRHS)
7045       return new FCmpInst(Pred, CanonLHS, CanonRHS, "", &I);
7046   }
7047 
7048   if (I.getType()->isVectorTy())
7049     if (Instruction *Res = foldVectorCmp(I, Builder))
7050       return Res;
7051 
7052   return Changed ? &I : nullptr;
7053 }
7054