1 //===- InstCombineCompares.cpp --------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the visitICmp and visitFCmp functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "InstCombineInternal.h" 14 #include "llvm/ADT/APSInt.h" 15 #include "llvm/ADT/SetVector.h" 16 #include "llvm/ADT/Statistic.h" 17 #include "llvm/Analysis/ConstantFolding.h" 18 #include "llvm/Analysis/InstructionSimplify.h" 19 #include "llvm/Analysis/TargetLibraryInfo.h" 20 #include "llvm/IR/ConstantRange.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/IR/GetElementPtrTypeIterator.h" 23 #include "llvm/IR/IntrinsicInst.h" 24 #include "llvm/IR/PatternMatch.h" 25 #include "llvm/Support/Debug.h" 26 #include "llvm/Support/KnownBits.h" 27 #include "llvm/Transforms/InstCombine/InstCombiner.h" 28 29 using namespace llvm; 30 using namespace PatternMatch; 31 32 #define DEBUG_TYPE "instcombine" 33 34 // How many times is a select replaced by one of its operands? 35 STATISTIC(NumSel, "Number of select opts"); 36 37 38 /// Compute Result = In1+In2, returning true if the result overflowed for this 39 /// type. 40 static bool addWithOverflow(APInt &Result, const APInt &In1, 41 const APInt &In2, bool IsSigned = false) { 42 bool Overflow; 43 if (IsSigned) 44 Result = In1.sadd_ov(In2, Overflow); 45 else 46 Result = In1.uadd_ov(In2, Overflow); 47 48 return Overflow; 49 } 50 51 /// Compute Result = In1-In2, returning true if the result overflowed for this 52 /// type. 53 static bool subWithOverflow(APInt &Result, const APInt &In1, 54 const APInt &In2, bool IsSigned = false) { 55 bool Overflow; 56 if (IsSigned) 57 Result = In1.ssub_ov(In2, Overflow); 58 else 59 Result = In1.usub_ov(In2, Overflow); 60 61 return Overflow; 62 } 63 64 /// Given an icmp instruction, return true if any use of this comparison is a 65 /// branch on sign bit comparison. 66 static bool hasBranchUse(ICmpInst &I) { 67 for (auto *U : I.users()) 68 if (isa<BranchInst>(U)) 69 return true; 70 return false; 71 } 72 73 /// Returns true if the exploded icmp can be expressed as a signed comparison 74 /// to zero and updates the predicate accordingly. 75 /// The signedness of the comparison is preserved. 76 /// TODO: Refactor with decomposeBitTestICmp()? 77 static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { 78 if (!ICmpInst::isSigned(Pred)) 79 return false; 80 81 if (C.isNullValue()) 82 return ICmpInst::isRelational(Pred); 83 84 if (C.isOneValue()) { 85 if (Pred == ICmpInst::ICMP_SLT) { 86 Pred = ICmpInst::ICMP_SLE; 87 return true; 88 } 89 } else if (C.isAllOnesValue()) { 90 if (Pred == ICmpInst::ICMP_SGT) { 91 Pred = ICmpInst::ICMP_SGE; 92 return true; 93 } 94 } 95 96 return false; 97 } 98 99 /// This is called when we see this pattern: 100 /// cmp pred (load (gep GV, ...)), cmpcst 101 /// where GV is a global variable with a constant initializer. Try to simplify 102 /// this into some simple computation that does not need the load. For example 103 /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3". 104 /// 105 /// If AndCst is non-null, then the loaded value is masked with that constant 106 /// before doing the comparison. This handles cases like "A[i]&4 == 0". 107 Instruction * 108 InstCombinerImpl::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, 109 GlobalVariable *GV, CmpInst &ICI, 110 ConstantInt *AndCst) { 111 Constant *Init = GV->getInitializer(); 112 if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init)) 113 return nullptr; 114 115 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); 116 // Don't blow up on huge arrays. 117 if (ArrayElementCount > MaxArraySizeForCombine) 118 return nullptr; 119 120 // There are many forms of this optimization we can handle, for now, just do 121 // the simple index into a single-dimensional array. 122 // 123 // Require: GEP GV, 0, i {{, constant indices}} 124 if (GEP->getNumOperands() < 3 || 125 !isa<ConstantInt>(GEP->getOperand(1)) || 126 !cast<ConstantInt>(GEP->getOperand(1))->isZero() || 127 isa<Constant>(GEP->getOperand(2))) 128 return nullptr; 129 130 // Check that indices after the variable are constants and in-range for the 131 // type they index. Collect the indices. This is typically for arrays of 132 // structs. 133 SmallVector<unsigned, 4> LaterIndices; 134 135 Type *EltTy = Init->getType()->getArrayElementType(); 136 for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) { 137 ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i)); 138 if (!Idx) return nullptr; // Variable index. 139 140 uint64_t IdxVal = Idx->getZExtValue(); 141 if ((unsigned)IdxVal != IdxVal) return nullptr; // Too large array index. 142 143 if (StructType *STy = dyn_cast<StructType>(EltTy)) 144 EltTy = STy->getElementType(IdxVal); 145 else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) { 146 if (IdxVal >= ATy->getNumElements()) return nullptr; 147 EltTy = ATy->getElementType(); 148 } else { 149 return nullptr; // Unknown type. 150 } 151 152 LaterIndices.push_back(IdxVal); 153 } 154 155 enum { Overdefined = -3, Undefined = -2 }; 156 157 // Variables for our state machines. 158 159 // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form 160 // "i == 47 | i == 87", where 47 is the first index the condition is true for, 161 // and 87 is the second (and last) index. FirstTrueElement is -2 when 162 // undefined, otherwise set to the first true element. SecondTrueElement is 163 // -2 when undefined, -3 when overdefined and >= 0 when that index is true. 164 int FirstTrueElement = Undefined, SecondTrueElement = Undefined; 165 166 // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the 167 // form "i != 47 & i != 87". Same state transitions as for true elements. 168 int FirstFalseElement = Undefined, SecondFalseElement = Undefined; 169 170 /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these 171 /// define a state machine that triggers for ranges of values that the index 172 /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'. 173 /// This is -2 when undefined, -3 when overdefined, and otherwise the last 174 /// index in the range (inclusive). We use -2 for undefined here because we 175 /// use relative comparisons and don't want 0-1 to match -1. 176 int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined; 177 178 // MagicBitvector - This is a magic bitvector where we set a bit if the 179 // comparison is true for element 'i'. If there are 64 elements or less in 180 // the array, this will fully represent all the comparison results. 181 uint64_t MagicBitvector = 0; 182 183 // Scan the array and see if one of our patterns matches. 184 Constant *CompareRHS = cast<Constant>(ICI.getOperand(1)); 185 for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) { 186 Constant *Elt = Init->getAggregateElement(i); 187 if (!Elt) return nullptr; 188 189 // If this is indexing an array of structures, get the structure element. 190 if (!LaterIndices.empty()) 191 Elt = ConstantExpr::getExtractValue(Elt, LaterIndices); 192 193 // If the element is masked, handle it. 194 if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst); 195 196 // Find out if the comparison would be true or false for the i'th element. 197 Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt, 198 CompareRHS, DL, &TLI); 199 // If the result is undef for this element, ignore it. 200 if (isa<UndefValue>(C)) { 201 // Extend range state machines to cover this element in case there is an 202 // undef in the middle of the range. 203 if (TrueRangeEnd == (int)i-1) 204 TrueRangeEnd = i; 205 if (FalseRangeEnd == (int)i-1) 206 FalseRangeEnd = i; 207 continue; 208 } 209 210 // If we can't compute the result for any of the elements, we have to give 211 // up evaluating the entire conditional. 212 if (!isa<ConstantInt>(C)) return nullptr; 213 214 // Otherwise, we know if the comparison is true or false for this element, 215 // update our state machines. 216 bool IsTrueForElt = !cast<ConstantInt>(C)->isZero(); 217 218 // State machine for single/double/range index comparison. 219 if (IsTrueForElt) { 220 // Update the TrueElement state machine. 221 if (FirstTrueElement == Undefined) 222 FirstTrueElement = TrueRangeEnd = i; // First true element. 223 else { 224 // Update double-compare state machine. 225 if (SecondTrueElement == Undefined) 226 SecondTrueElement = i; 227 else 228 SecondTrueElement = Overdefined; 229 230 // Update range state machine. 231 if (TrueRangeEnd == (int)i-1) 232 TrueRangeEnd = i; 233 else 234 TrueRangeEnd = Overdefined; 235 } 236 } else { 237 // Update the FalseElement state machine. 238 if (FirstFalseElement == Undefined) 239 FirstFalseElement = FalseRangeEnd = i; // First false element. 240 else { 241 // Update double-compare state machine. 242 if (SecondFalseElement == Undefined) 243 SecondFalseElement = i; 244 else 245 SecondFalseElement = Overdefined; 246 247 // Update range state machine. 248 if (FalseRangeEnd == (int)i-1) 249 FalseRangeEnd = i; 250 else 251 FalseRangeEnd = Overdefined; 252 } 253 } 254 255 // If this element is in range, update our magic bitvector. 256 if (i < 64 && IsTrueForElt) 257 MagicBitvector |= 1ULL << i; 258 259 // If all of our states become overdefined, bail out early. Since the 260 // predicate is expensive, only check it every 8 elements. This is only 261 // really useful for really huge arrays. 262 if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined && 263 SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined && 264 FalseRangeEnd == Overdefined) 265 return nullptr; 266 } 267 268 // Now that we've scanned the entire array, emit our new comparison(s). We 269 // order the state machines in complexity of the generated code. 270 Value *Idx = GEP->getOperand(2); 271 272 // If the index is larger than the pointer size of the target, truncate the 273 // index down like the GEP would do implicitly. We don't have to do this for 274 // an inbounds GEP because the index can't be out of range. 275 if (!GEP->isInBounds()) { 276 Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); 277 unsigned PtrSize = IntPtrTy->getIntegerBitWidth(); 278 if (Idx->getType()->getPrimitiveSizeInBits().getFixedSize() > PtrSize) 279 Idx = Builder.CreateTrunc(Idx, IntPtrTy); 280 } 281 282 // If inbounds keyword is not present, Idx * ElementSize can overflow. 283 // Let's assume that ElementSize is 2 and the wanted value is at offset 0. 284 // Then, there are two possible values for Idx to match offset 0: 285 // 0x00..00, 0x80..00. 286 // Emitting 'icmp eq Idx, 0' isn't correct in this case because the 287 // comparison is false if Idx was 0x80..00. 288 // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx. 289 unsigned ElementSize = 290 DL.getTypeAllocSize(Init->getType()->getArrayElementType()); 291 auto MaskIdx = [&](Value* Idx){ 292 if (!GEP->isInBounds() && countTrailingZeros(ElementSize) != 0) { 293 Value *Mask = ConstantInt::get(Idx->getType(), -1); 294 Mask = Builder.CreateLShr(Mask, countTrailingZeros(ElementSize)); 295 Idx = Builder.CreateAnd(Idx, Mask); 296 } 297 return Idx; 298 }; 299 300 // If the comparison is only true for one or two elements, emit direct 301 // comparisons. 302 if (SecondTrueElement != Overdefined) { 303 Idx = MaskIdx(Idx); 304 // None true -> false. 305 if (FirstTrueElement == Undefined) 306 return replaceInstUsesWith(ICI, Builder.getFalse()); 307 308 Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement); 309 310 // True for one element -> 'i == 47'. 311 if (SecondTrueElement == Undefined) 312 return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); 313 314 // True for two elements -> 'i == 47 | i == 72'. 315 Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx); 316 Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement); 317 Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx); 318 return BinaryOperator::CreateOr(C1, C2); 319 } 320 321 // If the comparison is only false for one or two elements, emit direct 322 // comparisons. 323 if (SecondFalseElement != Overdefined) { 324 Idx = MaskIdx(Idx); 325 // None false -> true. 326 if (FirstFalseElement == Undefined) 327 return replaceInstUsesWith(ICI, Builder.getTrue()); 328 329 Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement); 330 331 // False for one element -> 'i != 47'. 332 if (SecondFalseElement == Undefined) 333 return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); 334 335 // False for two elements -> 'i != 47 & i != 72'. 336 Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx); 337 Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement); 338 Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx); 339 return BinaryOperator::CreateAnd(C1, C2); 340 } 341 342 // If the comparison can be replaced with a range comparison for the elements 343 // where it is true, emit the range check. 344 if (TrueRangeEnd != Overdefined) { 345 assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare"); 346 Idx = MaskIdx(Idx); 347 348 // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1). 349 if (FirstTrueElement) { 350 Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement); 351 Idx = Builder.CreateAdd(Idx, Offs); 352 } 353 354 Value *End = ConstantInt::get(Idx->getType(), 355 TrueRangeEnd-FirstTrueElement+1); 356 return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End); 357 } 358 359 // False range check. 360 if (FalseRangeEnd != Overdefined) { 361 assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare"); 362 Idx = MaskIdx(Idx); 363 // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). 364 if (FirstFalseElement) { 365 Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement); 366 Idx = Builder.CreateAdd(Idx, Offs); 367 } 368 369 Value *End = ConstantInt::get(Idx->getType(), 370 FalseRangeEnd-FirstFalseElement); 371 return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End); 372 } 373 374 // If a magic bitvector captures the entire comparison state 375 // of this load, replace it with computation that does: 376 // ((magic_cst >> i) & 1) != 0 377 { 378 Type *Ty = nullptr; 379 380 // Look for an appropriate type: 381 // - The type of Idx if the magic fits 382 // - The smallest fitting legal type 383 if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth()) 384 Ty = Idx->getType(); 385 else 386 Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount); 387 388 if (Ty) { 389 Idx = MaskIdx(Idx); 390 Value *V = Builder.CreateIntCast(Idx, Ty, false); 391 V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); 392 V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V); 393 return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0)); 394 } 395 } 396 397 return nullptr; 398 } 399 400 /// Return a value that can be used to compare the *offset* implied by a GEP to 401 /// zero. For example, if we have &A[i], we want to return 'i' for 402 /// "icmp ne i, 0". Note that, in general, indices can be complex, and scales 403 /// are involved. The above expression would also be legal to codegen as 404 /// "icmp ne (i*4), 0" (assuming A is a pointer to i32). 405 /// This latter form is less amenable to optimization though, and we are allowed 406 /// to generate the first by knowing that pointer arithmetic doesn't overflow. 407 /// 408 /// If we can't emit an optimized form for this expression, this returns null. 409 /// 410 static Value *evaluateGEPOffsetExpression(User *GEP, InstCombinerImpl &IC, 411 const DataLayout &DL) { 412 gep_type_iterator GTI = gep_type_begin(GEP); 413 414 // Check to see if this gep only has a single variable index. If so, and if 415 // any constant indices are a multiple of its scale, then we can compute this 416 // in terms of the scale of the variable index. For example, if the GEP 417 // implies an offset of "12 + i*4", then we can codegen this as "3 + i", 418 // because the expression will cross zero at the same point. 419 unsigned i, e = GEP->getNumOperands(); 420 int64_t Offset = 0; 421 for (i = 1; i != e; ++i, ++GTI) { 422 if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { 423 // Compute the aggregate offset of constant indices. 424 if (CI->isZero()) continue; 425 426 // Handle a struct index, which adds its field offset to the pointer. 427 if (StructType *STy = GTI.getStructTypeOrNull()) { 428 Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); 429 } else { 430 uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); 431 Offset += Size*CI->getSExtValue(); 432 } 433 } else { 434 // Found our variable index. 435 break; 436 } 437 } 438 439 // If there are no variable indices, we must have a constant offset, just 440 // evaluate it the general way. 441 if (i == e) return nullptr; 442 443 Value *VariableIdx = GEP->getOperand(i); 444 // Determine the scale factor of the variable element. For example, this is 445 // 4 if the variable index is into an array of i32. 446 uint64_t VariableScale = DL.getTypeAllocSize(GTI.getIndexedType()); 447 448 // Verify that there are no other variable indices. If so, emit the hard way. 449 for (++i, ++GTI; i != e; ++i, ++GTI) { 450 ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i)); 451 if (!CI) return nullptr; 452 453 // Compute the aggregate offset of constant indices. 454 if (CI->isZero()) continue; 455 456 // Handle a struct index, which adds its field offset to the pointer. 457 if (StructType *STy = GTI.getStructTypeOrNull()) { 458 Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); 459 } else { 460 uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); 461 Offset += Size*CI->getSExtValue(); 462 } 463 } 464 465 // Okay, we know we have a single variable index, which must be a 466 // pointer/array/vector index. If there is no offset, life is simple, return 467 // the index. 468 Type *IntPtrTy = DL.getIntPtrType(GEP->getOperand(0)->getType()); 469 unsigned IntPtrWidth = IntPtrTy->getIntegerBitWidth(); 470 if (Offset == 0) { 471 // Cast to intptrty in case a truncation occurs. If an extension is needed, 472 // we don't need to bother extending: the extension won't affect where the 473 // computation crosses zero. 474 if (VariableIdx->getType()->getPrimitiveSizeInBits().getFixedSize() > 475 IntPtrWidth) { 476 VariableIdx = IC.Builder.CreateTrunc(VariableIdx, IntPtrTy); 477 } 478 return VariableIdx; 479 } 480 481 // Otherwise, there is an index. The computation we will do will be modulo 482 // the pointer size. 483 Offset = SignExtend64(Offset, IntPtrWidth); 484 VariableScale = SignExtend64(VariableScale, IntPtrWidth); 485 486 // To do this transformation, any constant index must be a multiple of the 487 // variable scale factor. For example, we can evaluate "12 + 4*i" as "3 + i", 488 // but we can't evaluate "10 + 3*i" in terms of i. Check that the offset is a 489 // multiple of the variable scale. 490 int64_t NewOffs = Offset / (int64_t)VariableScale; 491 if (Offset != NewOffs*(int64_t)VariableScale) 492 return nullptr; 493 494 // Okay, we can do this evaluation. Start by converting the index to intptr. 495 if (VariableIdx->getType() != IntPtrTy) 496 VariableIdx = IC.Builder.CreateIntCast(VariableIdx, IntPtrTy, 497 true /*Signed*/); 498 Constant *OffsetVal = ConstantInt::get(IntPtrTy, NewOffs); 499 return IC.Builder.CreateAdd(VariableIdx, OffsetVal, "offset"); 500 } 501 502 /// Returns true if we can rewrite Start as a GEP with pointer Base 503 /// and some integer offset. The nodes that need to be re-written 504 /// for this transformation will be added to Explored. 505 static bool canRewriteGEPAsOffset(Value *Start, Value *Base, 506 const DataLayout &DL, 507 SetVector<Value *> &Explored) { 508 SmallVector<Value *, 16> WorkList(1, Start); 509 Explored.insert(Base); 510 511 // The following traversal gives us an order which can be used 512 // when doing the final transformation. Since in the final 513 // transformation we create the PHI replacement instructions first, 514 // we don't have to get them in any particular order. 515 // 516 // However, for other instructions we will have to traverse the 517 // operands of an instruction first, which means that we have to 518 // do a post-order traversal. 519 while (!WorkList.empty()) { 520 SetVector<PHINode *> PHIs; 521 522 while (!WorkList.empty()) { 523 if (Explored.size() >= 100) 524 return false; 525 526 Value *V = WorkList.back(); 527 528 if (Explored.contains(V)) { 529 WorkList.pop_back(); 530 continue; 531 } 532 533 if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) && 534 !isa<GetElementPtrInst>(V) && !isa<PHINode>(V)) 535 // We've found some value that we can't explore which is different from 536 // the base. Therefore we can't do this transformation. 537 return false; 538 539 if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) { 540 auto *CI = cast<CastInst>(V); 541 if (!CI->isNoopCast(DL)) 542 return false; 543 544 if (Explored.count(CI->getOperand(0)) == 0) 545 WorkList.push_back(CI->getOperand(0)); 546 } 547 548 if (auto *GEP = dyn_cast<GEPOperator>(V)) { 549 // We're limiting the GEP to having one index. This will preserve 550 // the original pointer type. We could handle more cases in the 551 // future. 552 if (GEP->getNumIndices() != 1 || !GEP->isInBounds() || 553 GEP->getType() != Start->getType()) 554 return false; 555 556 if (Explored.count(GEP->getOperand(0)) == 0) 557 WorkList.push_back(GEP->getOperand(0)); 558 } 559 560 if (WorkList.back() == V) { 561 WorkList.pop_back(); 562 // We've finished visiting this node, mark it as such. 563 Explored.insert(V); 564 } 565 566 if (auto *PN = dyn_cast<PHINode>(V)) { 567 // We cannot transform PHIs on unsplittable basic blocks. 568 if (isa<CatchSwitchInst>(PN->getParent()->getTerminator())) 569 return false; 570 Explored.insert(PN); 571 PHIs.insert(PN); 572 } 573 } 574 575 // Explore the PHI nodes further. 576 for (auto *PN : PHIs) 577 for (Value *Op : PN->incoming_values()) 578 if (Explored.count(Op) == 0) 579 WorkList.push_back(Op); 580 } 581 582 // Make sure that we can do this. Since we can't insert GEPs in a basic 583 // block before a PHI node, we can't easily do this transformation if 584 // we have PHI node users of transformed instructions. 585 for (Value *Val : Explored) { 586 for (Value *Use : Val->uses()) { 587 588 auto *PHI = dyn_cast<PHINode>(Use); 589 auto *Inst = dyn_cast<Instruction>(Val); 590 591 if (Inst == Base || Inst == PHI || !Inst || !PHI || 592 Explored.count(PHI) == 0) 593 continue; 594 595 if (PHI->getParent() == Inst->getParent()) 596 return false; 597 } 598 } 599 return true; 600 } 601 602 // Sets the appropriate insert point on Builder where we can add 603 // a replacement Instruction for V (if that is possible). 604 static void setInsertionPoint(IRBuilder<> &Builder, Value *V, 605 bool Before = true) { 606 if (auto *PHI = dyn_cast<PHINode>(V)) { 607 Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt()); 608 return; 609 } 610 if (auto *I = dyn_cast<Instruction>(V)) { 611 if (!Before) 612 I = &*std::next(I->getIterator()); 613 Builder.SetInsertPoint(I); 614 return; 615 } 616 if (auto *A = dyn_cast<Argument>(V)) { 617 // Set the insertion point in the entry block. 618 BasicBlock &Entry = A->getParent()->getEntryBlock(); 619 Builder.SetInsertPoint(&*Entry.getFirstInsertionPt()); 620 return; 621 } 622 // Otherwise, this is a constant and we don't need to set a new 623 // insertion point. 624 assert(isa<Constant>(V) && "Setting insertion point for unknown value!"); 625 } 626 627 /// Returns a re-written value of Start as an indexed GEP using Base as a 628 /// pointer. 629 static Value *rewriteGEPAsOffset(Value *Start, Value *Base, 630 const DataLayout &DL, 631 SetVector<Value *> &Explored) { 632 // Perform all the substitutions. This is a bit tricky because we can 633 // have cycles in our use-def chains. 634 // 1. Create the PHI nodes without any incoming values. 635 // 2. Create all the other values. 636 // 3. Add the edges for the PHI nodes. 637 // 4. Emit GEPs to get the original pointers. 638 // 5. Remove the original instructions. 639 Type *IndexType = IntegerType::get( 640 Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType())); 641 642 DenseMap<Value *, Value *> NewInsts; 643 NewInsts[Base] = ConstantInt::getNullValue(IndexType); 644 645 // Create the new PHI nodes, without adding any incoming values. 646 for (Value *Val : Explored) { 647 if (Val == Base) 648 continue; 649 // Create empty phi nodes. This avoids cyclic dependencies when creating 650 // the remaining instructions. 651 if (auto *PHI = dyn_cast<PHINode>(Val)) 652 NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), 653 PHI->getName() + ".idx", PHI); 654 } 655 IRBuilder<> Builder(Base->getContext()); 656 657 // Create all the other instructions. 658 for (Value *Val : Explored) { 659 660 if (NewInsts.find(Val) != NewInsts.end()) 661 continue; 662 663 if (auto *CI = dyn_cast<CastInst>(Val)) { 664 // Don't get rid of the intermediate variable here; the store can grow 665 // the map which will invalidate the reference to the input value. 666 Value *V = NewInsts[CI->getOperand(0)]; 667 NewInsts[CI] = V; 668 continue; 669 } 670 if (auto *GEP = dyn_cast<GEPOperator>(Val)) { 671 Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)] 672 : GEP->getOperand(1); 673 setInsertionPoint(Builder, GEP); 674 // Indices might need to be sign extended. GEPs will magically do 675 // this, but we need to do it ourselves here. 676 if (Index->getType()->getScalarSizeInBits() != 677 NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) { 678 Index = Builder.CreateSExtOrTrunc( 679 Index, NewInsts[GEP->getOperand(0)]->getType(), 680 GEP->getOperand(0)->getName() + ".sext"); 681 } 682 683 auto *Op = NewInsts[GEP->getOperand(0)]; 684 if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero()) 685 NewInsts[GEP] = Index; 686 else 687 NewInsts[GEP] = Builder.CreateNSWAdd( 688 Op, Index, GEP->getOperand(0)->getName() + ".add"); 689 continue; 690 } 691 if (isa<PHINode>(Val)) 692 continue; 693 694 llvm_unreachable("Unexpected instruction type"); 695 } 696 697 // Add the incoming values to the PHI nodes. 698 for (Value *Val : Explored) { 699 if (Val == Base) 700 continue; 701 // All the instructions have been created, we can now add edges to the 702 // phi nodes. 703 if (auto *PHI = dyn_cast<PHINode>(Val)) { 704 PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); 705 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { 706 Value *NewIncoming = PHI->getIncomingValue(I); 707 708 if (NewInsts.find(NewIncoming) != NewInsts.end()) 709 NewIncoming = NewInsts[NewIncoming]; 710 711 NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I)); 712 } 713 } 714 } 715 716 for (Value *Val : Explored) { 717 if (Val == Base) 718 continue; 719 720 // Depending on the type, for external users we have to emit 721 // a GEP or a GEP + ptrtoint. 722 setInsertionPoint(Builder, Val, false); 723 724 // If required, create an inttoptr instruction for Base. 725 Value *NewBase = Base; 726 if (!Base->getType()->isPointerTy()) 727 NewBase = Builder.CreateBitOrPointerCast(Base, Start->getType(), 728 Start->getName() + "to.ptr"); 729 730 Value *GEP = Builder.CreateInBoundsGEP( 731 Start->getType()->getPointerElementType(), NewBase, 732 makeArrayRef(NewInsts[Val]), Val->getName() + ".ptr"); 733 734 if (!Val->getType()->isPointerTy()) { 735 Value *Cast = Builder.CreatePointerCast(GEP, Val->getType(), 736 Val->getName() + ".conv"); 737 GEP = Cast; 738 } 739 Val->replaceAllUsesWith(GEP); 740 } 741 742 return NewInsts[Start]; 743 } 744 745 /// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express 746 /// the input Value as a constant indexed GEP. Returns a pair containing 747 /// the GEPs Pointer and Index. 748 static std::pair<Value *, Value *> 749 getAsConstantIndexedAddress(Value *V, const DataLayout &DL) { 750 Type *IndexType = IntegerType::get(V->getContext(), 751 DL.getIndexTypeSizeInBits(V->getType())); 752 753 Constant *Index = ConstantInt::getNullValue(IndexType); 754 while (true) { 755 if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { 756 // We accept only inbouds GEPs here to exclude the possibility of 757 // overflow. 758 if (!GEP->isInBounds()) 759 break; 760 if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 && 761 GEP->getType() == V->getType()) { 762 V = GEP->getOperand(0); 763 Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1)); 764 Index = ConstantExpr::getAdd( 765 Index, ConstantExpr::getSExtOrBitCast(GEPIndex, IndexType)); 766 continue; 767 } 768 break; 769 } 770 if (auto *CI = dyn_cast<IntToPtrInst>(V)) { 771 if (!CI->isNoopCast(DL)) 772 break; 773 V = CI->getOperand(0); 774 continue; 775 } 776 if (auto *CI = dyn_cast<PtrToIntInst>(V)) { 777 if (!CI->isNoopCast(DL)) 778 break; 779 V = CI->getOperand(0); 780 continue; 781 } 782 break; 783 } 784 return {V, Index}; 785 } 786 787 /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. 788 /// We can look through PHIs, GEPs and casts in order to determine a common base 789 /// between GEPLHS and RHS. 790 static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, 791 ICmpInst::Predicate Cond, 792 const DataLayout &DL) { 793 // FIXME: Support vector of pointers. 794 if (GEPLHS->getType()->isVectorTy()) 795 return nullptr; 796 797 if (!GEPLHS->hasAllConstantIndices()) 798 return nullptr; 799 800 // Make sure the pointers have the same type. 801 if (GEPLHS->getType() != RHS->getType()) 802 return nullptr; 803 804 Value *PtrBase, *Index; 805 std::tie(PtrBase, Index) = getAsConstantIndexedAddress(GEPLHS, DL); 806 807 // The set of nodes that will take part in this transformation. 808 SetVector<Value *> Nodes; 809 810 if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes)) 811 return nullptr; 812 813 // We know we can re-write this as 814 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) 815 // Since we've only looked through inbouds GEPs we know that we 816 // can't have overflow on either side. We can therefore re-write 817 // this as: 818 // OFFSET1 cmp OFFSET2 819 Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes); 820 821 // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written 822 // GEP having PtrBase as the pointer base, and has returned in NewRHS the 823 // offset. Since Index is the offset of LHS to the base pointer, we will now 824 // compare the offsets instead of comparing the pointers. 825 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS); 826 } 827 828 /// Fold comparisons between a GEP instruction and something else. At this point 829 /// we know that the GEP is on the LHS of the comparison. 830 Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, 831 ICmpInst::Predicate Cond, 832 Instruction &I) { 833 // Don't transform signed compares of GEPs into index compares. Even if the 834 // GEP is inbounds, the final add of the base pointer can have signed overflow 835 // and would change the result of the icmp. 836 // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be 837 // the maximum signed value for the pointer type. 838 if (ICmpInst::isSigned(Cond)) 839 return nullptr; 840 841 // Look through bitcasts and addrspacecasts. We do not however want to remove 842 // 0 GEPs. 843 if (!isa<GetElementPtrInst>(RHS)) 844 RHS = RHS->stripPointerCasts(); 845 846 Value *PtrBase = GEPLHS->getOperand(0); 847 // FIXME: Support vector pointer GEPs. 848 if (PtrBase == RHS && GEPLHS->isInBounds() && 849 !GEPLHS->getType()->isVectorTy()) { 850 // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0). 851 // This transformation (ignoring the base and scales) is valid because we 852 // know pointers can't overflow since the gep is inbounds. See if we can 853 // output an optimized form. 854 Value *Offset = evaluateGEPOffsetExpression(GEPLHS, *this, DL); 855 856 // If not, synthesize the offset the hard way. 857 if (!Offset) 858 Offset = EmitGEPOffset(GEPLHS); 859 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset, 860 Constant::getNullValue(Offset->getType())); 861 } 862 863 if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) && 864 isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() && 865 !NullPointerIsDefined(I.getFunction(), 866 RHS->getType()->getPointerAddressSpace())) { 867 // For most address spaces, an allocation can't be placed at null, but null 868 // itself is treated as a 0 size allocation in the in bounds rules. Thus, 869 // the only valid inbounds address derived from null, is null itself. 870 // Thus, we have four cases to consider: 871 // 1) Base == nullptr, Offset == 0 -> inbounds, null 872 // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds 873 // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations) 874 // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison) 875 // 876 // (Note if we're indexing a type of size 0, that simply collapses into one 877 // of the buckets above.) 878 // 879 // In general, we're allowed to make values less poison (i.e. remove 880 // sources of full UB), so in this case, we just select between the two 881 // non-poison cases (1 and 4 above). 882 // 883 // For vectors, we apply the same reasoning on a per-lane basis. 884 auto *Base = GEPLHS->getPointerOperand(); 885 if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) { 886 auto EC = cast<VectorType>(GEPLHS->getType())->getElementCount(); 887 Base = Builder.CreateVectorSplat(EC, Base); 888 } 889 return new ICmpInst(Cond, Base, 890 ConstantExpr::getPointerBitCastOrAddrSpaceCast( 891 cast<Constant>(RHS), Base->getType())); 892 } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) { 893 // If the base pointers are different, but the indices are the same, just 894 // compare the base pointer. 895 if (PtrBase != GEPRHS->getOperand(0)) { 896 bool IndicesTheSame = GEPLHS->getNumOperands()==GEPRHS->getNumOperands(); 897 IndicesTheSame &= GEPLHS->getOperand(0)->getType() == 898 GEPRHS->getOperand(0)->getType(); 899 if (IndicesTheSame) 900 for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i) 901 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { 902 IndicesTheSame = false; 903 break; 904 } 905 906 // If all indices are the same, just compare the base pointers. 907 Type *BaseType = GEPLHS->getOperand(0)->getType(); 908 if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType()) 909 return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0)); 910 911 // If we're comparing GEPs with two base pointers that only differ in type 912 // and both GEPs have only constant indices or just one use, then fold 913 // the compare with the adjusted indices. 914 // FIXME: Support vector of pointers. 915 if (GEPLHS->isInBounds() && GEPRHS->isInBounds() && 916 (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && 917 (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) && 918 PtrBase->stripPointerCasts() == 919 GEPRHS->getOperand(0)->stripPointerCasts() && 920 !GEPLHS->getType()->isVectorTy()) { 921 Value *LOffset = EmitGEPOffset(GEPLHS); 922 Value *ROffset = EmitGEPOffset(GEPRHS); 923 924 // If we looked through an addrspacecast between different sized address 925 // spaces, the LHS and RHS pointers are different sized 926 // integers. Truncate to the smaller one. 927 Type *LHSIndexTy = LOffset->getType(); 928 Type *RHSIndexTy = ROffset->getType(); 929 if (LHSIndexTy != RHSIndexTy) { 930 if (LHSIndexTy->getPrimitiveSizeInBits().getFixedSize() < 931 RHSIndexTy->getPrimitiveSizeInBits().getFixedSize()) { 932 ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy); 933 } else 934 LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy); 935 } 936 937 Value *Cmp = Builder.CreateICmp(ICmpInst::getSignedPredicate(Cond), 938 LOffset, ROffset); 939 return replaceInstUsesWith(I, Cmp); 940 } 941 942 // Otherwise, the base pointers are different and the indices are 943 // different. Try convert this to an indexed compare by looking through 944 // PHIs/casts. 945 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); 946 } 947 948 // If one of the GEPs has all zero indices, recurse. 949 // FIXME: Handle vector of pointers. 950 if (!GEPLHS->getType()->isVectorTy() && GEPLHS->hasAllZeroIndices()) 951 return foldGEPICmp(GEPRHS, GEPLHS->getOperand(0), 952 ICmpInst::getSwappedPredicate(Cond), I); 953 954 // If the other GEP has all zero indices, recurse. 955 // FIXME: Handle vector of pointers. 956 if (!GEPRHS->getType()->isVectorTy() && GEPRHS->hasAllZeroIndices()) 957 return foldGEPICmp(GEPLHS, GEPRHS->getOperand(0), Cond, I); 958 959 bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds(); 960 if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands()) { 961 // If the GEPs only differ by one index, compare it. 962 unsigned NumDifferences = 0; // Keep track of # differences. 963 unsigned DiffOperand = 0; // The operand that differs. 964 for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i) 965 if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { 966 Type *LHSType = GEPLHS->getOperand(i)->getType(); 967 Type *RHSType = GEPRHS->getOperand(i)->getType(); 968 // FIXME: Better support for vector of pointers. 969 if (LHSType->getPrimitiveSizeInBits() != 970 RHSType->getPrimitiveSizeInBits() || 971 (GEPLHS->getType()->isVectorTy() && 972 (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) { 973 // Irreconcilable differences. 974 NumDifferences = 2; 975 break; 976 } 977 978 if (NumDifferences++) break; 979 DiffOperand = i; 980 } 981 982 if (NumDifferences == 0) // SAME GEP? 983 return replaceInstUsesWith(I, // No comparison is needed here. 984 ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond))); 985 986 else if (NumDifferences == 1 && GEPsInBounds) { 987 Value *LHSV = GEPLHS->getOperand(DiffOperand); 988 Value *RHSV = GEPRHS->getOperand(DiffOperand); 989 // Make sure we do a signed comparison here. 990 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV); 991 } 992 } 993 994 // Only lower this if the icmp is the only user of the GEP or if we expect 995 // the result to fold to a constant! 996 if (GEPsInBounds && (isa<ConstantExpr>(GEPLHS) || GEPLHS->hasOneUse()) && 997 (isa<ConstantExpr>(GEPRHS) || GEPRHS->hasOneUse())) { 998 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) 999 Value *L = EmitGEPOffset(GEPLHS); 1000 Value *R = EmitGEPOffset(GEPRHS); 1001 return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R); 1002 } 1003 } 1004 1005 // Try convert this to an indexed compare by looking through PHIs/casts as a 1006 // last resort. 1007 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); 1008 } 1009 1010 Instruction *InstCombinerImpl::foldAllocaCmp(ICmpInst &ICI, 1011 const AllocaInst *Alloca, 1012 const Value *Other) { 1013 assert(ICI.isEquality() && "Cannot fold non-equality comparison."); 1014 1015 // It would be tempting to fold away comparisons between allocas and any 1016 // pointer not based on that alloca (e.g. an argument). However, even 1017 // though such pointers cannot alias, they can still compare equal. 1018 // 1019 // But LLVM doesn't specify where allocas get their memory, so if the alloca 1020 // doesn't escape we can argue that it's impossible to guess its value, and we 1021 // can therefore act as if any such guesses are wrong. 1022 // 1023 // The code below checks that the alloca doesn't escape, and that it's only 1024 // used in a comparison once (the current instruction). The 1025 // single-comparison-use condition ensures that we're trivially folding all 1026 // comparisons against the alloca consistently, and avoids the risk of 1027 // erroneously folding a comparison of the pointer with itself. 1028 1029 unsigned MaxIter = 32; // Break cycles and bound to constant-time. 1030 1031 SmallVector<const Use *, 32> Worklist; 1032 for (const Use &U : Alloca->uses()) { 1033 if (Worklist.size() >= MaxIter) 1034 return nullptr; 1035 Worklist.push_back(&U); 1036 } 1037 1038 unsigned NumCmps = 0; 1039 while (!Worklist.empty()) { 1040 assert(Worklist.size() <= MaxIter); 1041 const Use *U = Worklist.pop_back_val(); 1042 const Value *V = U->getUser(); 1043 --MaxIter; 1044 1045 if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V) || isa<PHINode>(V) || 1046 isa<SelectInst>(V)) { 1047 // Track the uses. 1048 } else if (isa<LoadInst>(V)) { 1049 // Loading from the pointer doesn't escape it. 1050 continue; 1051 } else if (const auto *SI = dyn_cast<StoreInst>(V)) { 1052 // Storing *to* the pointer is fine, but storing the pointer escapes it. 1053 if (SI->getValueOperand() == U->get()) 1054 return nullptr; 1055 continue; 1056 } else if (isa<ICmpInst>(V)) { 1057 if (NumCmps++) 1058 return nullptr; // Found more than one cmp. 1059 continue; 1060 } else if (const auto *Intrin = dyn_cast<IntrinsicInst>(V)) { 1061 switch (Intrin->getIntrinsicID()) { 1062 // These intrinsics don't escape or compare the pointer. Memset is safe 1063 // because we don't allow ptrtoint. Memcpy and memmove are safe because 1064 // we don't allow stores, so src cannot point to V. 1065 case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: 1066 case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: 1067 continue; 1068 default: 1069 return nullptr; 1070 } 1071 } else { 1072 return nullptr; 1073 } 1074 for (const Use &U : V->uses()) { 1075 if (Worklist.size() >= MaxIter) 1076 return nullptr; 1077 Worklist.push_back(&U); 1078 } 1079 } 1080 1081 Type *CmpTy = CmpInst::makeCmpResultType(Other->getType()); 1082 return replaceInstUsesWith( 1083 ICI, 1084 ConstantInt::get(CmpTy, !CmpInst::isTrueWhenEqual(ICI.getPredicate()))); 1085 } 1086 1087 /// Fold "icmp pred (X+C), X". 1088 Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C, 1089 ICmpInst::Predicate Pred) { 1090 // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0, 1091 // so the values can never be equal. Similarly for all other "or equals" 1092 // operators. 1093 assert(!!C && "C should not be zero!"); 1094 1095 // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255 1096 // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253 1097 // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0 1098 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) { 1099 Constant *R = ConstantInt::get(X->getType(), 1100 APInt::getMaxValue(C.getBitWidth()) - C); 1101 return new ICmpInst(ICmpInst::ICMP_UGT, X, R); 1102 } 1103 1104 // (X+1) >u X --> X <u (0-1) --> X != 255 1105 // (X+2) >u X --> X <u (0-2) --> X <u 254 1106 // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0 1107 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) 1108 return new ICmpInst(ICmpInst::ICMP_ULT, X, 1109 ConstantInt::get(X->getType(), -C)); 1110 1111 APInt SMax = APInt::getSignedMaxValue(C.getBitWidth()); 1112 1113 // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127 1114 // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125 1115 // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0 1116 // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1 1117 // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126 1118 // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127 1119 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) 1120 return new ICmpInst(ICmpInst::ICMP_SGT, X, 1121 ConstantInt::get(X->getType(), SMax - C)); 1122 1123 // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127 1124 // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126 1125 // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1 1126 // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2 1127 // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126 1128 // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128 1129 1130 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE); 1131 return new ICmpInst(ICmpInst::ICMP_SLT, X, 1132 ConstantInt::get(X->getType(), SMax - (C - 1))); 1133 } 1134 1135 /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> 1136 /// (icmp eq/ne A, Log2(AP2/AP1)) -> 1137 /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)). 1138 Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A, 1139 const APInt &AP1, 1140 const APInt &AP2) { 1141 assert(I.isEquality() && "Cannot fold icmp gt/lt"); 1142 1143 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { 1144 if (I.getPredicate() == I.ICMP_NE) 1145 Pred = CmpInst::getInversePredicate(Pred); 1146 return new ICmpInst(Pred, LHS, RHS); 1147 }; 1148 1149 // Don't bother doing any work for cases which InstSimplify handles. 1150 if (AP2.isNullValue()) 1151 return nullptr; 1152 1153 bool IsAShr = isa<AShrOperator>(I.getOperand(0)); 1154 if (IsAShr) { 1155 if (AP2.isAllOnesValue()) 1156 return nullptr; 1157 if (AP2.isNegative() != AP1.isNegative()) 1158 return nullptr; 1159 if (AP2.sgt(AP1)) 1160 return nullptr; 1161 } 1162 1163 if (!AP1) 1164 // 'A' must be large enough to shift out the highest set bit. 1165 return getICmp(I.ICMP_UGT, A, 1166 ConstantInt::get(A->getType(), AP2.logBase2())); 1167 1168 if (AP1 == AP2) 1169 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); 1170 1171 int Shift; 1172 if (IsAShr && AP1.isNegative()) 1173 Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes(); 1174 else 1175 Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros(); 1176 1177 if (Shift > 0) { 1178 if (IsAShr && AP1 == AP2.ashr(Shift)) { 1179 // There are multiple solutions if we are comparing against -1 and the LHS 1180 // of the ashr is not a power of two. 1181 if (AP1.isAllOnesValue() && !AP2.isPowerOf2()) 1182 return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift)); 1183 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); 1184 } else if (AP1 == AP2.lshr(Shift)) { 1185 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); 1186 } 1187 } 1188 1189 // Shifting const2 will never be equal to const1. 1190 // FIXME: This should always be handled by InstSimplify? 1191 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); 1192 return replaceInstUsesWith(I, TorF); 1193 } 1194 1195 /// Handle "(icmp eq/ne (shl AP2, A), AP1)" -> 1196 /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)). 1197 Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A, 1198 const APInt &AP1, 1199 const APInt &AP2) { 1200 assert(I.isEquality() && "Cannot fold icmp gt/lt"); 1201 1202 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { 1203 if (I.getPredicate() == I.ICMP_NE) 1204 Pred = CmpInst::getInversePredicate(Pred); 1205 return new ICmpInst(Pred, LHS, RHS); 1206 }; 1207 1208 // Don't bother doing any work for cases which InstSimplify handles. 1209 if (AP2.isNullValue()) 1210 return nullptr; 1211 1212 unsigned AP2TrailingZeros = AP2.countTrailingZeros(); 1213 1214 if (!AP1 && AP2TrailingZeros != 0) 1215 return getICmp( 1216 I.ICMP_UGE, A, 1217 ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros)); 1218 1219 if (AP1 == AP2) 1220 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); 1221 1222 // Get the distance between the lowest bits that are set. 1223 int Shift = AP1.countTrailingZeros() - AP2TrailingZeros; 1224 1225 if (Shift > 0 && AP2.shl(Shift) == AP1) 1226 return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); 1227 1228 // Shifting const2 will never be equal to const1. 1229 // FIXME: This should always be handled by InstSimplify? 1230 auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); 1231 return replaceInstUsesWith(I, TorF); 1232 } 1233 1234 /// The caller has matched a pattern of the form: 1235 /// I = icmp ugt (add (add A, B), CI2), CI1 1236 /// If this is of the form: 1237 /// sum = a + b 1238 /// if (sum+128 >u 255) 1239 /// Then replace it with llvm.sadd.with.overflow.i8. 1240 /// 1241 static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, 1242 ConstantInt *CI2, ConstantInt *CI1, 1243 InstCombinerImpl &IC) { 1244 // The transformation we're trying to do here is to transform this into an 1245 // llvm.sadd.with.overflow. To do this, we have to replace the original add 1246 // with a narrower add, and discard the add-with-constant that is part of the 1247 // range check (if we can't eliminate it, this isn't profitable). 1248 1249 // In order to eliminate the add-with-constant, the compare can be its only 1250 // use. 1251 Instruction *AddWithCst = cast<Instruction>(I.getOperand(0)); 1252 if (!AddWithCst->hasOneUse()) 1253 return nullptr; 1254 1255 // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow. 1256 if (!CI2->getValue().isPowerOf2()) 1257 return nullptr; 1258 unsigned NewWidth = CI2->getValue().countTrailingZeros(); 1259 if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) 1260 return nullptr; 1261 1262 // The width of the new add formed is 1 more than the bias. 1263 ++NewWidth; 1264 1265 // Check to see that CI1 is an all-ones value with NewWidth bits. 1266 if (CI1->getBitWidth() == NewWidth || 1267 CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth)) 1268 return nullptr; 1269 1270 // This is only really a signed overflow check if the inputs have been 1271 // sign-extended; check for that condition. For example, if CI2 is 2^31 and 1272 // the operands of the add are 64 bits wide, we need at least 33 sign bits. 1273 unsigned NeededSignBits = CI1->getBitWidth() - NewWidth + 1; 1274 if (IC.ComputeNumSignBits(A, 0, &I) < NeededSignBits || 1275 IC.ComputeNumSignBits(B, 0, &I) < NeededSignBits) 1276 return nullptr; 1277 1278 // In order to replace the original add with a narrower 1279 // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant 1280 // and truncates that discard the high bits of the add. Verify that this is 1281 // the case. 1282 Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0)); 1283 for (User *U : OrigAdd->users()) { 1284 if (U == AddWithCst) 1285 continue; 1286 1287 // Only accept truncates for now. We would really like a nice recursive 1288 // predicate like SimplifyDemandedBits, but which goes downwards the use-def 1289 // chain to see which bits of a value are actually demanded. If the 1290 // original add had another add which was then immediately truncated, we 1291 // could still do the transformation. 1292 TruncInst *TI = dyn_cast<TruncInst>(U); 1293 if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth) 1294 return nullptr; 1295 } 1296 1297 // If the pattern matches, truncate the inputs to the narrower type and 1298 // use the sadd_with_overflow intrinsic to efficiently compute both the 1299 // result and the overflow bit. 1300 Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth); 1301 Function *F = Intrinsic::getDeclaration( 1302 I.getModule(), Intrinsic::sadd_with_overflow, NewType); 1303 1304 InstCombiner::BuilderTy &Builder = IC.Builder; 1305 1306 // Put the new code above the original add, in case there are any uses of the 1307 // add between the add and the compare. 1308 Builder.SetInsertPoint(OrigAdd); 1309 1310 Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc"); 1311 Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc"); 1312 CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd"); 1313 Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result"); 1314 Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType()); 1315 1316 // The inner add was the result of the narrow add, zero extended to the 1317 // wider type. Replace it with the result computed by the intrinsic. 1318 IC.replaceInstUsesWith(*OrigAdd, ZExt); 1319 IC.eraseInstFromFunction(*OrigAdd); 1320 1321 // The original icmp gets replaced with the overflow value. 1322 return ExtractValueInst::Create(Call, 1, "sadd.overflow"); 1323 } 1324 1325 /// If we have: 1326 /// icmp eq/ne (urem/srem %x, %y), 0 1327 /// iff %y is a power-of-two, we can replace this with a bit test: 1328 /// icmp eq/ne (and %x, (add %y, -1)), 0 1329 Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) { 1330 // This fold is only valid for equality predicates. 1331 if (!I.isEquality()) 1332 return nullptr; 1333 ICmpInst::Predicate Pred; 1334 Value *X, *Y, *Zero; 1335 if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))), 1336 m_CombineAnd(m_Zero(), m_Value(Zero))))) 1337 return nullptr; 1338 if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I)) 1339 return nullptr; 1340 // This may increase instruction count, we don't enforce that Y is a constant. 1341 Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType())); 1342 Value *Masked = Builder.CreateAnd(X, Mask); 1343 return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero); 1344 } 1345 1346 /// Fold equality-comparison between zero and any (maybe truncated) right-shift 1347 /// by one-less-than-bitwidth into a sign test on the original value. 1348 Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) { 1349 Instruction *Val; 1350 ICmpInst::Predicate Pred; 1351 if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero()))) 1352 return nullptr; 1353 1354 Value *X; 1355 Type *XTy; 1356 1357 Constant *C; 1358 if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) { 1359 XTy = X->getType(); 1360 unsigned XBitWidth = XTy->getScalarSizeInBits(); 1361 if (!match(C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ, 1362 APInt(XBitWidth, XBitWidth - 1)))) 1363 return nullptr; 1364 } else if (isa<BinaryOperator>(Val) && 1365 (X = reassociateShiftAmtsOfTwoSameDirectionShifts( 1366 cast<BinaryOperator>(Val), SQ.getWithInstruction(Val), 1367 /*AnalyzeForSignBitExtraction=*/true))) { 1368 XTy = X->getType(); 1369 } else 1370 return nullptr; 1371 1372 return ICmpInst::Create(Instruction::ICmp, 1373 Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE 1374 : ICmpInst::ICMP_SLT, 1375 X, ConstantInt::getNullValue(XTy)); 1376 } 1377 1378 // Handle icmp pred X, 0 1379 Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) { 1380 CmpInst::Predicate Pred = Cmp.getPredicate(); 1381 if (!match(Cmp.getOperand(1), m_Zero())) 1382 return nullptr; 1383 1384 // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0) 1385 if (Pred == ICmpInst::ICMP_SGT) { 1386 Value *A, *B; 1387 SelectPatternResult SPR = matchSelectPattern(Cmp.getOperand(0), A, B); 1388 if (SPR.Flavor == SPF_SMIN) { 1389 if (isKnownPositive(A, DL, 0, &AC, &Cmp, &DT)) 1390 return new ICmpInst(Pred, B, Cmp.getOperand(1)); 1391 if (isKnownPositive(B, DL, 0, &AC, &Cmp, &DT)) 1392 return new ICmpInst(Pred, A, Cmp.getOperand(1)); 1393 } 1394 } 1395 1396 if (Instruction *New = foldIRemByPowerOfTwoToBitTest(Cmp)) 1397 return New; 1398 1399 // Given: 1400 // icmp eq/ne (urem %x, %y), 0 1401 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem': 1402 // icmp eq/ne %x, 0 1403 Value *X, *Y; 1404 if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) && 1405 ICmpInst::isEquality(Pred)) { 1406 KnownBits XKnown = computeKnownBits(X, 0, &Cmp); 1407 KnownBits YKnown = computeKnownBits(Y, 0, &Cmp); 1408 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2) 1409 return new ICmpInst(Pred, X, Cmp.getOperand(1)); 1410 } 1411 1412 return nullptr; 1413 } 1414 1415 /// Fold icmp Pred X, C. 1416 /// TODO: This code structure does not make sense. The saturating add fold 1417 /// should be moved to some other helper and extended as noted below (it is also 1418 /// possible that code has been made unnecessary - do we canonicalize IR to 1419 /// overflow/saturating intrinsics or not?). 1420 Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) { 1421 // Match the following pattern, which is a common idiom when writing 1422 // overflow-safe integer arithmetic functions. The source performs an addition 1423 // in wider type and explicitly checks for overflow using comparisons against 1424 // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic. 1425 // 1426 // TODO: This could probably be generalized to handle other overflow-safe 1427 // operations if we worked out the formulas to compute the appropriate magic 1428 // constants. 1429 // 1430 // sum = a + b 1431 // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8 1432 CmpInst::Predicate Pred = Cmp.getPredicate(); 1433 Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1); 1434 Value *A, *B; 1435 ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI 1436 if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) && 1437 match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2)))) 1438 if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this)) 1439 return Res; 1440 1441 // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...). 1442 Constant *C = dyn_cast<Constant>(Op1); 1443 if (!C || C->canTrap()) 1444 return nullptr; 1445 1446 if (auto *Phi = dyn_cast<PHINode>(Op0)) 1447 if (all_of(Phi->operands(), [](Value *V) { return isa<Constant>(V); })) { 1448 Type *Ty = Cmp.getType(); 1449 Builder.SetInsertPoint(Phi); 1450 PHINode *NewPhi = 1451 Builder.CreatePHI(Ty, Phi->getNumOperands()); 1452 for (BasicBlock *Predecessor : predecessors(Phi->getParent())) { 1453 auto *Input = 1454 cast<Constant>(Phi->getIncomingValueForBlock(Predecessor)); 1455 auto *BoolInput = ConstantExpr::getCompare(Pred, Input, C); 1456 NewPhi->addIncoming(BoolInput, Predecessor); 1457 } 1458 NewPhi->takeName(&Cmp); 1459 return replaceInstUsesWith(Cmp, NewPhi); 1460 } 1461 1462 return nullptr; 1463 } 1464 1465 /// Canonicalize icmp instructions based on dominating conditions. 1466 Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) { 1467 // This is a cheap/incomplete check for dominance - just match a single 1468 // predecessor with a conditional branch. 1469 BasicBlock *CmpBB = Cmp.getParent(); 1470 BasicBlock *DomBB = CmpBB->getSinglePredecessor(); 1471 if (!DomBB) 1472 return nullptr; 1473 1474 Value *DomCond; 1475 BasicBlock *TrueBB, *FalseBB; 1476 if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB))) 1477 return nullptr; 1478 1479 assert((TrueBB == CmpBB || FalseBB == CmpBB) && 1480 "Predecessor block does not point to successor?"); 1481 1482 // The branch should get simplified. Don't bother simplifying this condition. 1483 if (TrueBB == FalseBB) 1484 return nullptr; 1485 1486 // Try to simplify this compare to T/F based on the dominating condition. 1487 Optional<bool> Imp = isImpliedCondition(DomCond, &Cmp, DL, TrueBB == CmpBB); 1488 if (Imp) 1489 return replaceInstUsesWith(Cmp, ConstantInt::get(Cmp.getType(), *Imp)); 1490 1491 CmpInst::Predicate Pred = Cmp.getPredicate(); 1492 Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1); 1493 ICmpInst::Predicate DomPred; 1494 const APInt *C, *DomC; 1495 if (match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))) && 1496 match(Y, m_APInt(C))) { 1497 // We have 2 compares of a variable with constants. Calculate the constant 1498 // ranges of those compares to see if we can transform the 2nd compare: 1499 // DomBB: 1500 // DomCond = icmp DomPred X, DomC 1501 // br DomCond, CmpBB, FalseBB 1502 // CmpBB: 1503 // Cmp = icmp Pred X, C 1504 ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *C); 1505 ConstantRange DominatingCR = 1506 (CmpBB == TrueBB) ? ConstantRange::makeExactICmpRegion(DomPred, *DomC) 1507 : ConstantRange::makeExactICmpRegion( 1508 CmpInst::getInversePredicate(DomPred), *DomC); 1509 ConstantRange Intersection = DominatingCR.intersectWith(CR); 1510 ConstantRange Difference = DominatingCR.difference(CR); 1511 if (Intersection.isEmptySet()) 1512 return replaceInstUsesWith(Cmp, Builder.getFalse()); 1513 if (Difference.isEmptySet()) 1514 return replaceInstUsesWith(Cmp, Builder.getTrue()); 1515 1516 // Canonicalizing a sign bit comparison that gets used in a branch, 1517 // pessimizes codegen by generating branch on zero instruction instead 1518 // of a test and branch. So we avoid canonicalizing in such situations 1519 // because test and branch instruction has better branch displacement 1520 // than compare and branch instruction. 1521 bool UnusedBit; 1522 bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit); 1523 if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp))) 1524 return nullptr; 1525 1526 // Avoid an infinite loop with min/max canonicalization. 1527 // TODO: This will be unnecessary if we canonicalize to min/max intrinsics. 1528 if (Cmp.hasOneUse() && 1529 match(Cmp.user_back(), m_MaxOrMin(m_Value(), m_Value()))) 1530 return nullptr; 1531 1532 if (const APInt *EqC = Intersection.getSingleElement()) 1533 return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC)); 1534 if (const APInt *NeC = Difference.getSingleElement()) 1535 return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC)); 1536 } 1537 1538 return nullptr; 1539 } 1540 1541 /// Fold icmp (trunc X, Y), C. 1542 Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp, 1543 TruncInst *Trunc, 1544 const APInt &C) { 1545 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1546 Value *X = Trunc->getOperand(0); 1547 if (C.isOneValue() && C.getBitWidth() > 1) { 1548 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 1549 Value *V = nullptr; 1550 if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V)))) 1551 return new ICmpInst(ICmpInst::ICMP_SLT, V, 1552 ConstantInt::get(V->getType(), 1)); 1553 } 1554 1555 unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), 1556 SrcBits = X->getType()->getScalarSizeInBits(); 1557 if (Cmp.isEquality() && Trunc->hasOneUse()) { 1558 // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all 1559 // of the high bits truncated out of x are known. 1560 KnownBits Known = computeKnownBits(X, 0, &Cmp); 1561 1562 // If all the high bits are known, we can do this xform. 1563 if ((Known.Zero | Known.One).countLeadingOnes() >= SrcBits - DstBits) { 1564 // Pull in the high bits from known-ones set. 1565 APInt NewRHS = C.zext(SrcBits); 1566 NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits); 1567 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), NewRHS)); 1568 } 1569 } 1570 1571 // Look through truncated right-shift of the sign-bit for a sign-bit check: 1572 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0 1573 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1 1574 Value *ShOp; 1575 const APInt *ShAmtC; 1576 bool TrueIfSigned; 1577 if (isSignBitCheck(Pred, C, TrueIfSigned) && 1578 match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) && 1579 DstBits == SrcBits - ShAmtC->getZExtValue()) { 1580 return TrueIfSigned 1581 ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp, 1582 ConstantInt::getNullValue(X->getType())) 1583 : new ICmpInst(ICmpInst::ICMP_SGT, ShOp, 1584 ConstantInt::getAllOnesValue(X->getType())); 1585 } 1586 1587 return nullptr; 1588 } 1589 1590 /// Fold icmp (xor X, Y), C. 1591 Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp, 1592 BinaryOperator *Xor, 1593 const APInt &C) { 1594 Value *X = Xor->getOperand(0); 1595 Value *Y = Xor->getOperand(1); 1596 const APInt *XorC; 1597 if (!match(Y, m_APInt(XorC))) 1598 return nullptr; 1599 1600 // If this is a comparison that tests the signbit (X < 0) or (x > -1), 1601 // fold the xor. 1602 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1603 bool TrueIfSigned = false; 1604 if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) { 1605 1606 // If the sign bit of the XorCst is not set, there is no change to 1607 // the operation, just stop using the Xor. 1608 if (!XorC->isNegative()) 1609 return replaceOperand(Cmp, 0, X); 1610 1611 // Emit the opposite comparison. 1612 if (TrueIfSigned) 1613 return new ICmpInst(ICmpInst::ICMP_SGT, X, 1614 ConstantInt::getAllOnesValue(X->getType())); 1615 else 1616 return new ICmpInst(ICmpInst::ICMP_SLT, X, 1617 ConstantInt::getNullValue(X->getType())); 1618 } 1619 1620 if (Xor->hasOneUse()) { 1621 // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask)) 1622 if (!Cmp.isEquality() && XorC->isSignMask()) { 1623 Pred = Cmp.getFlippedSignednessPredicate(); 1624 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); 1625 } 1626 1627 // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask)) 1628 if (!Cmp.isEquality() && XorC->isMaxSignedValue()) { 1629 Pred = Cmp.getFlippedSignednessPredicate(); 1630 Pred = Cmp.getSwappedPredicate(Pred); 1631 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); 1632 } 1633 } 1634 1635 // Mask constant magic can eliminate an 'xor' with unsigned compares. 1636 if (Pred == ICmpInst::ICMP_UGT) { 1637 // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2) 1638 if (*XorC == ~C && (C + 1).isPowerOf2()) 1639 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); 1640 // (xor X, C) >u C --> X >u C (when C+1 is a power of 2) 1641 if (*XorC == C && (C + 1).isPowerOf2()) 1642 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); 1643 } 1644 if (Pred == ICmpInst::ICMP_ULT) { 1645 // (xor X, -C) <u C --> X >u ~C (when C is a power of 2) 1646 if (*XorC == -C && C.isPowerOf2()) 1647 return new ICmpInst(ICmpInst::ICMP_UGT, X, 1648 ConstantInt::get(X->getType(), ~C)); 1649 // (xor X, C) <u C --> X >u ~C (when -C is a power of 2) 1650 if (*XorC == C && (-C).isPowerOf2()) 1651 return new ICmpInst(ICmpInst::ICMP_UGT, X, 1652 ConstantInt::get(X->getType(), ~C)); 1653 } 1654 return nullptr; 1655 } 1656 1657 /// Fold icmp (and (sh X, Y), C2), C1. 1658 Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp, 1659 BinaryOperator *And, 1660 const APInt &C1, 1661 const APInt &C2) { 1662 BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0)); 1663 if (!Shift || !Shift->isShift()) 1664 return nullptr; 1665 1666 // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could 1667 // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in 1668 // code produced by the clang front-end, for bitfield access. 1669 // This seemingly simple opportunity to fold away a shift turns out to be 1670 // rather complicated. See PR17827 for details. 1671 unsigned ShiftOpcode = Shift->getOpcode(); 1672 bool IsShl = ShiftOpcode == Instruction::Shl; 1673 const APInt *C3; 1674 if (match(Shift->getOperand(1), m_APInt(C3))) { 1675 APInt NewAndCst, NewCmpCst; 1676 bool AnyCmpCstBitsShiftedOut; 1677 if (ShiftOpcode == Instruction::Shl) { 1678 // For a left shift, we can fold if the comparison is not signed. We can 1679 // also fold a signed comparison if the mask value and comparison value 1680 // are not negative. These constraints may not be obvious, but we can 1681 // prove that they are correct using an SMT solver. 1682 if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative())) 1683 return nullptr; 1684 1685 NewCmpCst = C1.lshr(*C3); 1686 NewAndCst = C2.lshr(*C3); 1687 AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1; 1688 } else if (ShiftOpcode == Instruction::LShr) { 1689 // For a logical right shift, we can fold if the comparison is not signed. 1690 // We can also fold a signed comparison if the shifted mask value and the 1691 // shifted comparison value are not negative. These constraints may not be 1692 // obvious, but we can prove that they are correct using an SMT solver. 1693 NewCmpCst = C1.shl(*C3); 1694 NewAndCst = C2.shl(*C3); 1695 AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1; 1696 if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative())) 1697 return nullptr; 1698 } else { 1699 // For an arithmetic shift, check that both constants don't use (in a 1700 // signed sense) the top bits being shifted out. 1701 assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode"); 1702 NewCmpCst = C1.shl(*C3); 1703 NewAndCst = C2.shl(*C3); 1704 AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1; 1705 if (NewAndCst.ashr(*C3) != C2) 1706 return nullptr; 1707 } 1708 1709 if (AnyCmpCstBitsShiftedOut) { 1710 // If we shifted bits out, the fold is not going to work out. As a 1711 // special case, check to see if this means that the result is always 1712 // true or false now. 1713 if (Cmp.getPredicate() == ICmpInst::ICMP_EQ) 1714 return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType())); 1715 if (Cmp.getPredicate() == ICmpInst::ICMP_NE) 1716 return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType())); 1717 } else { 1718 Value *NewAnd = Builder.CreateAnd( 1719 Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst)); 1720 return new ICmpInst(Cmp.getPredicate(), 1721 NewAnd, ConstantInt::get(And->getType(), NewCmpCst)); 1722 } 1723 } 1724 1725 // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is 1726 // preferable because it allows the C2 << Y expression to be hoisted out of a 1727 // loop if Y is invariant and X is not. 1728 if (Shift->hasOneUse() && C1.isNullValue() && Cmp.isEquality() && 1729 !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) { 1730 // Compute C2 << Y. 1731 Value *NewShift = 1732 IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1)) 1733 : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1)); 1734 1735 // Compute X & (C2 << Y). 1736 Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift); 1737 return replaceOperand(Cmp, 0, NewAnd); 1738 } 1739 1740 return nullptr; 1741 } 1742 1743 /// Fold icmp (and X, C2), C1. 1744 Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp, 1745 BinaryOperator *And, 1746 const APInt &C1) { 1747 bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE; 1748 1749 // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1 1750 // TODO: We canonicalize to the longer form for scalars because we have 1751 // better analysis/folds for icmp, and codegen may be better with icmp. 1752 if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isNullValue() && 1753 match(And->getOperand(1), m_One())) 1754 return new TruncInst(And->getOperand(0), Cmp.getType()); 1755 1756 const APInt *C2; 1757 Value *X; 1758 if (!match(And, m_And(m_Value(X), m_APInt(C2)))) 1759 return nullptr; 1760 1761 // Don't perform the following transforms if the AND has multiple uses 1762 if (!And->hasOneUse()) 1763 return nullptr; 1764 1765 if (Cmp.isEquality() && C1.isNullValue()) { 1766 // Restrict this fold to single-use 'and' (PR10267). 1767 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0 1768 if (C2->isSignMask()) { 1769 Constant *Zero = Constant::getNullValue(X->getType()); 1770 auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; 1771 return new ICmpInst(NewPred, X, Zero); 1772 } 1773 1774 // Restrict this fold only for single-use 'and' (PR10267). 1775 // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two. 1776 if ((~(*C2) + 1).isPowerOf2()) { 1777 Constant *NegBOC = 1778 ConstantExpr::getNeg(cast<Constant>(And->getOperand(1))); 1779 auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; 1780 return new ICmpInst(NewPred, X, NegBOC); 1781 } 1782 } 1783 1784 // If the LHS is an 'and' of a truncate and we can widen the and/compare to 1785 // the input width without changing the value produced, eliminate the cast: 1786 // 1787 // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1' 1788 // 1789 // We can do this transformation if the constants do not have their sign bits 1790 // set or if it is an equality comparison. Extending a relational comparison 1791 // when we're checking the sign bit would not work. 1792 Value *W; 1793 if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) && 1794 (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) { 1795 // TODO: Is this a good transform for vectors? Wider types may reduce 1796 // throughput. Should this transform be limited (even for scalars) by using 1797 // shouldChangeType()? 1798 if (!Cmp.getType()->isVectorTy()) { 1799 Type *WideType = W->getType(); 1800 unsigned WideScalarBits = WideType->getScalarSizeInBits(); 1801 Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits)); 1802 Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits)); 1803 Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName()); 1804 return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); 1805 } 1806 } 1807 1808 if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2)) 1809 return I; 1810 1811 // (icmp pred (and (or (lshr A, B), A), 1), 0) --> 1812 // (icmp pred (and A, (or (shl 1, B), 1), 0)) 1813 // 1814 // iff pred isn't signed 1815 if (!Cmp.isSigned() && C1.isNullValue() && And->getOperand(0)->hasOneUse() && 1816 match(And->getOperand(1), m_One())) { 1817 Constant *One = cast<Constant>(And->getOperand(1)); 1818 Value *Or = And->getOperand(0); 1819 Value *A, *B, *LShr; 1820 if (match(Or, m_Or(m_Value(LShr), m_Value(A))) && 1821 match(LShr, m_LShr(m_Specific(A), m_Value(B)))) { 1822 unsigned UsesRemoved = 0; 1823 if (And->hasOneUse()) 1824 ++UsesRemoved; 1825 if (Or->hasOneUse()) 1826 ++UsesRemoved; 1827 if (LShr->hasOneUse()) 1828 ++UsesRemoved; 1829 1830 // Compute A & ((1 << B) | 1) 1831 Value *NewOr = nullptr; 1832 if (auto *C = dyn_cast<Constant>(B)) { 1833 if (UsesRemoved >= 1) 1834 NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One); 1835 } else { 1836 if (UsesRemoved >= 3) 1837 NewOr = Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(), 1838 /*HasNUW=*/true), 1839 One, Or->getName()); 1840 } 1841 if (NewOr) { 1842 Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName()); 1843 return replaceOperand(Cmp, 0, NewAnd); 1844 } 1845 } 1846 } 1847 1848 return nullptr; 1849 } 1850 1851 /// Fold icmp (and X, Y), C. 1852 Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp, 1853 BinaryOperator *And, 1854 const APInt &C) { 1855 if (Instruction *I = foldICmpAndConstConst(Cmp, And, C)) 1856 return I; 1857 1858 const ICmpInst::Predicate Pred = Cmp.getPredicate(); 1859 bool TrueIfNeg; 1860 if (isSignBitCheck(Pred, C, TrueIfNeg)) { 1861 // ((X - 1) & ~X) < 0 --> X == 0 1862 // ((X - 1) & ~X) >= 0 --> X != 0 1863 Value *X; 1864 if (match(And->getOperand(0), m_Add(m_Value(X), m_AllOnes())) && 1865 match(And->getOperand(1), m_Not(m_Specific(X)))) { 1866 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE; 1867 return new ICmpInst(NewPred, X, ConstantInt::getNullValue(X->getType())); 1868 } 1869 } 1870 1871 // TODO: These all require that Y is constant too, so refactor with the above. 1872 1873 // Try to optimize things like "A[i] & 42 == 0" to index computations. 1874 Value *X = And->getOperand(0); 1875 Value *Y = And->getOperand(1); 1876 if (auto *LI = dyn_cast<LoadInst>(X)) 1877 if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) 1878 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) 1879 if (GV->isConstant() && GV->hasDefinitiveInitializer() && 1880 !LI->isVolatile() && isa<ConstantInt>(Y)) { 1881 ConstantInt *C2 = cast<ConstantInt>(Y); 1882 if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, Cmp, C2)) 1883 return Res; 1884 } 1885 1886 if (!Cmp.isEquality()) 1887 return nullptr; 1888 1889 // X & -C == -C -> X > u ~C 1890 // X & -C != -C -> X <= u ~C 1891 // iff C is a power of 2 1892 if (Cmp.getOperand(1) == Y && (-C).isPowerOf2()) { 1893 auto NewPred = 1894 Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE; 1895 return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1)))); 1896 } 1897 1898 // (X & C2) == 0 -> (trunc X) >= 0 1899 // (X & C2) != 0 -> (trunc X) < 0 1900 // iff C2 is a power of 2 and it masks the sign bit of a legal integer type. 1901 const APInt *C2; 1902 if (And->hasOneUse() && C.isNullValue() && match(Y, m_APInt(C2))) { 1903 int32_t ExactLogBase2 = C2->exactLogBase2(); 1904 if (ExactLogBase2 != -1 && DL.isLegalInteger(ExactLogBase2 + 1)) { 1905 Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1); 1906 if (auto *AndVTy = dyn_cast<VectorType>(And->getType())) 1907 NTy = VectorType::get(NTy, AndVTy->getElementCount()); 1908 Value *Trunc = Builder.CreateTrunc(X, NTy); 1909 auto NewPred = 1910 Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE : CmpInst::ICMP_SLT; 1911 return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy)); 1912 } 1913 } 1914 1915 return nullptr; 1916 } 1917 1918 /// Fold icmp (or X, Y), C. 1919 Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp, 1920 BinaryOperator *Or, 1921 const APInt &C) { 1922 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1923 if (C.isOneValue()) { 1924 // icmp slt signum(V) 1 --> icmp slt V, 1 1925 Value *V = nullptr; 1926 if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V)))) 1927 return new ICmpInst(ICmpInst::ICMP_SLT, V, 1928 ConstantInt::get(V->getType(), 1)); 1929 } 1930 1931 Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1); 1932 const APInt *MaskC; 1933 if (match(OrOp1, m_APInt(MaskC)) && Cmp.isEquality()) { 1934 if (*MaskC == C && (C + 1).isPowerOf2()) { 1935 // X | C == C --> X <=u C 1936 // X | C != C --> X >u C 1937 // iff C+1 is a power of 2 (C is a bitmask of the low bits) 1938 Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT; 1939 return new ICmpInst(Pred, OrOp0, OrOp1); 1940 } 1941 1942 // More general: canonicalize 'equality with set bits mask' to 1943 // 'equality with clear bits mask'. 1944 // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC 1945 // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC 1946 if (Or->hasOneUse()) { 1947 Value *And = Builder.CreateAnd(OrOp0, ~(*MaskC)); 1948 Constant *NewC = ConstantInt::get(Or->getType(), C ^ (*MaskC)); 1949 return new ICmpInst(Pred, And, NewC); 1950 } 1951 } 1952 1953 if (!Cmp.isEquality() || !C.isNullValue() || !Or->hasOneUse()) 1954 return nullptr; 1955 1956 Value *P, *Q; 1957 if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) { 1958 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 1959 // -> and (icmp eq P, null), (icmp eq Q, null). 1960 Value *CmpP = 1961 Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); 1962 Value *CmpQ = 1963 Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); 1964 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; 1965 return BinaryOperator::Create(BOpc, CmpP, CmpQ); 1966 } 1967 1968 // Are we using xors to bitwise check for a pair of (in)equalities? Convert to 1969 // a shorter form that has more potential to be folded even further. 1970 Value *X1, *X2, *X3, *X4; 1971 if (match(OrOp0, m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) && 1972 match(OrOp1, m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) { 1973 // ((X1 ^ X2) || (X3 ^ X4)) == 0 --> (X1 == X2) && (X3 == X4) 1974 // ((X1 ^ X2) || (X3 ^ X4)) != 0 --> (X1 != X2) || (X3 != X4) 1975 Value *Cmp12 = Builder.CreateICmp(Pred, X1, X2); 1976 Value *Cmp34 = Builder.CreateICmp(Pred, X3, X4); 1977 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; 1978 return BinaryOperator::Create(BOpc, Cmp12, Cmp34); 1979 } 1980 1981 return nullptr; 1982 } 1983 1984 /// Fold icmp (mul X, Y), C. 1985 Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp, 1986 BinaryOperator *Mul, 1987 const APInt &C) { 1988 const APInt *MulC; 1989 if (!match(Mul->getOperand(1), m_APInt(MulC))) 1990 return nullptr; 1991 1992 // If this is a test of the sign bit and the multiply is sign-preserving with 1993 // a constant operand, use the multiply LHS operand instead. 1994 ICmpInst::Predicate Pred = Cmp.getPredicate(); 1995 if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) { 1996 if (MulC->isNegative()) 1997 Pred = ICmpInst::getSwappedPredicate(Pred); 1998 return new ICmpInst(Pred, Mul->getOperand(0), 1999 Constant::getNullValue(Mul->getType())); 2000 } 2001 2002 // If the multiply does not wrap, try to divide the compare constant by the 2003 // multiplication factor. 2004 if (Cmp.isEquality() && !MulC->isNullValue()) { 2005 // (mul nsw X, MulC) == C --> X == C /s MulC 2006 if (Mul->hasNoSignedWrap() && C.srem(*MulC).isNullValue()) { 2007 Constant *NewC = ConstantInt::get(Mul->getType(), C.sdiv(*MulC)); 2008 return new ICmpInst(Pred, Mul->getOperand(0), NewC); 2009 } 2010 // (mul nuw X, MulC) == C --> X == C /u MulC 2011 if (Mul->hasNoUnsignedWrap() && C.urem(*MulC).isNullValue()) { 2012 Constant *NewC = ConstantInt::get(Mul->getType(), C.udiv(*MulC)); 2013 return new ICmpInst(Pred, Mul->getOperand(0), NewC); 2014 } 2015 } 2016 2017 return nullptr; 2018 } 2019 2020 /// Fold icmp (shl 1, Y), C. 2021 static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, 2022 const APInt &C) { 2023 Value *Y; 2024 if (!match(Shl, m_Shl(m_One(), m_Value(Y)))) 2025 return nullptr; 2026 2027 Type *ShiftType = Shl->getType(); 2028 unsigned TypeBits = C.getBitWidth(); 2029 bool CIsPowerOf2 = C.isPowerOf2(); 2030 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2031 if (Cmp.isUnsigned()) { 2032 // (1 << Y) pred C -> Y pred Log2(C) 2033 if (!CIsPowerOf2) { 2034 // (1 << Y) < 30 -> Y <= 4 2035 // (1 << Y) <= 30 -> Y <= 4 2036 // (1 << Y) >= 30 -> Y > 4 2037 // (1 << Y) > 30 -> Y > 4 2038 if (Pred == ICmpInst::ICMP_ULT) 2039 Pred = ICmpInst::ICMP_ULE; 2040 else if (Pred == ICmpInst::ICMP_UGE) 2041 Pred = ICmpInst::ICMP_UGT; 2042 } 2043 2044 // (1 << Y) >= 2147483648 -> Y >= 31 -> Y == 31 2045 // (1 << Y) < 2147483648 -> Y < 31 -> Y != 31 2046 unsigned CLog2 = C.logBase2(); 2047 if (CLog2 == TypeBits - 1) { 2048 if (Pred == ICmpInst::ICMP_UGE) 2049 Pred = ICmpInst::ICMP_EQ; 2050 else if (Pred == ICmpInst::ICMP_ULT) 2051 Pred = ICmpInst::ICMP_NE; 2052 } 2053 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2)); 2054 } else if (Cmp.isSigned()) { 2055 Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1); 2056 if (C.isAllOnesValue()) { 2057 // (1 << Y) <= -1 -> Y == 31 2058 if (Pred == ICmpInst::ICMP_SLE) 2059 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); 2060 2061 // (1 << Y) > -1 -> Y != 31 2062 if (Pred == ICmpInst::ICMP_SGT) 2063 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); 2064 } else if (!C) { 2065 // (1 << Y) < 0 -> Y == 31 2066 // (1 << Y) <= 0 -> Y == 31 2067 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) 2068 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); 2069 2070 // (1 << Y) >= 0 -> Y != 31 2071 // (1 << Y) > 0 -> Y != 31 2072 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) 2073 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); 2074 } 2075 } else if (Cmp.isEquality() && CIsPowerOf2) { 2076 return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, C.logBase2())); 2077 } 2078 2079 return nullptr; 2080 } 2081 2082 /// Fold icmp (shl X, Y), C. 2083 Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp, 2084 BinaryOperator *Shl, 2085 const APInt &C) { 2086 const APInt *ShiftVal; 2087 if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal))) 2088 return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal); 2089 2090 const APInt *ShiftAmt; 2091 if (!match(Shl->getOperand(1), m_APInt(ShiftAmt))) 2092 return foldICmpShlOne(Cmp, Shl, C); 2093 2094 // Check that the shift amount is in range. If not, don't perform undefined 2095 // shifts. When the shift is visited, it will be simplified. 2096 unsigned TypeBits = C.getBitWidth(); 2097 if (ShiftAmt->uge(TypeBits)) 2098 return nullptr; 2099 2100 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2101 Value *X = Shl->getOperand(0); 2102 Type *ShType = Shl->getType(); 2103 2104 // NSW guarantees that we are only shifting out sign bits from the high bits, 2105 // so we can ASHR the compare constant without needing a mask and eliminate 2106 // the shift. 2107 if (Shl->hasNoSignedWrap()) { 2108 if (Pred == ICmpInst::ICMP_SGT) { 2109 // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt) 2110 APInt ShiftedC = C.ashr(*ShiftAmt); 2111 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2112 } 2113 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && 2114 C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) { 2115 APInt ShiftedC = C.ashr(*ShiftAmt); 2116 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2117 } 2118 if (Pred == ICmpInst::ICMP_SLT) { 2119 // SLE is the same as above, but SLE is canonicalized to SLT, so convert: 2120 // (X << S) <=s C is equiv to X <=s (C >> S) for all C 2121 // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX 2122 // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN 2123 assert(!C.isMinSignedValue() && "Unexpected icmp slt"); 2124 APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1; 2125 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2126 } 2127 // If this is a signed comparison to 0 and the shift is sign preserving, 2128 // use the shift LHS operand instead; isSignTest may change 'Pred', so only 2129 // do that if we're sure to not continue on in this function. 2130 if (isSignTest(Pred, C)) 2131 return new ICmpInst(Pred, X, Constant::getNullValue(ShType)); 2132 } 2133 2134 // NUW guarantees that we are only shifting out zero bits from the high bits, 2135 // so we can LSHR the compare constant without needing a mask and eliminate 2136 // the shift. 2137 if (Shl->hasNoUnsignedWrap()) { 2138 if (Pred == ICmpInst::ICMP_UGT) { 2139 // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt) 2140 APInt ShiftedC = C.lshr(*ShiftAmt); 2141 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2142 } 2143 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && 2144 C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) { 2145 APInt ShiftedC = C.lshr(*ShiftAmt); 2146 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2147 } 2148 if (Pred == ICmpInst::ICMP_ULT) { 2149 // ULE is the same as above, but ULE is canonicalized to ULT, so convert: 2150 // (X << S) <=u C is equiv to X <=u (C >> S) for all C 2151 // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u 2152 // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0 2153 assert(C.ugt(0) && "ult 0 should have been eliminated"); 2154 APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1; 2155 return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); 2156 } 2157 } 2158 2159 if (Cmp.isEquality() && Shl->hasOneUse()) { 2160 // Strength-reduce the shift into an 'and'. 2161 Constant *Mask = ConstantInt::get( 2162 ShType, 2163 APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue())); 2164 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); 2165 Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt)); 2166 return new ICmpInst(Pred, And, LShrC); 2167 } 2168 2169 // Otherwise, if this is a comparison of the sign bit, simplify to and/test. 2170 bool TrueIfSigned = false; 2171 if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) { 2172 // (X << 31) <s 0 --> (X & 1) != 0 2173 Constant *Mask = ConstantInt::get( 2174 ShType, 2175 APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1)); 2176 Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); 2177 return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, 2178 And, Constant::getNullValue(ShType)); 2179 } 2180 2181 // Simplify 'shl' inequality test into 'and' equality test. 2182 if (Cmp.isUnsigned() && Shl->hasOneUse()) { 2183 // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0 2184 if ((C + 1).isPowerOf2() && 2185 (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) { 2186 Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue())); 2187 return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ 2188 : ICmpInst::ICMP_NE, 2189 And, Constant::getNullValue(ShType)); 2190 } 2191 // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0 2192 if (C.isPowerOf2() && 2193 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { 2194 Value *And = 2195 Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue())); 2196 return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ 2197 : ICmpInst::ICMP_NE, 2198 And, Constant::getNullValue(ShType)); 2199 } 2200 } 2201 2202 // Transform (icmp pred iM (shl iM %v, N), C) 2203 // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N)) 2204 // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N. 2205 // This enables us to get rid of the shift in favor of a trunc that may be 2206 // free on the target. It has the additional benefit of comparing to a 2207 // smaller constant that may be more target-friendly. 2208 unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1); 2209 if (Shl->hasOneUse() && Amt != 0 && C.countTrailingZeros() >= Amt && 2210 DL.isLegalInteger(TypeBits - Amt)) { 2211 Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt); 2212 if (auto *ShVTy = dyn_cast<VectorType>(ShType)) 2213 TruncTy = VectorType::get(TruncTy, ShVTy->getElementCount()); 2214 Constant *NewC = 2215 ConstantInt::get(TruncTy, C.ashr(*ShiftAmt).trunc(TypeBits - Amt)); 2216 return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC); 2217 } 2218 2219 return nullptr; 2220 } 2221 2222 /// Fold icmp ({al}shr X, Y), C. 2223 Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp, 2224 BinaryOperator *Shr, 2225 const APInt &C) { 2226 // An exact shr only shifts out zero bits, so: 2227 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 2228 Value *X = Shr->getOperand(0); 2229 CmpInst::Predicate Pred = Cmp.getPredicate(); 2230 if (Cmp.isEquality() && Shr->isExact() && Shr->hasOneUse() && 2231 C.isNullValue()) 2232 return new ICmpInst(Pred, X, Cmp.getOperand(1)); 2233 2234 const APInt *ShiftVal; 2235 if (Cmp.isEquality() && match(Shr->getOperand(0), m_APInt(ShiftVal))) 2236 return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftVal); 2237 2238 const APInt *ShiftAmt; 2239 if (!match(Shr->getOperand(1), m_APInt(ShiftAmt))) 2240 return nullptr; 2241 2242 // Check that the shift amount is in range. If not, don't perform undefined 2243 // shifts. When the shift is visited it will be simplified. 2244 unsigned TypeBits = C.getBitWidth(); 2245 unsigned ShAmtVal = ShiftAmt->getLimitedValue(TypeBits); 2246 if (ShAmtVal >= TypeBits || ShAmtVal == 0) 2247 return nullptr; 2248 2249 bool IsAShr = Shr->getOpcode() == Instruction::AShr; 2250 bool IsExact = Shr->isExact(); 2251 Type *ShrTy = Shr->getType(); 2252 // TODO: If we could guarantee that InstSimplify would handle all of the 2253 // constant-value-based preconditions in the folds below, then we could assert 2254 // those conditions rather than checking them. This is difficult because of 2255 // undef/poison (PR34838). 2256 if (IsAShr) { 2257 if (Pred == CmpInst::ICMP_SLT || (Pred == CmpInst::ICMP_SGT && IsExact)) { 2258 // icmp slt (ashr X, ShAmtC), C --> icmp slt X, (C << ShAmtC) 2259 // icmp sgt (ashr exact X, ShAmtC), C --> icmp sgt X, (C << ShAmtC) 2260 APInt ShiftedC = C.shl(ShAmtVal); 2261 if (ShiftedC.ashr(ShAmtVal) == C) 2262 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2263 } 2264 if (Pred == CmpInst::ICMP_SGT) { 2265 // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1 2266 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; 2267 if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() && 2268 (ShiftedC + 1).ashr(ShAmtVal) == (C + 1)) 2269 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2270 } 2271 2272 // If the compare constant has significant bits above the lowest sign-bit, 2273 // then convert an unsigned cmp to a test of the sign-bit: 2274 // (ashr X, ShiftC) u> C --> X s< 0 2275 // (ashr X, ShiftC) u< C --> X s> -1 2276 if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) { 2277 if (Pred == CmpInst::ICMP_UGT) { 2278 return new ICmpInst(CmpInst::ICMP_SLT, X, 2279 ConstantInt::getNullValue(ShrTy)); 2280 } 2281 if (Pred == CmpInst::ICMP_ULT) { 2282 return new ICmpInst(CmpInst::ICMP_SGT, X, 2283 ConstantInt::getAllOnesValue(ShrTy)); 2284 } 2285 } 2286 } else { 2287 if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) { 2288 // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC) 2289 // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC) 2290 APInt ShiftedC = C.shl(ShAmtVal); 2291 if (ShiftedC.lshr(ShAmtVal) == C) 2292 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2293 } 2294 if (Pred == CmpInst::ICMP_UGT) { 2295 // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 2296 APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; 2297 if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1)) 2298 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); 2299 } 2300 } 2301 2302 if (!Cmp.isEquality()) 2303 return nullptr; 2304 2305 // Handle equality comparisons of shift-by-constant. 2306 2307 // If the comparison constant changes with the shift, the comparison cannot 2308 // succeed (bits of the comparison constant cannot match the shifted value). 2309 // This should be known by InstSimplify and already be folded to true/false. 2310 assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || 2311 (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && 2312 "Expected icmp+shr simplify did not occur."); 2313 2314 // If the bits shifted out are known zero, compare the unshifted value: 2315 // (X & 4) >> 1 == 2 --> (X & 4) == 4. 2316 if (Shr->isExact()) 2317 return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal)); 2318 2319 if (C.isNullValue()) { 2320 // == 0 is u< 1. 2321 if (Pred == CmpInst::ICMP_EQ) 2322 return new ICmpInst(CmpInst::ICMP_ULT, X, 2323 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal))); 2324 else 2325 return new ICmpInst(CmpInst::ICMP_UGT, X, 2326 ConstantInt::get(ShrTy, (C + 1).shl(ShAmtVal) - 1)); 2327 } 2328 2329 if (Shr->hasOneUse()) { 2330 // Canonicalize the shift into an 'and': 2331 // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt) 2332 APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal)); 2333 Constant *Mask = ConstantInt::get(ShrTy, Val); 2334 Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask"); 2335 return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal)); 2336 } 2337 2338 return nullptr; 2339 } 2340 2341 Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp, 2342 BinaryOperator *SRem, 2343 const APInt &C) { 2344 // Match an 'is positive' or 'is negative' comparison of remainder by a 2345 // constant power-of-2 value: 2346 // (X % pow2C) sgt/slt 0 2347 const ICmpInst::Predicate Pred = Cmp.getPredicate(); 2348 if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT) 2349 return nullptr; 2350 2351 // TODO: The one-use check is standard because we do not typically want to 2352 // create longer instruction sequences, but this might be a special-case 2353 // because srem is not good for analysis or codegen. 2354 if (!SRem->hasOneUse()) 2355 return nullptr; 2356 2357 const APInt *DivisorC; 2358 if (!C.isNullValue() || !match(SRem->getOperand(1), m_Power2(DivisorC))) 2359 return nullptr; 2360 2361 // Mask off the sign bit and the modulo bits (low-bits). 2362 Type *Ty = SRem->getType(); 2363 APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits()); 2364 Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1)); 2365 Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC); 2366 2367 // For 'is positive?' check that the sign-bit is clear and at least 1 masked 2368 // bit is set. Example: 2369 // (i8 X % 32) s> 0 --> (X & 159) s> 0 2370 if (Pred == ICmpInst::ICMP_SGT) 2371 return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty)); 2372 2373 // For 'is negative?' check that the sign-bit is set and at least 1 masked 2374 // bit is set. Example: 2375 // (i16 X % 4) s< 0 --> (X & 32771) u> 32768 2376 return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask)); 2377 } 2378 2379 /// Fold icmp (udiv X, Y), C. 2380 Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp, 2381 BinaryOperator *UDiv, 2382 const APInt &C) { 2383 const APInt *C2; 2384 if (!match(UDiv->getOperand(0), m_APInt(C2))) 2385 return nullptr; 2386 2387 assert(*C2 != 0 && "udiv 0, X should have been simplified already."); 2388 2389 // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1)) 2390 Value *Y = UDiv->getOperand(1); 2391 if (Cmp.getPredicate() == ICmpInst::ICMP_UGT) { 2392 assert(!C.isMaxValue() && 2393 "icmp ugt X, UINT_MAX should have been simplified already."); 2394 return new ICmpInst(ICmpInst::ICMP_ULE, Y, 2395 ConstantInt::get(Y->getType(), C2->udiv(C + 1))); 2396 } 2397 2398 // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C) 2399 if (Cmp.getPredicate() == ICmpInst::ICMP_ULT) { 2400 assert(C != 0 && "icmp ult X, 0 should have been simplified already."); 2401 return new ICmpInst(ICmpInst::ICMP_UGT, Y, 2402 ConstantInt::get(Y->getType(), C2->udiv(C))); 2403 } 2404 2405 return nullptr; 2406 } 2407 2408 /// Fold icmp ({su}div X, Y), C. 2409 Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp, 2410 BinaryOperator *Div, 2411 const APInt &C) { 2412 // Fold: icmp pred ([us]div X, C2), C -> range test 2413 // Fold this div into the comparison, producing a range check. 2414 // Determine, based on the divide type, what the range is being 2415 // checked. If there is an overflow on the low or high side, remember 2416 // it, otherwise compute the range [low, hi) bounding the new value. 2417 // See: InsertRangeTest above for the kinds of replacements possible. 2418 const APInt *C2; 2419 if (!match(Div->getOperand(1), m_APInt(C2))) 2420 return nullptr; 2421 2422 // FIXME: If the operand types don't match the type of the divide 2423 // then don't attempt this transform. The code below doesn't have the 2424 // logic to deal with a signed divide and an unsigned compare (and 2425 // vice versa). This is because (x /s C2) <s C produces different 2426 // results than (x /s C2) <u C or (x /u C2) <s C or even 2427 // (x /u C2) <u C. Simply casting the operands and result won't 2428 // work. :( The if statement below tests that condition and bails 2429 // if it finds it. 2430 bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; 2431 if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) 2432 return nullptr; 2433 2434 // The ProdOV computation fails on divide by 0 and divide by -1. Cases with 2435 // INT_MIN will also fail if the divisor is 1. Although folds of all these 2436 // division-by-constant cases should be present, we can not assert that they 2437 // have happened before we reach this icmp instruction. 2438 if (C2->isNullValue() || C2->isOneValue() || 2439 (DivIsSigned && C2->isAllOnesValue())) 2440 return nullptr; 2441 2442 // Compute Prod = C * C2. We are essentially solving an equation of 2443 // form X / C2 = C. We solve for X by multiplying C2 and C. 2444 // By solving for X, we can turn this into a range check instead of computing 2445 // a divide. 2446 APInt Prod = C * *C2; 2447 2448 // Determine if the product overflows by seeing if the product is not equal to 2449 // the divide. Make sure we do the same kind of divide as in the LHS 2450 // instruction that we're folding. 2451 bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C; 2452 2453 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2454 2455 // If the division is known to be exact, then there is no remainder from the 2456 // divide, so the covered range size is unit, otherwise it is the divisor. 2457 APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2; 2458 2459 // Figure out the interval that is being checked. For example, a comparison 2460 // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). 2461 // Compute this interval based on the constants involved and the signedness of 2462 // the compare/divide. This computes a half-open interval, keeping track of 2463 // whether either value in the interval overflows. After analysis each 2464 // overflow variable is set to 0 if it's corresponding bound variable is valid 2465 // -1 if overflowed off the bottom end, or +1 if overflowed off the top end. 2466 int LoOverflow = 0, HiOverflow = 0; 2467 APInt LoBound, HiBound; 2468 2469 if (!DivIsSigned) { // udiv 2470 // e.g. X/5 op 3 --> [15, 20) 2471 LoBound = Prod; 2472 HiOverflow = LoOverflow = ProdOV; 2473 if (!HiOverflow) { 2474 // If this is not an exact divide, then many values in the range collapse 2475 // to the same result value. 2476 HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false); 2477 } 2478 } else if (C2->isStrictlyPositive()) { // Divisor is > 0. 2479 if (C.isNullValue()) { // (X / pos) op 0 2480 // Can't overflow. e.g. X/2 op 0 --> [-1, 2) 2481 LoBound = -(RangeSize - 1); 2482 HiBound = RangeSize; 2483 } else if (C.isStrictlyPositive()) { // (X / pos) op pos 2484 LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) 2485 HiOverflow = LoOverflow = ProdOV; 2486 if (!HiOverflow) 2487 HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true); 2488 } else { // (X / pos) op neg 2489 // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) 2490 HiBound = Prod + 1; 2491 LoOverflow = HiOverflow = ProdOV ? -1 : 0; 2492 if (!LoOverflow) { 2493 APInt DivNeg = -RangeSize; 2494 LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0; 2495 } 2496 } 2497 } else if (C2->isNegative()) { // Divisor is < 0. 2498 if (Div->isExact()) 2499 RangeSize.negate(); 2500 if (C.isNullValue()) { // (X / neg) op 0 2501 // e.g. X/-5 op 0 --> [-4, 5) 2502 LoBound = RangeSize + 1; 2503 HiBound = -RangeSize; 2504 if (HiBound == *C2) { // -INTMIN = INTMIN 2505 HiOverflow = 1; // [INTMIN+1, overflow) 2506 HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN 2507 } 2508 } else if (C.isStrictlyPositive()) { // (X / neg) op pos 2509 // e.g. X/-5 op 3 --> [-19, -14) 2510 HiBound = Prod + 1; 2511 HiOverflow = LoOverflow = ProdOV ? -1 : 0; 2512 if (!LoOverflow) 2513 LoOverflow = addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1:0; 2514 } else { // (X / neg) op neg 2515 LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) 2516 LoOverflow = HiOverflow = ProdOV; 2517 if (!HiOverflow) 2518 HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true); 2519 } 2520 2521 // Dividing by a negative swaps the condition. LT <-> GT 2522 Pred = ICmpInst::getSwappedPredicate(Pred); 2523 } 2524 2525 Value *X = Div->getOperand(0); 2526 switch (Pred) { 2527 default: llvm_unreachable("Unhandled icmp opcode!"); 2528 case ICmpInst::ICMP_EQ: 2529 if (LoOverflow && HiOverflow) 2530 return replaceInstUsesWith(Cmp, Builder.getFalse()); 2531 if (HiOverflow) 2532 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : 2533 ICmpInst::ICMP_UGE, X, 2534 ConstantInt::get(Div->getType(), LoBound)); 2535 if (LoOverflow) 2536 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : 2537 ICmpInst::ICMP_ULT, X, 2538 ConstantInt::get(Div->getType(), HiBound)); 2539 return replaceInstUsesWith( 2540 Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true)); 2541 case ICmpInst::ICMP_NE: 2542 if (LoOverflow && HiOverflow) 2543 return replaceInstUsesWith(Cmp, Builder.getTrue()); 2544 if (HiOverflow) 2545 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : 2546 ICmpInst::ICMP_ULT, X, 2547 ConstantInt::get(Div->getType(), LoBound)); 2548 if (LoOverflow) 2549 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : 2550 ICmpInst::ICMP_UGE, X, 2551 ConstantInt::get(Div->getType(), HiBound)); 2552 return replaceInstUsesWith(Cmp, 2553 insertRangeTest(X, LoBound, HiBound, 2554 DivIsSigned, false)); 2555 case ICmpInst::ICMP_ULT: 2556 case ICmpInst::ICMP_SLT: 2557 if (LoOverflow == +1) // Low bound is greater than input range. 2558 return replaceInstUsesWith(Cmp, Builder.getTrue()); 2559 if (LoOverflow == -1) // Low bound is less than input range. 2560 return replaceInstUsesWith(Cmp, Builder.getFalse()); 2561 return new ICmpInst(Pred, X, ConstantInt::get(Div->getType(), LoBound)); 2562 case ICmpInst::ICMP_UGT: 2563 case ICmpInst::ICMP_SGT: 2564 if (HiOverflow == +1) // High bound greater than input range. 2565 return replaceInstUsesWith(Cmp, Builder.getFalse()); 2566 if (HiOverflow == -1) // High bound less than input range. 2567 return replaceInstUsesWith(Cmp, Builder.getTrue()); 2568 if (Pred == ICmpInst::ICMP_UGT) 2569 return new ICmpInst(ICmpInst::ICMP_UGE, X, 2570 ConstantInt::get(Div->getType(), HiBound)); 2571 return new ICmpInst(ICmpInst::ICMP_SGE, X, 2572 ConstantInt::get(Div->getType(), HiBound)); 2573 } 2574 2575 return nullptr; 2576 } 2577 2578 /// Fold icmp (sub X, Y), C. 2579 Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp, 2580 BinaryOperator *Sub, 2581 const APInt &C) { 2582 Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1); 2583 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2584 const APInt *C2; 2585 APInt SubResult; 2586 2587 // icmp eq/ne (sub C, Y), C -> icmp eq/ne Y, 0 2588 if (match(X, m_APInt(C2)) && *C2 == C && Cmp.isEquality()) 2589 return new ICmpInst(Cmp.getPredicate(), Y, 2590 ConstantInt::get(Y->getType(), 0)); 2591 2592 // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C) 2593 if (match(X, m_APInt(C2)) && 2594 ((Cmp.isUnsigned() && Sub->hasNoUnsignedWrap()) || 2595 (Cmp.isSigned() && Sub->hasNoSignedWrap())) && 2596 !subWithOverflow(SubResult, *C2, C, Cmp.isSigned())) 2597 return new ICmpInst(Cmp.getSwappedPredicate(), Y, 2598 ConstantInt::get(Y->getType(), SubResult)); 2599 2600 // The following transforms are only worth it if the only user of the subtract 2601 // is the icmp. 2602 if (!Sub->hasOneUse()) 2603 return nullptr; 2604 2605 if (Sub->hasNoSignedWrap()) { 2606 // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) 2607 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnesValue()) 2608 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); 2609 2610 // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) 2611 if (Pred == ICmpInst::ICMP_SGT && C.isNullValue()) 2612 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); 2613 2614 // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) 2615 if (Pred == ICmpInst::ICMP_SLT && C.isNullValue()) 2616 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); 2617 2618 // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) 2619 if (Pred == ICmpInst::ICMP_SLT && C.isOneValue()) 2620 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); 2621 } 2622 2623 if (!match(X, m_APInt(C2))) 2624 return nullptr; 2625 2626 // C2 - Y <u C -> (Y | (C - 1)) == C2 2627 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 2628 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && 2629 (*C2 & (C - 1)) == (C - 1)) 2630 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X); 2631 2632 // C2 - Y >u C -> (Y | C) != C2 2633 // iff C2 & C == C and C + 1 is a power of 2 2634 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C) 2635 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X); 2636 2637 return nullptr; 2638 } 2639 2640 /// Fold icmp (add X, Y), C. 2641 Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp, 2642 BinaryOperator *Add, 2643 const APInt &C) { 2644 Value *Y = Add->getOperand(1); 2645 const APInt *C2; 2646 if (Cmp.isEquality() || !match(Y, m_APInt(C2))) 2647 return nullptr; 2648 2649 // Fold icmp pred (add X, C2), C. 2650 Value *X = Add->getOperand(0); 2651 Type *Ty = Add->getType(); 2652 const CmpInst::Predicate Pred = Cmp.getPredicate(); 2653 2654 // If the add does not wrap, we can always adjust the compare by subtracting 2655 // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE 2656 // are canonicalized to SGT/SLT/UGT/ULT. 2657 if ((Add->hasNoSignedWrap() && 2658 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) || 2659 (Add->hasNoUnsignedWrap() && 2660 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) { 2661 bool Overflow; 2662 APInt NewC = 2663 Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow); 2664 // If there is overflow, the result must be true or false. 2665 // TODO: Can we assert there is no overflow because InstSimplify always 2666 // handles those cases? 2667 if (!Overflow) 2668 // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2) 2669 return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC)); 2670 } 2671 2672 auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2); 2673 const APInt &Upper = CR.getUpper(); 2674 const APInt &Lower = CR.getLower(); 2675 if (Cmp.isSigned()) { 2676 if (Lower.isSignMask()) 2677 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper)); 2678 if (Upper.isSignMask()) 2679 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower)); 2680 } else { 2681 if (Lower.isMinValue()) 2682 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper)); 2683 if (Upper.isMinValue()) 2684 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower)); 2685 } 2686 2687 // This set of folds is intentionally placed after folds that use no-wrapping 2688 // flags because those folds are likely better for later analysis/codegen. 2689 const APInt SMax = APInt::getSignedMaxValue(Ty->getScalarSizeInBits()); 2690 const APInt SMin = APInt::getSignedMinValue(Ty->getScalarSizeInBits()); 2691 2692 // Fold compare with offset to opposite sign compare if it eliminates offset: 2693 // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX) 2694 if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax) 2695 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, -(*C2))); 2696 2697 // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN) 2698 if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin) 2699 return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, ~(*C2))); 2700 2701 // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1) 2702 if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1) 2703 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, SMax - C)); 2704 2705 // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2) 2706 if (Pred == CmpInst::ICMP_SLT && C == *C2) 2707 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, C ^ SMax)); 2708 2709 if (!Add->hasOneUse()) 2710 return nullptr; 2711 2712 // X+C <u C2 -> (X & -C2) == C 2713 // iff C & (C2-1) == 0 2714 // C2 is a power of 2 2715 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0) 2716 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -C), 2717 ConstantExpr::getNeg(cast<Constant>(Y))); 2718 2719 // X+C >u C2 -> (X & ~C2) != C 2720 // iff C & C2 == 0 2721 // C2+1 is a power of 2 2722 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0) 2723 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~C), 2724 ConstantExpr::getNeg(cast<Constant>(Y))); 2725 2726 return nullptr; 2727 } 2728 2729 bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, 2730 Value *&RHS, ConstantInt *&Less, 2731 ConstantInt *&Equal, 2732 ConstantInt *&Greater) { 2733 // TODO: Generalize this to work with other comparison idioms or ensure 2734 // they get canonicalized into this form. 2735 2736 // select i1 (a == b), 2737 // i32 Equal, 2738 // i32 (select i1 (a < b), i32 Less, i32 Greater) 2739 // where Equal, Less and Greater are placeholders for any three constants. 2740 ICmpInst::Predicate PredA; 2741 if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) || 2742 !ICmpInst::isEquality(PredA)) 2743 return false; 2744 Value *EqualVal = SI->getTrueValue(); 2745 Value *UnequalVal = SI->getFalseValue(); 2746 // We still can get non-canonical predicate here, so canonicalize. 2747 if (PredA == ICmpInst::ICMP_NE) 2748 std::swap(EqualVal, UnequalVal); 2749 if (!match(EqualVal, m_ConstantInt(Equal))) 2750 return false; 2751 ICmpInst::Predicate PredB; 2752 Value *LHS2, *RHS2; 2753 if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)), 2754 m_ConstantInt(Less), m_ConstantInt(Greater)))) 2755 return false; 2756 // We can get predicate mismatch here, so canonicalize if possible: 2757 // First, ensure that 'LHS' match. 2758 if (LHS2 != LHS) { 2759 // x sgt y <--> y slt x 2760 std::swap(LHS2, RHS2); 2761 PredB = ICmpInst::getSwappedPredicate(PredB); 2762 } 2763 if (LHS2 != LHS) 2764 return false; 2765 // We also need to canonicalize 'RHS'. 2766 if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) { 2767 // x sgt C-1 <--> x sge C <--> not(x slt C) 2768 auto FlippedStrictness = 2769 InstCombiner::getFlippedStrictnessPredicateAndConstant( 2770 PredB, cast<Constant>(RHS2)); 2771 if (!FlippedStrictness) 2772 return false; 2773 assert(FlippedStrictness->first == ICmpInst::ICMP_SGE && "Sanity check"); 2774 RHS2 = FlippedStrictness->second; 2775 // And kind-of perform the result swap. 2776 std::swap(Less, Greater); 2777 PredB = ICmpInst::ICMP_SLT; 2778 } 2779 return PredB == ICmpInst::ICMP_SLT && RHS == RHS2; 2780 } 2781 2782 Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp, 2783 SelectInst *Select, 2784 ConstantInt *C) { 2785 2786 assert(C && "Cmp RHS should be a constant int!"); 2787 // If we're testing a constant value against the result of a three way 2788 // comparison, the result can be expressed directly in terms of the 2789 // original values being compared. Note: We could possibly be more 2790 // aggressive here and remove the hasOneUse test. The original select is 2791 // really likely to simplify or sink when we remove a test of the result. 2792 Value *OrigLHS, *OrigRHS; 2793 ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan; 2794 if (Cmp.hasOneUse() && 2795 matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal, 2796 C3GreaterThan)) { 2797 assert(C1LessThan && C2Equal && C3GreaterThan); 2798 2799 bool TrueWhenLessThan = 2800 ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C) 2801 ->isAllOnesValue(); 2802 bool TrueWhenEqual = 2803 ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C) 2804 ->isAllOnesValue(); 2805 bool TrueWhenGreaterThan = 2806 ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C) 2807 ->isAllOnesValue(); 2808 2809 // This generates the new instruction that will replace the original Cmp 2810 // Instruction. Instead of enumerating the various combinations when 2811 // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus 2812 // false, we rely on chaining of ORs and future passes of InstCombine to 2813 // simplify the OR further (i.e. a s< b || a == b becomes a s<= b). 2814 2815 // When none of the three constants satisfy the predicate for the RHS (C), 2816 // the entire original Cmp can be simplified to a false. 2817 Value *Cond = Builder.getFalse(); 2818 if (TrueWhenLessThan) 2819 Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, 2820 OrigLHS, OrigRHS)); 2821 if (TrueWhenEqual) 2822 Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, 2823 OrigLHS, OrigRHS)); 2824 if (TrueWhenGreaterThan) 2825 Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, 2826 OrigLHS, OrigRHS)); 2827 2828 return replaceInstUsesWith(Cmp, Cond); 2829 } 2830 return nullptr; 2831 } 2832 2833 static Instruction *foldICmpBitCast(ICmpInst &Cmp, 2834 InstCombiner::BuilderTy &Builder) { 2835 auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0)); 2836 if (!Bitcast) 2837 return nullptr; 2838 2839 ICmpInst::Predicate Pred = Cmp.getPredicate(); 2840 Value *Op1 = Cmp.getOperand(1); 2841 Value *BCSrcOp = Bitcast->getOperand(0); 2842 2843 // Make sure the bitcast doesn't change the number of vector elements. 2844 if (Bitcast->getSrcTy()->getScalarSizeInBits() == 2845 Bitcast->getDestTy()->getScalarSizeInBits()) { 2846 // Zero-equality and sign-bit checks are preserved through sitofp + bitcast. 2847 Value *X; 2848 if (match(BCSrcOp, m_SIToFP(m_Value(X)))) { 2849 // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0 2850 // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0 2851 // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0 2852 // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0 2853 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT || 2854 Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) && 2855 match(Op1, m_Zero())) 2856 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); 2857 2858 // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1 2859 if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One())) 2860 return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1)); 2861 2862 // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1 2863 if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes())) 2864 return new ICmpInst(Pred, X, 2865 ConstantInt::getAllOnesValue(X->getType())); 2866 } 2867 2868 // Zero-equality checks are preserved through unsigned floating-point casts: 2869 // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0 2870 // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0 2871 if (match(BCSrcOp, m_UIToFP(m_Value(X)))) 2872 if (Cmp.isEquality() && match(Op1, m_Zero())) 2873 return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); 2874 2875 // If this is a sign-bit test of a bitcast of a casted FP value, eliminate 2876 // the FP extend/truncate because that cast does not change the sign-bit. 2877 // This is true for all standard IEEE-754 types and the X86 80-bit type. 2878 // The sign-bit is always the most significant bit in those types. 2879 const APInt *C; 2880 bool TrueIfSigned; 2881 if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse() && 2882 InstCombiner::isSignBitCheck(Pred, *C, TrueIfSigned)) { 2883 if (match(BCSrcOp, m_FPExt(m_Value(X))) || 2884 match(BCSrcOp, m_FPTrunc(m_Value(X)))) { 2885 // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0 2886 // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1 2887 Type *XType = X->getType(); 2888 2889 // We can't currently handle Power style floating point operations here. 2890 if (!(XType->isPPC_FP128Ty() || BCSrcOp->getType()->isPPC_FP128Ty())) { 2891 2892 Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits()); 2893 if (auto *XVTy = dyn_cast<VectorType>(XType)) 2894 NewType = VectorType::get(NewType, XVTy->getElementCount()); 2895 Value *NewBitcast = Builder.CreateBitCast(X, NewType); 2896 if (TrueIfSigned) 2897 return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast, 2898 ConstantInt::getNullValue(NewType)); 2899 else 2900 return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast, 2901 ConstantInt::getAllOnesValue(NewType)); 2902 } 2903 } 2904 } 2905 } 2906 2907 // Test to see if the operands of the icmp are casted versions of other 2908 // values. If the ptr->ptr cast can be stripped off both arguments, do so. 2909 if (Bitcast->getType()->isPointerTy() && 2910 (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) { 2911 // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast 2912 // so eliminate it as well. 2913 if (auto *BC2 = dyn_cast<BitCastInst>(Op1)) 2914 Op1 = BC2->getOperand(0); 2915 2916 Op1 = Builder.CreateBitCast(Op1, BCSrcOp->getType()); 2917 return new ICmpInst(Pred, BCSrcOp, Op1); 2918 } 2919 2920 // Folding: icmp <pred> iN X, C 2921 // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN 2922 // and C is a splat of a K-bit pattern 2923 // and SC is a constant vector = <C', C', C', ..., C'> 2924 // Into: 2925 // %E = extractelement <M x iK> %vec, i32 C' 2926 // icmp <pred> iK %E, trunc(C) 2927 const APInt *C; 2928 if (!match(Cmp.getOperand(1), m_APInt(C)) || 2929 !Bitcast->getType()->isIntegerTy() || 2930 !Bitcast->getSrcTy()->isIntOrIntVectorTy()) 2931 return nullptr; 2932 2933 Value *Vec; 2934 ArrayRef<int> Mask; 2935 if (match(BCSrcOp, m_Shuffle(m_Value(Vec), m_Undef(), m_Mask(Mask)))) { 2936 // Check whether every element of Mask is the same constant 2937 if (is_splat(Mask)) { 2938 auto *VecTy = cast<VectorType>(BCSrcOp->getType()); 2939 auto *EltTy = cast<IntegerType>(VecTy->getElementType()); 2940 if (C->isSplat(EltTy->getBitWidth())) { 2941 // Fold the icmp based on the value of C 2942 // If C is M copies of an iK sized bit pattern, 2943 // then: 2944 // => %E = extractelement <N x iK> %vec, i32 Elem 2945 // icmp <pred> iK %SplatVal, <pattern> 2946 Value *Elem = Builder.getInt32(Mask[0]); 2947 Value *Extract = Builder.CreateExtractElement(Vec, Elem); 2948 Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth())); 2949 return new ICmpInst(Pred, Extract, NewC); 2950 } 2951 } 2952 } 2953 return nullptr; 2954 } 2955 2956 /// Try to fold integer comparisons with a constant operand: icmp Pred X, C 2957 /// where X is some kind of instruction. 2958 Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) { 2959 const APInt *C; 2960 if (!match(Cmp.getOperand(1), m_APInt(C))) 2961 return nullptr; 2962 2963 if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0))) { 2964 switch (BO->getOpcode()) { 2965 case Instruction::Xor: 2966 if (Instruction *I = foldICmpXorConstant(Cmp, BO, *C)) 2967 return I; 2968 break; 2969 case Instruction::And: 2970 if (Instruction *I = foldICmpAndConstant(Cmp, BO, *C)) 2971 return I; 2972 break; 2973 case Instruction::Or: 2974 if (Instruction *I = foldICmpOrConstant(Cmp, BO, *C)) 2975 return I; 2976 break; 2977 case Instruction::Mul: 2978 if (Instruction *I = foldICmpMulConstant(Cmp, BO, *C)) 2979 return I; 2980 break; 2981 case Instruction::Shl: 2982 if (Instruction *I = foldICmpShlConstant(Cmp, BO, *C)) 2983 return I; 2984 break; 2985 case Instruction::LShr: 2986 case Instruction::AShr: 2987 if (Instruction *I = foldICmpShrConstant(Cmp, BO, *C)) 2988 return I; 2989 break; 2990 case Instruction::SRem: 2991 if (Instruction *I = foldICmpSRemConstant(Cmp, BO, *C)) 2992 return I; 2993 break; 2994 case Instruction::UDiv: 2995 if (Instruction *I = foldICmpUDivConstant(Cmp, BO, *C)) 2996 return I; 2997 LLVM_FALLTHROUGH; 2998 case Instruction::SDiv: 2999 if (Instruction *I = foldICmpDivConstant(Cmp, BO, *C)) 3000 return I; 3001 break; 3002 case Instruction::Sub: 3003 if (Instruction *I = foldICmpSubConstant(Cmp, BO, *C)) 3004 return I; 3005 break; 3006 case Instruction::Add: 3007 if (Instruction *I = foldICmpAddConstant(Cmp, BO, *C)) 3008 return I; 3009 break; 3010 default: 3011 break; 3012 } 3013 // TODO: These folds could be refactored to be part of the above calls. 3014 if (Instruction *I = foldICmpBinOpEqualityWithConstant(Cmp, BO, *C)) 3015 return I; 3016 } 3017 3018 // Match against CmpInst LHS being instructions other than binary operators. 3019 3020 if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0))) { 3021 // For now, we only support constant integers while folding the 3022 // ICMP(SELECT)) pattern. We can extend this to support vector of integers 3023 // similar to the cases handled by binary ops above. 3024 if (ConstantInt *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1))) 3025 if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS)) 3026 return I; 3027 } 3028 3029 if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0))) { 3030 if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C)) 3031 return I; 3032 } 3033 3034 if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) 3035 if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C)) 3036 return I; 3037 3038 return nullptr; 3039 } 3040 3041 /// Fold an icmp equality instruction with binary operator LHS and constant RHS: 3042 /// icmp eq/ne BO, C. 3043 Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant( 3044 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) { 3045 // TODO: Some of these folds could work with arbitrary constants, but this 3046 // function is limited to scalar and vector splat constants. 3047 if (!Cmp.isEquality()) 3048 return nullptr; 3049 3050 ICmpInst::Predicate Pred = Cmp.getPredicate(); 3051 bool isICMP_NE = Pred == ICmpInst::ICMP_NE; 3052 Constant *RHS = cast<Constant>(Cmp.getOperand(1)); 3053 Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1); 3054 3055 switch (BO->getOpcode()) { 3056 case Instruction::SRem: 3057 // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. 3058 if (C.isNullValue() && BO->hasOneUse()) { 3059 const APInt *BOC; 3060 if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { 3061 Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName()); 3062 return new ICmpInst(Pred, NewRem, 3063 Constant::getNullValue(BO->getType())); 3064 } 3065 } 3066 break; 3067 case Instruction::Add: { 3068 // Replace ((add A, B) != C) with (A != C-B) if B & C are constants. 3069 if (Constant *BOC = dyn_cast<Constant>(BOp1)) { 3070 if (BO->hasOneUse()) 3071 return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, BOC)); 3072 } else if (C.isNullValue()) { 3073 // Replace ((add A, B) != 0) with (A != -B) if A or B is 3074 // efficiently invertible, or if the add has just this one use. 3075 if (Value *NegVal = dyn_castNegVal(BOp1)) 3076 return new ICmpInst(Pred, BOp0, NegVal); 3077 if (Value *NegVal = dyn_castNegVal(BOp0)) 3078 return new ICmpInst(Pred, NegVal, BOp1); 3079 if (BO->hasOneUse()) { 3080 Value *Neg = Builder.CreateNeg(BOp1); 3081 Neg->takeName(BO); 3082 return new ICmpInst(Pred, BOp0, Neg); 3083 } 3084 } 3085 break; 3086 } 3087 case Instruction::Xor: 3088 if (BO->hasOneUse()) { 3089 if (Constant *BOC = dyn_cast<Constant>(BOp1)) { 3090 // For the xor case, we can xor two constants together, eliminating 3091 // the explicit xor. 3092 return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC)); 3093 } else if (C.isNullValue()) { 3094 // Replace ((xor A, B) != 0) with (A != B) 3095 return new ICmpInst(Pred, BOp0, BOp1); 3096 } 3097 } 3098 break; 3099 case Instruction::Sub: 3100 if (BO->hasOneUse()) { 3101 // Only check for constant LHS here, as constant RHS will be canonicalized 3102 // to add and use the fold above. 3103 if (Constant *BOC = dyn_cast<Constant>(BOp0)) { 3104 // Replace ((sub BOC, B) != C) with (B != BOC-C). 3105 return new ICmpInst(Pred, BOp1, ConstantExpr::getSub(BOC, RHS)); 3106 } else if (C.isNullValue()) { 3107 // Replace ((sub A, B) != 0) with (A != B). 3108 return new ICmpInst(Pred, BOp0, BOp1); 3109 } 3110 } 3111 break; 3112 case Instruction::Or: { 3113 const APInt *BOC; 3114 if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { 3115 // Comparing if all bits outside of a constant mask are set? 3116 // Replace (X | C) == -1 with (X & ~C) == ~C. 3117 // This removes the -1 constant. 3118 Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1)); 3119 Value *And = Builder.CreateAnd(BOp0, NotBOC); 3120 return new ICmpInst(Pred, And, NotBOC); 3121 } 3122 break; 3123 } 3124 case Instruction::And: { 3125 const APInt *BOC; 3126 if (match(BOp1, m_APInt(BOC))) { 3127 // If we have ((X & C) == C), turn it into ((X & C) != 0). 3128 if (C == *BOC && C.isPowerOf2()) 3129 return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE, 3130 BO, Constant::getNullValue(RHS->getType())); 3131 } 3132 break; 3133 } 3134 case Instruction::UDiv: 3135 if (C.isNullValue()) { 3136 // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) 3137 auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; 3138 return new ICmpInst(NewPred, BOp1, BOp0); 3139 } 3140 break; 3141 default: 3142 break; 3143 } 3144 return nullptr; 3145 } 3146 3147 /// Fold an equality icmp with LLVM intrinsic and constant operand. 3148 Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant( 3149 ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) { 3150 Type *Ty = II->getType(); 3151 unsigned BitWidth = C.getBitWidth(); 3152 switch (II->getIntrinsicID()) { 3153 case Intrinsic::abs: 3154 // abs(A) == 0 -> A == 0 3155 // abs(A) == INT_MIN -> A == INT_MIN 3156 if (C.isNullValue() || C.isMinSignedValue()) 3157 return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), 3158 ConstantInt::get(Ty, C)); 3159 break; 3160 3161 case Intrinsic::bswap: 3162 // bswap(A) == C -> A == bswap(C) 3163 return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), 3164 ConstantInt::get(Ty, C.byteSwap())); 3165 3166 case Intrinsic::ctlz: 3167 case Intrinsic::cttz: { 3168 // ctz(A) == bitwidth(A) -> A == 0 and likewise for != 3169 if (C == BitWidth) 3170 return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), 3171 ConstantInt::getNullValue(Ty)); 3172 3173 // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set 3174 // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits. 3175 // Limit to one use to ensure we don't increase instruction count. 3176 unsigned Num = C.getLimitedValue(BitWidth); 3177 if (Num != BitWidth && II->hasOneUse()) { 3178 bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz; 3179 APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1) 3180 : APInt::getHighBitsSet(BitWidth, Num + 1); 3181 APInt Mask2 = IsTrailing 3182 ? APInt::getOneBitSet(BitWidth, Num) 3183 : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); 3184 return new ICmpInst(Cmp.getPredicate(), 3185 Builder.CreateAnd(II->getArgOperand(0), Mask1), 3186 ConstantInt::get(Ty, Mask2)); 3187 } 3188 break; 3189 } 3190 3191 case Intrinsic::ctpop: { 3192 // popcount(A) == 0 -> A == 0 and likewise for != 3193 // popcount(A) == bitwidth(A) -> A == -1 and likewise for != 3194 bool IsZero = C.isNullValue(); 3195 if (IsZero || C == BitWidth) 3196 return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), 3197 IsZero ? Constant::getNullValue(Ty) : Constant::getAllOnesValue(Ty)); 3198 3199 break; 3200 } 3201 3202 case Intrinsic::uadd_sat: { 3203 // uadd.sat(a, b) == 0 -> (a | b) == 0 3204 if (C.isNullValue()) { 3205 Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1)); 3206 return new ICmpInst(Cmp.getPredicate(), Or, Constant::getNullValue(Ty)); 3207 } 3208 break; 3209 } 3210 3211 case Intrinsic::usub_sat: { 3212 // usub.sat(a, b) == 0 -> a <= b 3213 if (C.isNullValue()) { 3214 ICmpInst::Predicate NewPred = Cmp.getPredicate() == ICmpInst::ICMP_EQ 3215 ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; 3216 return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1)); 3217 } 3218 break; 3219 } 3220 default: 3221 break; 3222 } 3223 3224 return nullptr; 3225 } 3226 3227 /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C. 3228 Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, 3229 IntrinsicInst *II, 3230 const APInt &C) { 3231 if (Cmp.isEquality()) 3232 return foldICmpEqIntrinsicWithConstant(Cmp, II, C); 3233 3234 Type *Ty = II->getType(); 3235 unsigned BitWidth = C.getBitWidth(); 3236 ICmpInst::Predicate Pred = Cmp.getPredicate(); 3237 switch (II->getIntrinsicID()) { 3238 case Intrinsic::ctpop: { 3239 // (ctpop X > BitWidth - 1) --> X == -1 3240 Value *X = II->getArgOperand(0); 3241 if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT) 3242 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, X, 3243 ConstantInt::getAllOnesValue(Ty)); 3244 // (ctpop X < BitWidth) --> X != -1 3245 if (C == BitWidth && Pred == ICmpInst::ICMP_ULT) 3246 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, X, 3247 ConstantInt::getAllOnesValue(Ty)); 3248 break; 3249 } 3250 case Intrinsic::ctlz: { 3251 // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000 3252 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { 3253 unsigned Num = C.getLimitedValue(); 3254 APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); 3255 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, 3256 II->getArgOperand(0), ConstantInt::get(Ty, Limit)); 3257 } 3258 3259 // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111 3260 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) { 3261 unsigned Num = C.getLimitedValue(); 3262 APInt Limit = APInt::getLowBitsSet(BitWidth, BitWidth - Num); 3263 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, 3264 II->getArgOperand(0), ConstantInt::get(Ty, Limit)); 3265 } 3266 break; 3267 } 3268 case Intrinsic::cttz: { 3269 // Limit to one use to ensure we don't increase instruction count. 3270 if (!II->hasOneUse()) 3271 return nullptr; 3272 3273 // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0 3274 if (Pred == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { 3275 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1); 3276 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, 3277 Builder.CreateAnd(II->getArgOperand(0), Mask), 3278 ConstantInt::getNullValue(Ty)); 3279 } 3280 3281 // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0 3282 if (Pred == ICmpInst::ICMP_ULT && C.uge(1) && C.ule(BitWidth)) { 3283 APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue()); 3284 return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, 3285 Builder.CreateAnd(II->getArgOperand(0), Mask), 3286 ConstantInt::getNullValue(Ty)); 3287 } 3288 break; 3289 } 3290 default: 3291 break; 3292 } 3293 3294 return nullptr; 3295 } 3296 3297 /// Handle icmp with constant (but not simple integer constant) RHS. 3298 Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) { 3299 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 3300 Constant *RHSC = dyn_cast<Constant>(Op1); 3301 Instruction *LHSI = dyn_cast<Instruction>(Op0); 3302 if (!RHSC || !LHSI) 3303 return nullptr; 3304 3305 switch (LHSI->getOpcode()) { 3306 case Instruction::GetElementPtr: 3307 // icmp pred GEP (P, int 0, int 0, int 0), null -> icmp pred P, null 3308 if (RHSC->isNullValue() && 3309 cast<GetElementPtrInst>(LHSI)->hasAllZeroIndices()) 3310 return new ICmpInst( 3311 I.getPredicate(), LHSI->getOperand(0), 3312 Constant::getNullValue(LHSI->getOperand(0)->getType())); 3313 break; 3314 case Instruction::PHI: 3315 // Only fold icmp into the PHI if the phi and icmp are in the same 3316 // block. If in the same block, we're encouraging jump threading. If 3317 // not, we are just pessimizing the code by making an i1 phi. 3318 if (LHSI->getParent() == I.getParent()) 3319 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) 3320 return NV; 3321 break; 3322 case Instruction::Select: { 3323 // If either operand of the select is a constant, we can fold the 3324 // comparison into the select arms, which will cause one to be 3325 // constant folded and the select turned into a bitwise or. 3326 Value *Op1 = nullptr, *Op2 = nullptr; 3327 ConstantInt *CI = nullptr; 3328 3329 auto SimplifyOp = [&](Value *V) { 3330 Value *Op = nullptr; 3331 if (Constant *C = dyn_cast<Constant>(V)) { 3332 Op = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); 3333 } else if (RHSC->isNullValue()) { 3334 // If null is being compared, check if it can be further simplified. 3335 Op = SimplifyICmpInst(I.getPredicate(), V, RHSC, SQ); 3336 } 3337 return Op; 3338 }; 3339 Op1 = SimplifyOp(LHSI->getOperand(1)); 3340 if (Op1) 3341 CI = dyn_cast<ConstantInt>(Op1); 3342 3343 Op2 = SimplifyOp(LHSI->getOperand(2)); 3344 if (Op2) 3345 CI = dyn_cast<ConstantInt>(Op2); 3346 3347 // We only want to perform this transformation if it will not lead to 3348 // additional code. This is true if either both sides of the select 3349 // fold to a constant (in which case the icmp is replaced with a select 3350 // which will usually simplify) or this is the only user of the 3351 // select (in which case we are trading a select+icmp for a simpler 3352 // select+icmp) or all uses of the select can be replaced based on 3353 // dominance information ("Global cases"). 3354 bool Transform = false; 3355 if (Op1 && Op2) 3356 Transform = true; 3357 else if (Op1 || Op2) { 3358 // Local case 3359 if (LHSI->hasOneUse()) 3360 Transform = true; 3361 // Global cases 3362 else if (CI && !CI->isZero()) 3363 // When Op1 is constant try replacing select with second operand. 3364 // Otherwise Op2 is constant and try replacing select with first 3365 // operand. 3366 Transform = 3367 replacedSelectWithOperand(cast<SelectInst>(LHSI), &I, Op1 ? 2 : 1); 3368 } 3369 if (Transform) { 3370 if (!Op1) 3371 Op1 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(1), RHSC, 3372 I.getName()); 3373 if (!Op2) 3374 Op2 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(2), RHSC, 3375 I.getName()); 3376 return SelectInst::Create(LHSI->getOperand(0), Op1, Op2); 3377 } 3378 break; 3379 } 3380 case Instruction::IntToPtr: 3381 // icmp pred inttoptr(X), null -> icmp pred X, 0 3382 if (RHSC->isNullValue() && 3383 DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType()) 3384 return new ICmpInst( 3385 I.getPredicate(), LHSI->getOperand(0), 3386 Constant::getNullValue(LHSI->getOperand(0)->getType())); 3387 break; 3388 3389 case Instruction::Load: 3390 // Try to optimize things like "A[i] > 4" to index computations. 3391 if (GetElementPtrInst *GEP = 3392 dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) { 3393 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) 3394 if (GV->isConstant() && GV->hasDefinitiveInitializer() && 3395 !cast<LoadInst>(LHSI)->isVolatile()) 3396 if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) 3397 return Res; 3398 } 3399 break; 3400 } 3401 3402 return nullptr; 3403 } 3404 3405 /// Some comparisons can be simplified. 3406 /// In this case, we are looking for comparisons that look like 3407 /// a check for a lossy truncation. 3408 /// Folds: 3409 /// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask 3410 /// Where Mask is some pattern that produces all-ones in low bits: 3411 /// (-1 >> y) 3412 /// ((-1 << y) >> y) <- non-canonical, has extra uses 3413 /// ~(-1 << y) 3414 /// ((1 << y) + (-1)) <- non-canonical, has extra uses 3415 /// The Mask can be a constant, too. 3416 /// For some predicates, the operands are commutative. 3417 /// For others, x can only be on a specific side. 3418 static Value *foldICmpWithLowBitMaskedVal(ICmpInst &I, 3419 InstCombiner::BuilderTy &Builder) { 3420 ICmpInst::Predicate SrcPred; 3421 Value *X, *M, *Y; 3422 auto m_VariableMask = m_CombineOr( 3423 m_CombineOr(m_Not(m_Shl(m_AllOnes(), m_Value())), 3424 m_Add(m_Shl(m_One(), m_Value()), m_AllOnes())), 3425 m_CombineOr(m_LShr(m_AllOnes(), m_Value()), 3426 m_LShr(m_Shl(m_AllOnes(), m_Value(Y)), m_Deferred(Y)))); 3427 auto m_Mask = m_CombineOr(m_VariableMask, m_LowBitMask()); 3428 if (!match(&I, m_c_ICmp(SrcPred, 3429 m_c_And(m_CombineAnd(m_Mask, m_Value(M)), m_Value(X)), 3430 m_Deferred(X)))) 3431 return nullptr; 3432 3433 ICmpInst::Predicate DstPred; 3434 switch (SrcPred) { 3435 case ICmpInst::Predicate::ICMP_EQ: 3436 // x & (-1 >> y) == x -> x u<= (-1 >> y) 3437 DstPred = ICmpInst::Predicate::ICMP_ULE; 3438 break; 3439 case ICmpInst::Predicate::ICMP_NE: 3440 // x & (-1 >> y) != x -> x u> (-1 >> y) 3441 DstPred = ICmpInst::Predicate::ICMP_UGT; 3442 break; 3443 case ICmpInst::Predicate::ICMP_ULT: 3444 // x & (-1 >> y) u< x -> x u> (-1 >> y) 3445 // x u> x & (-1 >> y) -> x u> (-1 >> y) 3446 DstPred = ICmpInst::Predicate::ICMP_UGT; 3447 break; 3448 case ICmpInst::Predicate::ICMP_UGE: 3449 // x & (-1 >> y) u>= x -> x u<= (-1 >> y) 3450 // x u<= x & (-1 >> y) -> x u<= (-1 >> y) 3451 DstPred = ICmpInst::Predicate::ICMP_ULE; 3452 break; 3453 case ICmpInst::Predicate::ICMP_SLT: 3454 // x & (-1 >> y) s< x -> x s> (-1 >> y) 3455 // x s> x & (-1 >> y) -> x s> (-1 >> y) 3456 if (!match(M, m_Constant())) // Can not do this fold with non-constant. 3457 return nullptr; 3458 if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. 3459 return nullptr; 3460 DstPred = ICmpInst::Predicate::ICMP_SGT; 3461 break; 3462 case ICmpInst::Predicate::ICMP_SGE: 3463 // x & (-1 >> y) s>= x -> x s<= (-1 >> y) 3464 // x s<= x & (-1 >> y) -> x s<= (-1 >> y) 3465 if (!match(M, m_Constant())) // Can not do this fold with non-constant. 3466 return nullptr; 3467 if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. 3468 return nullptr; 3469 DstPred = ICmpInst::Predicate::ICMP_SLE; 3470 break; 3471 case ICmpInst::Predicate::ICMP_SGT: 3472 case ICmpInst::Predicate::ICMP_SLE: 3473 return nullptr; 3474 case ICmpInst::Predicate::ICMP_UGT: 3475 case ICmpInst::Predicate::ICMP_ULE: 3476 llvm_unreachable("Instsimplify took care of commut. variant"); 3477 break; 3478 default: 3479 llvm_unreachable("All possible folds are handled."); 3480 } 3481 3482 // The mask value may be a vector constant that has undefined elements. But it 3483 // may not be safe to propagate those undefs into the new compare, so replace 3484 // those elements by copying an existing, defined, and safe scalar constant. 3485 Type *OpTy = M->getType(); 3486 auto *VecC = dyn_cast<Constant>(M); 3487 auto *OpVTy = dyn_cast<FixedVectorType>(OpTy); 3488 if (OpVTy && VecC && VecC->containsUndefOrPoisonElement()) { 3489 Constant *SafeReplacementConstant = nullptr; 3490 for (unsigned i = 0, e = OpVTy->getNumElements(); i != e; ++i) { 3491 if (!isa<UndefValue>(VecC->getAggregateElement(i))) { 3492 SafeReplacementConstant = VecC->getAggregateElement(i); 3493 break; 3494 } 3495 } 3496 assert(SafeReplacementConstant && "Failed to find undef replacement"); 3497 M = Constant::replaceUndefsWith(VecC, SafeReplacementConstant); 3498 } 3499 3500 return Builder.CreateICmp(DstPred, X, M); 3501 } 3502 3503 /// Some comparisons can be simplified. 3504 /// In this case, we are looking for comparisons that look like 3505 /// a check for a lossy signed truncation. 3506 /// Folds: (MaskedBits is a constant.) 3507 /// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x 3508 /// Into: 3509 /// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits) 3510 /// Where KeptBits = bitwidth(%x) - MaskedBits 3511 static Value * 3512 foldICmpWithTruncSignExtendedVal(ICmpInst &I, 3513 InstCombiner::BuilderTy &Builder) { 3514 ICmpInst::Predicate SrcPred; 3515 Value *X; 3516 const APInt *C0, *C1; // FIXME: non-splats, potentially with undef. 3517 // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use. 3518 if (!match(&I, m_c_ICmp(SrcPred, 3519 m_OneUse(m_AShr(m_Shl(m_Value(X), m_APInt(C0)), 3520 m_APInt(C1))), 3521 m_Deferred(X)))) 3522 return nullptr; 3523 3524 // Potential handling of non-splats: for each element: 3525 // * if both are undef, replace with constant 0. 3526 // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0. 3527 // * if both are not undef, and are different, bailout. 3528 // * else, only one is undef, then pick the non-undef one. 3529 3530 // The shift amount must be equal. 3531 if (*C0 != *C1) 3532 return nullptr; 3533 const APInt &MaskedBits = *C0; 3534 assert(MaskedBits != 0 && "shift by zero should be folded away already."); 3535 3536 ICmpInst::Predicate DstPred; 3537 switch (SrcPred) { 3538 case ICmpInst::Predicate::ICMP_EQ: 3539 // ((%x << MaskedBits) a>> MaskedBits) == %x 3540 // => 3541 // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits) 3542 DstPred = ICmpInst::Predicate::ICMP_ULT; 3543 break; 3544 case ICmpInst::Predicate::ICMP_NE: 3545 // ((%x << MaskedBits) a>> MaskedBits) != %x 3546 // => 3547 // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits) 3548 DstPred = ICmpInst::Predicate::ICMP_UGE; 3549 break; 3550 // FIXME: are more folds possible? 3551 default: 3552 return nullptr; 3553 } 3554 3555 auto *XType = X->getType(); 3556 const unsigned XBitWidth = XType->getScalarSizeInBits(); 3557 const APInt BitWidth = APInt(XBitWidth, XBitWidth); 3558 assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched"); 3559 3560 // KeptBits = bitwidth(%x) - MaskedBits 3561 const APInt KeptBits = BitWidth - MaskedBits; 3562 assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable"); 3563 // ICmpCst = (1 << KeptBits) 3564 const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits); 3565 assert(ICmpCst.isPowerOf2()); 3566 // AddCst = (1 << (KeptBits-1)) 3567 const APInt AddCst = ICmpCst.lshr(1); 3568 assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2()); 3569 3570 // T0 = add %x, AddCst 3571 Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst)); 3572 // T1 = T0 DstPred ICmpCst 3573 Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst)); 3574 3575 return T1; 3576 } 3577 3578 // Given pattern: 3579 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 3580 // we should move shifts to the same hand of 'and', i.e. rewrite as 3581 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) 3582 // We are only interested in opposite logical shifts here. 3583 // One of the shifts can be truncated. 3584 // If we can, we want to end up creating 'lshr' shift. 3585 static Value * 3586 foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, 3587 InstCombiner::BuilderTy &Builder) { 3588 if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) || 3589 !I.getOperand(0)->hasOneUse()) 3590 return nullptr; 3591 3592 auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value()); 3593 3594 // Look for an 'and' of two logical shifts, one of which may be truncated. 3595 // We use m_TruncOrSelf() on the RHS to correctly handle commutative case. 3596 Instruction *XShift, *MaybeTruncation, *YShift; 3597 if (!match( 3598 I.getOperand(0), 3599 m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)), 3600 m_CombineAnd(m_TruncOrSelf(m_CombineAnd( 3601 m_AnyLogicalShift, m_Instruction(YShift))), 3602 m_Instruction(MaybeTruncation))))) 3603 return nullptr; 3604 3605 // We potentially looked past 'trunc', but only when matching YShift, 3606 // therefore YShift must have the widest type. 3607 Instruction *WidestShift = YShift; 3608 // Therefore XShift must have the shallowest type. 3609 // Or they both have identical types if there was no truncation. 3610 Instruction *NarrowestShift = XShift; 3611 3612 Type *WidestTy = WidestShift->getType(); 3613 Type *NarrowestTy = NarrowestShift->getType(); 3614 assert(NarrowestTy == I.getOperand(0)->getType() && 3615 "We did not look past any shifts while matching XShift though."); 3616 bool HadTrunc = WidestTy != I.getOperand(0)->getType(); 3617 3618 // If YShift is a 'lshr', swap the shifts around. 3619 if (match(YShift, m_LShr(m_Value(), m_Value()))) 3620 std::swap(XShift, YShift); 3621 3622 // The shifts must be in opposite directions. 3623 auto XShiftOpcode = XShift->getOpcode(); 3624 if (XShiftOpcode == YShift->getOpcode()) 3625 return nullptr; // Do not care about same-direction shifts here. 3626 3627 Value *X, *XShAmt, *Y, *YShAmt; 3628 match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt)))); 3629 match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt)))); 3630 3631 // If one of the values being shifted is a constant, then we will end with 3632 // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not, 3633 // however, we will need to ensure that we won't increase instruction count. 3634 if (!isa<Constant>(X) && !isa<Constant>(Y)) { 3635 // At least one of the hands of the 'and' should be one-use shift. 3636 if (!match(I.getOperand(0), 3637 m_c_And(m_OneUse(m_AnyLogicalShift), m_Value()))) 3638 return nullptr; 3639 if (HadTrunc) { 3640 // Due to the 'trunc', we will need to widen X. For that either the old 3641 // 'trunc' or the shift amt in the non-truncated shift should be one-use. 3642 if (!MaybeTruncation->hasOneUse() && 3643 !NarrowestShift->getOperand(1)->hasOneUse()) 3644 return nullptr; 3645 } 3646 } 3647 3648 // We have two shift amounts from two different shifts. The types of those 3649 // shift amounts may not match. If that's the case let's bailout now. 3650 if (XShAmt->getType() != YShAmt->getType()) 3651 return nullptr; 3652 3653 // As input, we have the following pattern: 3654 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 3655 // We want to rewrite that as: 3656 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) 3657 // While we know that originally (Q+K) would not overflow 3658 // (because 2 * (N-1) u<= iN -1), we have looked past extensions of 3659 // shift amounts. so it may now overflow in smaller bitwidth. 3660 // To ensure that does not happen, we need to ensure that the total maximal 3661 // shift amount is still representable in that smaller bit width. 3662 unsigned MaximalPossibleTotalShiftAmount = 3663 (WidestTy->getScalarSizeInBits() - 1) + 3664 (NarrowestTy->getScalarSizeInBits() - 1); 3665 APInt MaximalRepresentableShiftAmount = 3666 APInt::getAllOnesValue(XShAmt->getType()->getScalarSizeInBits()); 3667 if (MaximalRepresentableShiftAmount.ult(MaximalPossibleTotalShiftAmount)) 3668 return nullptr; 3669 3670 // Can we fold (XShAmt+YShAmt) ? 3671 auto *NewShAmt = dyn_cast_or_null<Constant>( 3672 SimplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false, 3673 /*isNUW=*/false, SQ.getWithInstruction(&I))); 3674 if (!NewShAmt) 3675 return nullptr; 3676 NewShAmt = ConstantExpr::getZExtOrBitCast(NewShAmt, WidestTy); 3677 unsigned WidestBitWidth = WidestTy->getScalarSizeInBits(); 3678 3679 // Is the new shift amount smaller than the bit width? 3680 // FIXME: could also rely on ConstantRange. 3681 if (!match(NewShAmt, 3682 m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_ULT, 3683 APInt(WidestBitWidth, WidestBitWidth)))) 3684 return nullptr; 3685 3686 // An extra legality check is needed if we had trunc-of-lshr. 3687 if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) { 3688 auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ, 3689 WidestShift]() { 3690 // It isn't obvious whether it's worth it to analyze non-constants here. 3691 // Also, let's basically give up on non-splat cases, pessimizing vectors. 3692 // If *any* of these preconditions matches we can perform the fold. 3693 Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy() 3694 ? NewShAmt->getSplatValue() 3695 : NewShAmt; 3696 // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold. 3697 if (NewShAmtSplat && 3698 (NewShAmtSplat->isNullValue() || 3699 NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1)) 3700 return true; 3701 // We consider *min* leading zeros so a single outlier 3702 // blocks the transform as opposed to allowing it. 3703 if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) { 3704 KnownBits Known = computeKnownBits(C, SQ.DL); 3705 unsigned MinLeadZero = Known.countMinLeadingZeros(); 3706 // If the value being shifted has at most lowest bit set we can fold. 3707 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; 3708 if (MaxActiveBits <= 1) 3709 return true; 3710 // Precondition: NewShAmt u<= countLeadingZeros(C) 3711 if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero)) 3712 return true; 3713 } 3714 if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) { 3715 KnownBits Known = computeKnownBits(C, SQ.DL); 3716 unsigned MinLeadZero = Known.countMinLeadingZeros(); 3717 // If the value being shifted has at most lowest bit set we can fold. 3718 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; 3719 if (MaxActiveBits <= 1) 3720 return true; 3721 // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C) 3722 if (NewShAmtSplat) { 3723 APInt AdjNewShAmt = 3724 (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger(); 3725 if (AdjNewShAmt.ule(MinLeadZero)) 3726 return true; 3727 } 3728 } 3729 return false; // Can't tell if it's ok. 3730 }; 3731 if (!CanFold()) 3732 return nullptr; 3733 } 3734 3735 // All good, we can do this fold. 3736 X = Builder.CreateZExt(X, WidestTy); 3737 Y = Builder.CreateZExt(Y, WidestTy); 3738 // The shift is the same that was for X. 3739 Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr 3740 ? Builder.CreateLShr(X, NewShAmt) 3741 : Builder.CreateShl(X, NewShAmt); 3742 Value *T1 = Builder.CreateAnd(T0, Y); 3743 return Builder.CreateICmp(I.getPredicate(), T1, 3744 Constant::getNullValue(WidestTy)); 3745 } 3746 3747 /// Fold 3748 /// (-1 u/ x) u< y 3749 /// ((x * y) u/ x) != y 3750 /// to 3751 /// @llvm.umul.with.overflow(x, y) plus extraction of overflow bit 3752 /// Note that the comparison is commutative, while inverted (u>=, ==) predicate 3753 /// will mean that we are looking for the opposite answer. 3754 Value *InstCombinerImpl::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) { 3755 ICmpInst::Predicate Pred; 3756 Value *X, *Y; 3757 Instruction *Mul; 3758 bool NeedNegation; 3759 // Look for: (-1 u/ x) u</u>= y 3760 if (!I.isEquality() && 3761 match(&I, m_c_ICmp(Pred, m_OneUse(m_UDiv(m_AllOnes(), m_Value(X))), 3762 m_Value(Y)))) { 3763 Mul = nullptr; 3764 3765 // Are we checking that overflow does not happen, or does happen? 3766 switch (Pred) { 3767 case ICmpInst::Predicate::ICMP_ULT: 3768 NeedNegation = false; 3769 break; // OK 3770 case ICmpInst::Predicate::ICMP_UGE: 3771 NeedNegation = true; 3772 break; // OK 3773 default: 3774 return nullptr; // Wrong predicate. 3775 } 3776 } else // Look for: ((x * y) u/ x) !=/== y 3777 if (I.isEquality() && 3778 match(&I, m_c_ICmp(Pred, m_Value(Y), 3779 m_OneUse(m_UDiv(m_CombineAnd(m_c_Mul(m_Deferred(Y), 3780 m_Value(X)), 3781 m_Instruction(Mul)), 3782 m_Deferred(X)))))) { 3783 NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ; 3784 } else 3785 return nullptr; 3786 3787 BuilderTy::InsertPointGuard Guard(Builder); 3788 // If the pattern included (x * y), we'll want to insert new instructions 3789 // right before that original multiplication so that we can replace it. 3790 bool MulHadOtherUses = Mul && !Mul->hasOneUse(); 3791 if (MulHadOtherUses) 3792 Builder.SetInsertPoint(Mul); 3793 3794 Function *F = Intrinsic::getDeclaration( 3795 I.getModule(), Intrinsic::umul_with_overflow, X->getType()); 3796 CallInst *Call = Builder.CreateCall(F, {X, Y}, "umul"); 3797 3798 // If the multiplication was used elsewhere, to ensure that we don't leave 3799 // "duplicate" instructions, replace uses of that original multiplication 3800 // with the multiplication result from the with.overflow intrinsic. 3801 if (MulHadOtherUses) 3802 replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "umul.val")); 3803 3804 Value *Res = Builder.CreateExtractValue(Call, 1, "umul.ov"); 3805 if (NeedNegation) // This technically increases instruction count. 3806 Res = Builder.CreateNot(Res, "umul.not.ov"); 3807 3808 // If we replaced the mul, erase it. Do this after all uses of Builder, 3809 // as the mul is used as insertion point. 3810 if (MulHadOtherUses) 3811 eraseInstFromFunction(*Mul); 3812 3813 return Res; 3814 } 3815 3816 static Instruction *foldICmpXNegX(ICmpInst &I) { 3817 CmpInst::Predicate Pred; 3818 Value *X; 3819 if (!match(&I, m_c_ICmp(Pred, m_NSWNeg(m_Value(X)), m_Deferred(X)))) 3820 return nullptr; 3821 3822 if (ICmpInst::isSigned(Pred)) 3823 Pred = ICmpInst::getSwappedPredicate(Pred); 3824 else if (ICmpInst::isUnsigned(Pred)) 3825 Pred = ICmpInst::getSignedPredicate(Pred); 3826 // else for equality-comparisons just keep the predicate. 3827 3828 return ICmpInst::Create(Instruction::ICmp, Pred, X, 3829 Constant::getNullValue(X->getType()), I.getName()); 3830 } 3831 3832 /// Try to fold icmp (binop), X or icmp X, (binop). 3833 /// TODO: A large part of this logic is duplicated in InstSimplify's 3834 /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code 3835 /// duplication. 3836 Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, 3837 const SimplifyQuery &SQ) { 3838 const SimplifyQuery Q = SQ.getWithInstruction(&I); 3839 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 3840 3841 // Special logic for binary operators. 3842 BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0); 3843 BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1); 3844 if (!BO0 && !BO1) 3845 return nullptr; 3846 3847 if (Instruction *NewICmp = foldICmpXNegX(I)) 3848 return NewICmp; 3849 3850 const CmpInst::Predicate Pred = I.getPredicate(); 3851 Value *X; 3852 3853 // Convert add-with-unsigned-overflow comparisons into a 'not' with compare. 3854 // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X 3855 if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) && 3856 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) 3857 return new ICmpInst(Pred, Builder.CreateNot(Op1), X); 3858 // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0 3859 if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) && 3860 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) 3861 return new ICmpInst(Pred, X, Builder.CreateNot(Op0)); 3862 3863 bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; 3864 if (BO0 && isa<OverflowingBinaryOperator>(BO0)) 3865 NoOp0WrapProblem = 3866 ICmpInst::isEquality(Pred) || 3867 (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) || 3868 (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap()); 3869 if (BO1 && isa<OverflowingBinaryOperator>(BO1)) 3870 NoOp1WrapProblem = 3871 ICmpInst::isEquality(Pred) || 3872 (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) || 3873 (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap()); 3874 3875 // Analyze the case when either Op0 or Op1 is an add instruction. 3876 // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). 3877 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; 3878 if (BO0 && BO0->getOpcode() == Instruction::Add) { 3879 A = BO0->getOperand(0); 3880 B = BO0->getOperand(1); 3881 } 3882 if (BO1 && BO1->getOpcode() == Instruction::Add) { 3883 C = BO1->getOperand(0); 3884 D = BO1->getOperand(1); 3885 } 3886 3887 // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow. 3888 // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow. 3889 if ((A == Op1 || B == Op1) && NoOp0WrapProblem) 3890 return new ICmpInst(Pred, A == Op1 ? B : A, 3891 Constant::getNullValue(Op1->getType())); 3892 3893 // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow. 3894 // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow. 3895 if ((C == Op0 || D == Op0) && NoOp1WrapProblem) 3896 return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()), 3897 C == Op0 ? D : C); 3898 3899 // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow. 3900 if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem && 3901 NoOp1WrapProblem) { 3902 // Determine Y and Z in the form icmp (X+Y), (X+Z). 3903 Value *Y, *Z; 3904 if (A == C) { 3905 // C + B == C + D -> B == D 3906 Y = B; 3907 Z = D; 3908 } else if (A == D) { 3909 // D + B == C + D -> B == C 3910 Y = B; 3911 Z = C; 3912 } else if (B == C) { 3913 // A + C == C + D -> A == D 3914 Y = A; 3915 Z = D; 3916 } else { 3917 assert(B == D); 3918 // A + D == C + D -> A == C 3919 Y = A; 3920 Z = C; 3921 } 3922 return new ICmpInst(Pred, Y, Z); 3923 } 3924 3925 // icmp slt (A + -1), Op1 -> icmp sle A, Op1 3926 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT && 3927 match(B, m_AllOnes())) 3928 return new ICmpInst(CmpInst::ICMP_SLE, A, Op1); 3929 3930 // icmp sge (A + -1), Op1 -> icmp sgt A, Op1 3931 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE && 3932 match(B, m_AllOnes())) 3933 return new ICmpInst(CmpInst::ICMP_SGT, A, Op1); 3934 3935 // icmp sle (A + 1), Op1 -> icmp slt A, Op1 3936 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One())) 3937 return new ICmpInst(CmpInst::ICMP_SLT, A, Op1); 3938 3939 // icmp sgt (A + 1), Op1 -> icmp sge A, Op1 3940 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One())) 3941 return new ICmpInst(CmpInst::ICMP_SGE, A, Op1); 3942 3943 // icmp sgt Op0, (C + -1) -> icmp sge Op0, C 3944 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT && 3945 match(D, m_AllOnes())) 3946 return new ICmpInst(CmpInst::ICMP_SGE, Op0, C); 3947 3948 // icmp sle Op0, (C + -1) -> icmp slt Op0, C 3949 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE && 3950 match(D, m_AllOnes())) 3951 return new ICmpInst(CmpInst::ICMP_SLT, Op0, C); 3952 3953 // icmp sge Op0, (C + 1) -> icmp sgt Op0, C 3954 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One())) 3955 return new ICmpInst(CmpInst::ICMP_SGT, Op0, C); 3956 3957 // icmp slt Op0, (C + 1) -> icmp sle Op0, C 3958 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One())) 3959 return new ICmpInst(CmpInst::ICMP_SLE, Op0, C); 3960 3961 // TODO: The subtraction-related identities shown below also hold, but 3962 // canonicalization from (X -nuw 1) to (X + -1) means that the combinations 3963 // wouldn't happen even if they were implemented. 3964 // 3965 // icmp ult (A - 1), Op1 -> icmp ule A, Op1 3966 // icmp uge (A - 1), Op1 -> icmp ugt A, Op1 3967 // icmp ugt Op0, (C - 1) -> icmp uge Op0, C 3968 // icmp ule Op0, (C - 1) -> icmp ult Op0, C 3969 3970 // icmp ule (A + 1), Op0 -> icmp ult A, Op1 3971 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One())) 3972 return new ICmpInst(CmpInst::ICMP_ULT, A, Op1); 3973 3974 // icmp ugt (A + 1), Op0 -> icmp uge A, Op1 3975 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One())) 3976 return new ICmpInst(CmpInst::ICMP_UGE, A, Op1); 3977 3978 // icmp uge Op0, (C + 1) -> icmp ugt Op0, C 3979 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One())) 3980 return new ICmpInst(CmpInst::ICMP_UGT, Op0, C); 3981 3982 // icmp ult Op0, (C + 1) -> icmp ule Op0, C 3983 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One())) 3984 return new ICmpInst(CmpInst::ICMP_ULE, Op0, C); 3985 3986 // if C1 has greater magnitude than C2: 3987 // icmp (A + C1), (C + C2) -> icmp (A + C3), C 3988 // s.t. C3 = C1 - C2 3989 // 3990 // if C2 has greater magnitude than C1: 3991 // icmp (A + C1), (C + C2) -> icmp A, (C + C3) 3992 // s.t. C3 = C2 - C1 3993 if (A && C && NoOp0WrapProblem && NoOp1WrapProblem && 3994 (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) 3995 if (ConstantInt *C1 = dyn_cast<ConstantInt>(B)) 3996 if (ConstantInt *C2 = dyn_cast<ConstantInt>(D)) { 3997 const APInt &AP1 = C1->getValue(); 3998 const APInt &AP2 = C2->getValue(); 3999 if (AP1.isNegative() == AP2.isNegative()) { 4000 APInt AP1Abs = C1->getValue().abs(); 4001 APInt AP2Abs = C2->getValue().abs(); 4002 if (AP1Abs.uge(AP2Abs)) { 4003 ConstantInt *C3 = Builder.getInt(AP1 - AP2); 4004 bool HasNUW = BO0->hasNoUnsignedWrap() && C3->getValue().ule(AP1); 4005 bool HasNSW = BO0->hasNoSignedWrap(); 4006 Value *NewAdd = Builder.CreateAdd(A, C3, "", HasNUW, HasNSW); 4007 return new ICmpInst(Pred, NewAdd, C); 4008 } else { 4009 ConstantInt *C3 = Builder.getInt(AP2 - AP1); 4010 bool HasNUW = BO1->hasNoUnsignedWrap() && C3->getValue().ule(AP2); 4011 bool HasNSW = BO1->hasNoSignedWrap(); 4012 Value *NewAdd = Builder.CreateAdd(C, C3, "", HasNUW, HasNSW); 4013 return new ICmpInst(Pred, A, NewAdd); 4014 } 4015 } 4016 } 4017 4018 // Analyze the case when either Op0 or Op1 is a sub instruction. 4019 // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null). 4020 A = nullptr; 4021 B = nullptr; 4022 C = nullptr; 4023 D = nullptr; 4024 if (BO0 && BO0->getOpcode() == Instruction::Sub) { 4025 A = BO0->getOperand(0); 4026 B = BO0->getOperand(1); 4027 } 4028 if (BO1 && BO1->getOpcode() == Instruction::Sub) { 4029 C = BO1->getOperand(0); 4030 D = BO1->getOperand(1); 4031 } 4032 4033 // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow. 4034 if (A == Op1 && NoOp0WrapProblem) 4035 return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B); 4036 // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow. 4037 if (C == Op0 && NoOp1WrapProblem) 4038 return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType())); 4039 4040 // Convert sub-with-unsigned-overflow comparisons into a comparison of args. 4041 // (A - B) u>/u<= A --> B u>/u<= A 4042 if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) 4043 return new ICmpInst(Pred, B, A); 4044 // C u</u>= (C - D) --> C u</u>= D 4045 if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) 4046 return new ICmpInst(Pred, C, D); 4047 // (A - B) u>=/u< A --> B u>/u<= A iff B != 0 4048 if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) && 4049 isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) 4050 return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A); 4051 // C u<=/u> (C - D) --> C u</u>= D iff B != 0 4052 if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) && 4053 isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) 4054 return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D); 4055 4056 // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow. 4057 if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem) 4058 return new ICmpInst(Pred, A, C); 4059 4060 // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow. 4061 if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem) 4062 return new ICmpInst(Pred, D, B); 4063 4064 // icmp (0-X) < cst --> x > -cst 4065 if (NoOp0WrapProblem && ICmpInst::isSigned(Pred)) { 4066 Value *X; 4067 if (match(BO0, m_Neg(m_Value(X)))) 4068 if (Constant *RHSC = dyn_cast<Constant>(Op1)) 4069 if (RHSC->isNotMinSignedValue()) 4070 return new ICmpInst(I.getSwappedPredicate(), X, 4071 ConstantExpr::getNeg(RHSC)); 4072 } 4073 4074 { 4075 // Try to remove shared constant multiplier from equality comparison: 4076 // X * C == Y * C (with no overflowing/aliasing) --> X == Y 4077 Value *X, *Y; 4078 const APInt *C; 4079 if (match(Op0, m_Mul(m_Value(X), m_APInt(C))) && *C != 0 && 4080 match(Op1, m_Mul(m_Value(Y), m_SpecificInt(*C))) && I.isEquality()) 4081 if (!C->countTrailingZeros() || 4082 (BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap()) || 4083 (BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap())) 4084 return new ICmpInst(Pred, X, Y); 4085 } 4086 4087 BinaryOperator *SRem = nullptr; 4088 // icmp (srem X, Y), Y 4089 if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(1)) 4090 SRem = BO0; 4091 // icmp Y, (srem X, Y) 4092 else if (BO1 && BO1->getOpcode() == Instruction::SRem && 4093 Op0 == BO1->getOperand(1)) 4094 SRem = BO1; 4095 if (SRem) { 4096 // We don't check hasOneUse to avoid increasing register pressure because 4097 // the value we use is the same value this instruction was already using. 4098 switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) { 4099 default: 4100 break; 4101 case ICmpInst::ICMP_EQ: 4102 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 4103 case ICmpInst::ICMP_NE: 4104 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 4105 case ICmpInst::ICMP_SGT: 4106 case ICmpInst::ICMP_SGE: 4107 return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1), 4108 Constant::getAllOnesValue(SRem->getType())); 4109 case ICmpInst::ICMP_SLT: 4110 case ICmpInst::ICMP_SLE: 4111 return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1), 4112 Constant::getNullValue(SRem->getType())); 4113 } 4114 } 4115 4116 if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() && BO0->hasOneUse() && 4117 BO1->hasOneUse() && BO0->getOperand(1) == BO1->getOperand(1)) { 4118 switch (BO0->getOpcode()) { 4119 default: 4120 break; 4121 case Instruction::Add: 4122 case Instruction::Sub: 4123 case Instruction::Xor: { 4124 if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b 4125 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4126 4127 const APInt *C; 4128 if (match(BO0->getOperand(1), m_APInt(C))) { 4129 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b 4130 if (C->isSignMask()) { 4131 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); 4132 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); 4133 } 4134 4135 // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b 4136 if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) { 4137 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate(); 4138 NewPred = I.getSwappedPredicate(NewPred); 4139 return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); 4140 } 4141 } 4142 break; 4143 } 4144 case Instruction::Mul: { 4145 if (!I.isEquality()) 4146 break; 4147 4148 const APInt *C; 4149 if (match(BO0->getOperand(1), m_APInt(C)) && !C->isNullValue() && 4150 !C->isOneValue()) { 4151 // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask) 4152 // Mask = -1 >> count-trailing-zeros(C). 4153 if (unsigned TZs = C->countTrailingZeros()) { 4154 Constant *Mask = ConstantInt::get( 4155 BO0->getType(), 4156 APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs)); 4157 Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask); 4158 Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask); 4159 return new ICmpInst(Pred, And1, And2); 4160 } 4161 } 4162 break; 4163 } 4164 case Instruction::UDiv: 4165 case Instruction::LShr: 4166 if (I.isSigned() || !BO0->isExact() || !BO1->isExact()) 4167 break; 4168 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4169 4170 case Instruction::SDiv: 4171 if (!I.isEquality() || !BO0->isExact() || !BO1->isExact()) 4172 break; 4173 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4174 4175 case Instruction::AShr: 4176 if (!BO0->isExact() || !BO1->isExact()) 4177 break; 4178 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4179 4180 case Instruction::Shl: { 4181 bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap(); 4182 bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap(); 4183 if (!NUW && !NSW) 4184 break; 4185 if (!NSW && I.isSigned()) 4186 break; 4187 return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); 4188 } 4189 } 4190 } 4191 4192 if (BO0) { 4193 // Transform A & (L - 1) `ult` L --> L != 0 4194 auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes()); 4195 auto BitwiseAnd = m_c_And(m_Value(), LSubOne); 4196 4197 if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) { 4198 auto *Zero = Constant::getNullValue(BO0->getType()); 4199 return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero); 4200 } 4201 } 4202 4203 if (Value *V = foldUnsignedMultiplicationOverflowCheck(I)) 4204 return replaceInstUsesWith(I, V); 4205 4206 if (Value *V = foldICmpWithLowBitMaskedVal(I, Builder)) 4207 return replaceInstUsesWith(I, V); 4208 4209 if (Value *V = foldICmpWithTruncSignExtendedVal(I, Builder)) 4210 return replaceInstUsesWith(I, V); 4211 4212 if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder)) 4213 return replaceInstUsesWith(I, V); 4214 4215 return nullptr; 4216 } 4217 4218 /// Fold icmp Pred min|max(X, Y), X. 4219 static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) { 4220 ICmpInst::Predicate Pred = Cmp.getPredicate(); 4221 Value *Op0 = Cmp.getOperand(0); 4222 Value *X = Cmp.getOperand(1); 4223 4224 // Canonicalize minimum or maximum operand to LHS of the icmp. 4225 if (match(X, m_c_SMin(m_Specific(Op0), m_Value())) || 4226 match(X, m_c_SMax(m_Specific(Op0), m_Value())) || 4227 match(X, m_c_UMin(m_Specific(Op0), m_Value())) || 4228 match(X, m_c_UMax(m_Specific(Op0), m_Value()))) { 4229 std::swap(Op0, X); 4230 Pred = Cmp.getSwappedPredicate(); 4231 } 4232 4233 Value *Y; 4234 if (match(Op0, m_c_SMin(m_Specific(X), m_Value(Y)))) { 4235 // smin(X, Y) == X --> X s<= Y 4236 // smin(X, Y) s>= X --> X s<= Y 4237 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SGE) 4238 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); 4239 4240 // smin(X, Y) != X --> X s> Y 4241 // smin(X, Y) s< X --> X s> Y 4242 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SLT) 4243 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); 4244 4245 // These cases should be handled in InstSimplify: 4246 // smin(X, Y) s<= X --> true 4247 // smin(X, Y) s> X --> false 4248 return nullptr; 4249 } 4250 4251 if (match(Op0, m_c_SMax(m_Specific(X), m_Value(Y)))) { 4252 // smax(X, Y) == X --> X s>= Y 4253 // smax(X, Y) s<= X --> X s>= Y 4254 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLE) 4255 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); 4256 4257 // smax(X, Y) != X --> X s< Y 4258 // smax(X, Y) s> X --> X s< Y 4259 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SGT) 4260 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); 4261 4262 // These cases should be handled in InstSimplify: 4263 // smax(X, Y) s>= X --> true 4264 // smax(X, Y) s< X --> false 4265 return nullptr; 4266 } 4267 4268 if (match(Op0, m_c_UMin(m_Specific(X), m_Value(Y)))) { 4269 // umin(X, Y) == X --> X u<= Y 4270 // umin(X, Y) u>= X --> X u<= Y 4271 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_UGE) 4272 return new ICmpInst(ICmpInst::ICMP_ULE, X, Y); 4273 4274 // umin(X, Y) != X --> X u> Y 4275 // umin(X, Y) u< X --> X u> Y 4276 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT) 4277 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); 4278 4279 // These cases should be handled in InstSimplify: 4280 // umin(X, Y) u<= X --> true 4281 // umin(X, Y) u> X --> false 4282 return nullptr; 4283 } 4284 4285 if (match(Op0, m_c_UMax(m_Specific(X), m_Value(Y)))) { 4286 // umax(X, Y) == X --> X u>= Y 4287 // umax(X, Y) u<= X --> X u>= Y 4288 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_ULE) 4289 return new ICmpInst(ICmpInst::ICMP_UGE, X, Y); 4290 4291 // umax(X, Y) != X --> X u< Y 4292 // umax(X, Y) u> X --> X u< Y 4293 if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_UGT) 4294 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); 4295 4296 // These cases should be handled in InstSimplify: 4297 // umax(X, Y) u>= X --> true 4298 // umax(X, Y) u< X --> false 4299 return nullptr; 4300 } 4301 4302 return nullptr; 4303 } 4304 4305 Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) { 4306 if (!I.isEquality()) 4307 return nullptr; 4308 4309 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 4310 const CmpInst::Predicate Pred = I.getPredicate(); 4311 Value *A, *B, *C, *D; 4312 if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) { 4313 if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0 4314 Value *OtherVal = A == Op1 ? B : A; 4315 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); 4316 } 4317 4318 if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) { 4319 // A^c1 == C^c2 --> A == C^(c1^c2) 4320 ConstantInt *C1, *C2; 4321 if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) && 4322 Op1->hasOneUse()) { 4323 Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue()); 4324 Value *Xor = Builder.CreateXor(C, NC); 4325 return new ICmpInst(Pred, A, Xor); 4326 } 4327 4328 // A^B == A^D -> B == D 4329 if (A == C) 4330 return new ICmpInst(Pred, B, D); 4331 if (A == D) 4332 return new ICmpInst(Pred, B, C); 4333 if (B == C) 4334 return new ICmpInst(Pred, A, D); 4335 if (B == D) 4336 return new ICmpInst(Pred, A, C); 4337 } 4338 } 4339 4340 if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) { 4341 // A == (A^B) -> B == 0 4342 Value *OtherVal = A == Op0 ? B : A; 4343 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); 4344 } 4345 4346 // (X&Z) == (Y&Z) -> (X^Y) & Z == 0 4347 if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) && 4348 match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) { 4349 Value *X = nullptr, *Y = nullptr, *Z = nullptr; 4350 4351 if (A == C) { 4352 X = B; 4353 Y = D; 4354 Z = A; 4355 } else if (A == D) { 4356 X = B; 4357 Y = C; 4358 Z = A; 4359 } else if (B == C) { 4360 X = A; 4361 Y = D; 4362 Z = B; 4363 } else if (B == D) { 4364 X = A; 4365 Y = C; 4366 Z = B; 4367 } 4368 4369 if (X) { // Build (X^Y) & Z 4370 Op1 = Builder.CreateXor(X, Y); 4371 Op1 = Builder.CreateAnd(Op1, Z); 4372 return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType())); 4373 } 4374 } 4375 4376 // Transform (zext A) == (B & (1<<X)-1) --> A == (trunc B) 4377 // and (B & (1<<X)-1) == (zext A) --> A == (trunc B) 4378 ConstantInt *Cst1; 4379 if ((Op0->hasOneUse() && match(Op0, m_ZExt(m_Value(A))) && 4380 match(Op1, m_And(m_Value(B), m_ConstantInt(Cst1)))) || 4381 (Op1->hasOneUse() && match(Op0, m_And(m_Value(B), m_ConstantInt(Cst1))) && 4382 match(Op1, m_ZExt(m_Value(A))))) { 4383 APInt Pow2 = Cst1->getValue() + 1; 4384 if (Pow2.isPowerOf2() && isa<IntegerType>(A->getType()) && 4385 Pow2.logBase2() == cast<IntegerType>(A->getType())->getBitWidth()) 4386 return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType())); 4387 } 4388 4389 // (A >> C) == (B >> C) --> (A^B) u< (1 << C) 4390 // For lshr and ashr pairs. 4391 if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_ConstantInt(Cst1)))) && 4392 match(Op1, m_OneUse(m_LShr(m_Value(B), m_Specific(Cst1))))) || 4393 (match(Op0, m_OneUse(m_AShr(m_Value(A), m_ConstantInt(Cst1)))) && 4394 match(Op1, m_OneUse(m_AShr(m_Value(B), m_Specific(Cst1)))))) { 4395 unsigned TypeBits = Cst1->getBitWidth(); 4396 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); 4397 if (ShAmt < TypeBits && ShAmt != 0) { 4398 ICmpInst::Predicate NewPred = 4399 Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; 4400 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); 4401 APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt); 4402 return new ICmpInst(NewPred, Xor, Builder.getInt(CmpVal)); 4403 } 4404 } 4405 4406 // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0 4407 if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) && 4408 match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) { 4409 unsigned TypeBits = Cst1->getBitWidth(); 4410 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); 4411 if (ShAmt < TypeBits && ShAmt != 0) { 4412 Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); 4413 APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt); 4414 Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal), 4415 I.getName() + ".mask"); 4416 return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType())); 4417 } 4418 } 4419 4420 // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to 4421 // "icmp (and X, mask), cst" 4422 uint64_t ShAmt = 0; 4423 if (Op0->hasOneUse() && 4424 match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) && 4425 match(Op1, m_ConstantInt(Cst1)) && 4426 // Only do this when A has multiple uses. This is most important to do 4427 // when it exposes other optimizations. 4428 !A->hasOneUse()) { 4429 unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits(); 4430 4431 if (ShAmt < ASize) { 4432 APInt MaskV = 4433 APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits()); 4434 MaskV <<= ShAmt; 4435 4436 APInt CmpV = Cst1->getValue().zext(ASize); 4437 CmpV <<= ShAmt; 4438 4439 Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV)); 4440 return new ICmpInst(Pred, Mask, Builder.getInt(CmpV)); 4441 } 4442 } 4443 4444 // If both operands are byte-swapped or bit-reversed, just compare the 4445 // original values. 4446 // TODO: Move this to a function similar to foldICmpIntrinsicWithConstant() 4447 // and handle more intrinsics. 4448 if ((match(Op0, m_BSwap(m_Value(A))) && match(Op1, m_BSwap(m_Value(B)))) || 4449 (match(Op0, m_BitReverse(m_Value(A))) && 4450 match(Op1, m_BitReverse(m_Value(B))))) 4451 return new ICmpInst(Pred, A, B); 4452 4453 // Canonicalize checking for a power-of-2-or-zero value: 4454 // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants) 4455 // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants) 4456 if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()), 4457 m_Deferred(A)))) || 4458 !match(Op1, m_ZeroInt())) 4459 A = nullptr; 4460 4461 // (A & -A) == A --> ctpop(A) < 2 (four commuted variants) 4462 // (-A & A) != A --> ctpop(A) > 1 (four commuted variants) 4463 if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1))))) 4464 A = Op1; 4465 else if (match(Op1, 4466 m_OneUse(m_c_And(m_Neg(m_Specific(Op0)), m_Specific(Op0))))) 4467 A = Op0; 4468 4469 if (A) { 4470 Type *Ty = A->getType(); 4471 CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A); 4472 return Pred == ICmpInst::ICMP_EQ 4473 ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop, ConstantInt::get(Ty, 2)) 4474 : new ICmpInst(ICmpInst::ICMP_UGT, CtPop, ConstantInt::get(Ty, 1)); 4475 } 4476 4477 return nullptr; 4478 } 4479 4480 static Instruction *foldICmpWithZextOrSext(ICmpInst &ICmp, 4481 InstCombiner::BuilderTy &Builder) { 4482 assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0"); 4483 auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0)); 4484 Value *X; 4485 if (!match(CastOp0, m_ZExtOrSExt(m_Value(X)))) 4486 return nullptr; 4487 4488 bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt; 4489 bool IsSignedCmp = ICmp.isSigned(); 4490 if (auto *CastOp1 = dyn_cast<CastInst>(ICmp.getOperand(1))) { 4491 // If the signedness of the two casts doesn't agree (i.e. one is a sext 4492 // and the other is a zext), then we can't handle this. 4493 // TODO: This is too strict. We can handle some predicates (equality?). 4494 if (CastOp0->getOpcode() != CastOp1->getOpcode()) 4495 return nullptr; 4496 4497 // Not an extension from the same type? 4498 Value *Y = CastOp1->getOperand(0); 4499 Type *XTy = X->getType(), *YTy = Y->getType(); 4500 if (XTy != YTy) { 4501 // One of the casts must have one use because we are creating a new cast. 4502 if (!CastOp0->hasOneUse() && !CastOp1->hasOneUse()) 4503 return nullptr; 4504 // Extend the narrower operand to the type of the wider operand. 4505 if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits()) 4506 X = Builder.CreateCast(CastOp0->getOpcode(), X, YTy); 4507 else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits()) 4508 Y = Builder.CreateCast(CastOp0->getOpcode(), Y, XTy); 4509 else 4510 return nullptr; 4511 } 4512 4513 // (zext X) == (zext Y) --> X == Y 4514 // (sext X) == (sext Y) --> X == Y 4515 if (ICmp.isEquality()) 4516 return new ICmpInst(ICmp.getPredicate(), X, Y); 4517 4518 // A signed comparison of sign extended values simplifies into a 4519 // signed comparison. 4520 if (IsSignedCmp && IsSignedExt) 4521 return new ICmpInst(ICmp.getPredicate(), X, Y); 4522 4523 // The other three cases all fold into an unsigned comparison. 4524 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y); 4525 } 4526 4527 // Below here, we are only folding a compare with constant. 4528 auto *C = dyn_cast<Constant>(ICmp.getOperand(1)); 4529 if (!C) 4530 return nullptr; 4531 4532 // Compute the constant that would happen if we truncated to SrcTy then 4533 // re-extended to DestTy. 4534 Type *SrcTy = CastOp0->getSrcTy(); 4535 Type *DestTy = CastOp0->getDestTy(); 4536 Constant *Res1 = ConstantExpr::getTrunc(C, SrcTy); 4537 Constant *Res2 = ConstantExpr::getCast(CastOp0->getOpcode(), Res1, DestTy); 4538 4539 // If the re-extended constant didn't change... 4540 if (Res2 == C) { 4541 if (ICmp.isEquality()) 4542 return new ICmpInst(ICmp.getPredicate(), X, Res1); 4543 4544 // A signed comparison of sign extended values simplifies into a 4545 // signed comparison. 4546 if (IsSignedExt && IsSignedCmp) 4547 return new ICmpInst(ICmp.getPredicate(), X, Res1); 4548 4549 // The other three cases all fold into an unsigned comparison. 4550 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res1); 4551 } 4552 4553 // The re-extended constant changed, partly changed (in the case of a vector), 4554 // or could not be determined to be equal (in the case of a constant 4555 // expression), so the constant cannot be represented in the shorter type. 4556 // All the cases that fold to true or false will have already been handled 4557 // by SimplifyICmpInst, so only deal with the tricky case. 4558 if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C)) 4559 return nullptr; 4560 4561 // Is source op positive? 4562 // icmp ult (sext X), C --> icmp sgt X, -1 4563 if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) 4564 return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(SrcTy)); 4565 4566 // Is source op negative? 4567 // icmp ugt (sext X), C --> icmp slt X, 0 4568 assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!"); 4569 return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(SrcTy)); 4570 } 4571 4572 /// Handle icmp (cast x), (cast or constant). 4573 Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) { 4574 // If any operand of ICmp is a inttoptr roundtrip cast then remove it as 4575 // icmp compares only pointer's value. 4576 // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2. 4577 Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(0)); 4578 Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(ICmp.getOperand(1)); 4579 if (SimplifiedOp0 || SimplifiedOp1) 4580 return new ICmpInst(ICmp.getPredicate(), 4581 SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(0), 4582 SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(1)); 4583 4584 auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0)); 4585 if (!CastOp0) 4586 return nullptr; 4587 if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1))) 4588 return nullptr; 4589 4590 Value *Op0Src = CastOp0->getOperand(0); 4591 Type *SrcTy = CastOp0->getSrcTy(); 4592 Type *DestTy = CastOp0->getDestTy(); 4593 4594 // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the 4595 // integer type is the same size as the pointer type. 4596 auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) { 4597 if (isa<VectorType>(SrcTy)) { 4598 SrcTy = cast<VectorType>(SrcTy)->getElementType(); 4599 DestTy = cast<VectorType>(DestTy)->getElementType(); 4600 } 4601 return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth(); 4602 }; 4603 if (CastOp0->getOpcode() == Instruction::PtrToInt && 4604 CompatibleSizes(SrcTy, DestTy)) { 4605 Value *NewOp1 = nullptr; 4606 if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) { 4607 Value *PtrSrc = PtrToIntOp1->getOperand(0); 4608 if (PtrSrc->getType()->getPointerAddressSpace() == 4609 Op0Src->getType()->getPointerAddressSpace()) { 4610 NewOp1 = PtrToIntOp1->getOperand(0); 4611 // If the pointer types don't match, insert a bitcast. 4612 if (Op0Src->getType() != NewOp1->getType()) 4613 NewOp1 = Builder.CreateBitCast(NewOp1, Op0Src->getType()); 4614 } 4615 } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) { 4616 NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy); 4617 } 4618 4619 if (NewOp1) 4620 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1); 4621 } 4622 4623 return foldICmpWithZextOrSext(ICmp, Builder); 4624 } 4625 4626 static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS) { 4627 switch (BinaryOp) { 4628 default: 4629 llvm_unreachable("Unsupported binary op"); 4630 case Instruction::Add: 4631 case Instruction::Sub: 4632 return match(RHS, m_Zero()); 4633 case Instruction::Mul: 4634 return match(RHS, m_One()); 4635 } 4636 } 4637 4638 OverflowResult 4639 InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp, 4640 bool IsSigned, Value *LHS, Value *RHS, 4641 Instruction *CxtI) const { 4642 switch (BinaryOp) { 4643 default: 4644 llvm_unreachable("Unsupported binary op"); 4645 case Instruction::Add: 4646 if (IsSigned) 4647 return computeOverflowForSignedAdd(LHS, RHS, CxtI); 4648 else 4649 return computeOverflowForUnsignedAdd(LHS, RHS, CxtI); 4650 case Instruction::Sub: 4651 if (IsSigned) 4652 return computeOverflowForSignedSub(LHS, RHS, CxtI); 4653 else 4654 return computeOverflowForUnsignedSub(LHS, RHS, CxtI); 4655 case Instruction::Mul: 4656 if (IsSigned) 4657 return computeOverflowForSignedMul(LHS, RHS, CxtI); 4658 else 4659 return computeOverflowForUnsignedMul(LHS, RHS, CxtI); 4660 } 4661 } 4662 4663 bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp, 4664 bool IsSigned, Value *LHS, 4665 Value *RHS, Instruction &OrigI, 4666 Value *&Result, 4667 Constant *&Overflow) { 4668 if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS)) 4669 std::swap(LHS, RHS); 4670 4671 // If the overflow check was an add followed by a compare, the insertion point 4672 // may be pointing to the compare. We want to insert the new instructions 4673 // before the add in case there are uses of the add between the add and the 4674 // compare. 4675 Builder.SetInsertPoint(&OrigI); 4676 4677 Type *OverflowTy = Type::getInt1Ty(LHS->getContext()); 4678 if (auto *LHSTy = dyn_cast<VectorType>(LHS->getType())) 4679 OverflowTy = VectorType::get(OverflowTy, LHSTy->getElementCount()); 4680 4681 if (isNeutralValue(BinaryOp, RHS)) { 4682 Result = LHS; 4683 Overflow = ConstantInt::getFalse(OverflowTy); 4684 return true; 4685 } 4686 4687 switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) { 4688 case OverflowResult::MayOverflow: 4689 return false; 4690 case OverflowResult::AlwaysOverflowsLow: 4691 case OverflowResult::AlwaysOverflowsHigh: 4692 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); 4693 Result->takeName(&OrigI); 4694 Overflow = ConstantInt::getTrue(OverflowTy); 4695 return true; 4696 case OverflowResult::NeverOverflows: 4697 Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); 4698 Result->takeName(&OrigI); 4699 Overflow = ConstantInt::getFalse(OverflowTy); 4700 if (auto *Inst = dyn_cast<Instruction>(Result)) { 4701 if (IsSigned) 4702 Inst->setHasNoSignedWrap(); 4703 else 4704 Inst->setHasNoUnsignedWrap(); 4705 } 4706 return true; 4707 } 4708 4709 llvm_unreachable("Unexpected overflow result"); 4710 } 4711 4712 /// Recognize and process idiom involving test for multiplication 4713 /// overflow. 4714 /// 4715 /// The caller has matched a pattern of the form: 4716 /// I = cmp u (mul(zext A, zext B), V 4717 /// The function checks if this is a test for overflow and if so replaces 4718 /// multiplication with call to 'mul.with.overflow' intrinsic. 4719 /// 4720 /// \param I Compare instruction. 4721 /// \param MulVal Result of 'mult' instruction. It is one of the arguments of 4722 /// the compare instruction. Must be of integer type. 4723 /// \param OtherVal The other argument of compare instruction. 4724 /// \returns Instruction which must replace the compare instruction, NULL if no 4725 /// replacement required. 4726 static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, 4727 Value *OtherVal, 4728 InstCombinerImpl &IC) { 4729 // Don't bother doing this transformation for pointers, don't do it for 4730 // vectors. 4731 if (!isa<IntegerType>(MulVal->getType())) 4732 return nullptr; 4733 4734 assert(I.getOperand(0) == MulVal || I.getOperand(1) == MulVal); 4735 assert(I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal); 4736 auto *MulInstr = dyn_cast<Instruction>(MulVal); 4737 if (!MulInstr) 4738 return nullptr; 4739 assert(MulInstr->getOpcode() == Instruction::Mul); 4740 4741 auto *LHS = cast<ZExtOperator>(MulInstr->getOperand(0)), 4742 *RHS = cast<ZExtOperator>(MulInstr->getOperand(1)); 4743 assert(LHS->getOpcode() == Instruction::ZExt); 4744 assert(RHS->getOpcode() == Instruction::ZExt); 4745 Value *A = LHS->getOperand(0), *B = RHS->getOperand(0); 4746 4747 // Calculate type and width of the result produced by mul.with.overflow. 4748 Type *TyA = A->getType(), *TyB = B->getType(); 4749 unsigned WidthA = TyA->getPrimitiveSizeInBits(), 4750 WidthB = TyB->getPrimitiveSizeInBits(); 4751 unsigned MulWidth; 4752 Type *MulType; 4753 if (WidthB > WidthA) { 4754 MulWidth = WidthB; 4755 MulType = TyB; 4756 } else { 4757 MulWidth = WidthA; 4758 MulType = TyA; 4759 } 4760 4761 // In order to replace the original mul with a narrower mul.with.overflow, 4762 // all uses must ignore upper bits of the product. The number of used low 4763 // bits must be not greater than the width of mul.with.overflow. 4764 if (MulVal->hasNUsesOrMore(2)) 4765 for (User *U : MulVal->users()) { 4766 if (U == &I) 4767 continue; 4768 if (TruncInst *TI = dyn_cast<TruncInst>(U)) { 4769 // Check if truncation ignores bits above MulWidth. 4770 unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits(); 4771 if (TruncWidth > MulWidth) 4772 return nullptr; 4773 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { 4774 // Check if AND ignores bits above MulWidth. 4775 if (BO->getOpcode() != Instruction::And) 4776 return nullptr; 4777 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) { 4778 const APInt &CVal = CI->getValue(); 4779 if (CVal.getBitWidth() - CVal.countLeadingZeros() > MulWidth) 4780 return nullptr; 4781 } else { 4782 // In this case we could have the operand of the binary operation 4783 // being defined in another block, and performing the replacement 4784 // could break the dominance relation. 4785 return nullptr; 4786 } 4787 } else { 4788 // Other uses prohibit this transformation. 4789 return nullptr; 4790 } 4791 } 4792 4793 // Recognize patterns 4794 switch (I.getPredicate()) { 4795 case ICmpInst::ICMP_EQ: 4796 case ICmpInst::ICMP_NE: 4797 // Recognize pattern: 4798 // mulval = mul(zext A, zext B) 4799 // cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits. 4800 ConstantInt *CI; 4801 Value *ValToMask; 4802 if (match(OtherVal, m_And(m_Value(ValToMask), m_ConstantInt(CI)))) { 4803 if (ValToMask != MulVal) 4804 return nullptr; 4805 const APInt &CVal = CI->getValue() + 1; 4806 if (CVal.isPowerOf2()) { 4807 unsigned MaskWidth = CVal.logBase2(); 4808 if (MaskWidth == MulWidth) 4809 break; // Recognized 4810 } 4811 } 4812 return nullptr; 4813 4814 case ICmpInst::ICMP_UGT: 4815 // Recognize pattern: 4816 // mulval = mul(zext A, zext B) 4817 // cmp ugt mulval, max 4818 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 4819 APInt MaxVal = APInt::getMaxValue(MulWidth); 4820 MaxVal = MaxVal.zext(CI->getBitWidth()); 4821 if (MaxVal.eq(CI->getValue())) 4822 break; // Recognized 4823 } 4824 return nullptr; 4825 4826 case ICmpInst::ICMP_UGE: 4827 // Recognize pattern: 4828 // mulval = mul(zext A, zext B) 4829 // cmp uge mulval, max+1 4830 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 4831 APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); 4832 if (MaxVal.eq(CI->getValue())) 4833 break; // Recognized 4834 } 4835 return nullptr; 4836 4837 case ICmpInst::ICMP_ULE: 4838 // Recognize pattern: 4839 // mulval = mul(zext A, zext B) 4840 // cmp ule mulval, max 4841 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 4842 APInt MaxVal = APInt::getMaxValue(MulWidth); 4843 MaxVal = MaxVal.zext(CI->getBitWidth()); 4844 if (MaxVal.eq(CI->getValue())) 4845 break; // Recognized 4846 } 4847 return nullptr; 4848 4849 case ICmpInst::ICMP_ULT: 4850 // Recognize pattern: 4851 // mulval = mul(zext A, zext B) 4852 // cmp ule mulval, max + 1 4853 if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { 4854 APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); 4855 if (MaxVal.eq(CI->getValue())) 4856 break; // Recognized 4857 } 4858 return nullptr; 4859 4860 default: 4861 return nullptr; 4862 } 4863 4864 InstCombiner::BuilderTy &Builder = IC.Builder; 4865 Builder.SetInsertPoint(MulInstr); 4866 4867 // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) 4868 Value *MulA = A, *MulB = B; 4869 if (WidthA < MulWidth) 4870 MulA = Builder.CreateZExt(A, MulType); 4871 if (WidthB < MulWidth) 4872 MulB = Builder.CreateZExt(B, MulType); 4873 Function *F = Intrinsic::getDeclaration( 4874 I.getModule(), Intrinsic::umul_with_overflow, MulType); 4875 CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul"); 4876 IC.addToWorklist(MulInstr); 4877 4878 // If there are uses of mul result other than the comparison, we know that 4879 // they are truncation or binary AND. Change them to use result of 4880 // mul.with.overflow and adjust properly mask/size. 4881 if (MulVal->hasNUsesOrMore(2)) { 4882 Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value"); 4883 for (User *U : make_early_inc_range(MulVal->users())) { 4884 if (U == &I || U == OtherVal) 4885 continue; 4886 if (TruncInst *TI = dyn_cast<TruncInst>(U)) { 4887 if (TI->getType()->getPrimitiveSizeInBits() == MulWidth) 4888 IC.replaceInstUsesWith(*TI, Mul); 4889 else 4890 TI->setOperand(0, Mul); 4891 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { 4892 assert(BO->getOpcode() == Instruction::And); 4893 // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) 4894 ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1)); 4895 APInt ShortMask = CI->getValue().trunc(MulWidth); 4896 Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask); 4897 Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType()); 4898 IC.replaceInstUsesWith(*BO, Zext); 4899 } else { 4900 llvm_unreachable("Unexpected Binary operation"); 4901 } 4902 IC.addToWorklist(cast<Instruction>(U)); 4903 } 4904 } 4905 if (isa<Instruction>(OtherVal)) 4906 IC.addToWorklist(cast<Instruction>(OtherVal)); 4907 4908 // The original icmp gets replaced with the overflow value, maybe inverted 4909 // depending on predicate. 4910 bool Inverse = false; 4911 switch (I.getPredicate()) { 4912 case ICmpInst::ICMP_NE: 4913 break; 4914 case ICmpInst::ICMP_EQ: 4915 Inverse = true; 4916 break; 4917 case ICmpInst::ICMP_UGT: 4918 case ICmpInst::ICMP_UGE: 4919 if (I.getOperand(0) == MulVal) 4920 break; 4921 Inverse = true; 4922 break; 4923 case ICmpInst::ICMP_ULT: 4924 case ICmpInst::ICMP_ULE: 4925 if (I.getOperand(1) == MulVal) 4926 break; 4927 Inverse = true; 4928 break; 4929 default: 4930 llvm_unreachable("Unexpected predicate"); 4931 } 4932 if (Inverse) { 4933 Value *Res = Builder.CreateExtractValue(Call, 1); 4934 return BinaryOperator::CreateNot(Res); 4935 } 4936 4937 return ExtractValueInst::Create(Call, 1); 4938 } 4939 4940 /// When performing a comparison against a constant, it is possible that not all 4941 /// the bits in the LHS are demanded. This helper method computes the mask that 4942 /// IS demanded. 4943 static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) { 4944 const APInt *RHS; 4945 if (!match(I.getOperand(1), m_APInt(RHS))) 4946 return APInt::getAllOnesValue(BitWidth); 4947 4948 // If this is a normal comparison, it demands all bits. If it is a sign bit 4949 // comparison, it only demands the sign bit. 4950 bool UnusedBit; 4951 if (InstCombiner::isSignBitCheck(I.getPredicate(), *RHS, UnusedBit)) 4952 return APInt::getSignMask(BitWidth); 4953 4954 switch (I.getPredicate()) { 4955 // For a UGT comparison, we don't care about any bits that 4956 // correspond to the trailing ones of the comparand. The value of these 4957 // bits doesn't impact the outcome of the comparison, because any value 4958 // greater than the RHS must differ in a bit higher than these due to carry. 4959 case ICmpInst::ICMP_UGT: 4960 return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingOnes()); 4961 4962 // Similarly, for a ULT comparison, we don't care about the trailing zeros. 4963 // Any value less than the RHS must differ in a higher bit because of carries. 4964 case ICmpInst::ICMP_ULT: 4965 return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingZeros()); 4966 4967 default: 4968 return APInt::getAllOnesValue(BitWidth); 4969 } 4970 } 4971 4972 /// Check if the order of \p Op0 and \p Op1 as operands in an ICmpInst 4973 /// should be swapped. 4974 /// The decision is based on how many times these two operands are reused 4975 /// as subtract operands and their positions in those instructions. 4976 /// The rationale is that several architectures use the same instruction for 4977 /// both subtract and cmp. Thus, it is better if the order of those operands 4978 /// match. 4979 /// \return true if Op0 and Op1 should be swapped. 4980 static bool swapMayExposeCSEOpportunities(const Value *Op0, const Value *Op1) { 4981 // Filter out pointer values as those cannot appear directly in subtract. 4982 // FIXME: we may want to go through inttoptrs or bitcasts. 4983 if (Op0->getType()->isPointerTy()) 4984 return false; 4985 // If a subtract already has the same operands as a compare, swapping would be 4986 // bad. If a subtract has the same operands as a compare but in reverse order, 4987 // then swapping is good. 4988 int GoodToSwap = 0; 4989 for (const User *U : Op0->users()) { 4990 if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0)))) 4991 GoodToSwap++; 4992 else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1)))) 4993 GoodToSwap--; 4994 } 4995 return GoodToSwap > 0; 4996 } 4997 4998 /// Check that one use is in the same block as the definition and all 4999 /// other uses are in blocks dominated by a given block. 5000 /// 5001 /// \param DI Definition 5002 /// \param UI Use 5003 /// \param DB Block that must dominate all uses of \p DI outside 5004 /// the parent block 5005 /// \return true when \p UI is the only use of \p DI in the parent block 5006 /// and all other uses of \p DI are in blocks dominated by \p DB. 5007 /// 5008 bool InstCombinerImpl::dominatesAllUses(const Instruction *DI, 5009 const Instruction *UI, 5010 const BasicBlock *DB) const { 5011 assert(DI && UI && "Instruction not defined\n"); 5012 // Ignore incomplete definitions. 5013 if (!DI->getParent()) 5014 return false; 5015 // DI and UI must be in the same block. 5016 if (DI->getParent() != UI->getParent()) 5017 return false; 5018 // Protect from self-referencing blocks. 5019 if (DI->getParent() == DB) 5020 return false; 5021 for (const User *U : DI->users()) { 5022 auto *Usr = cast<Instruction>(U); 5023 if (Usr != UI && !DT.dominates(DB, Usr->getParent())) 5024 return false; 5025 } 5026 return true; 5027 } 5028 5029 /// Return true when the instruction sequence within a block is select-cmp-br. 5030 static bool isChainSelectCmpBranch(const SelectInst *SI) { 5031 const BasicBlock *BB = SI->getParent(); 5032 if (!BB) 5033 return false; 5034 auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator()); 5035 if (!BI || BI->getNumSuccessors() != 2) 5036 return false; 5037 auto *IC = dyn_cast<ICmpInst>(BI->getCondition()); 5038 if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI)) 5039 return false; 5040 return true; 5041 } 5042 5043 /// True when a select result is replaced by one of its operands 5044 /// in select-icmp sequence. This will eventually result in the elimination 5045 /// of the select. 5046 /// 5047 /// \param SI Select instruction 5048 /// \param Icmp Compare instruction 5049 /// \param SIOpd Operand that replaces the select 5050 /// 5051 /// Notes: 5052 /// - The replacement is global and requires dominator information 5053 /// - The caller is responsible for the actual replacement 5054 /// 5055 /// Example: 5056 /// 5057 /// entry: 5058 /// %4 = select i1 %3, %C* %0, %C* null 5059 /// %5 = icmp eq %C* %4, null 5060 /// br i1 %5, label %9, label %7 5061 /// ... 5062 /// ; <label>:7 ; preds = %entry 5063 /// %8 = getelementptr inbounds %C* %4, i64 0, i32 0 5064 /// ... 5065 /// 5066 /// can be transformed to 5067 /// 5068 /// %5 = icmp eq %C* %0, null 5069 /// %6 = select i1 %3, i1 %5, i1 true 5070 /// br i1 %6, label %9, label %7 5071 /// ... 5072 /// ; <label>:7 ; preds = %entry 5073 /// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0! 5074 /// 5075 /// Similar when the first operand of the select is a constant or/and 5076 /// the compare is for not equal rather than equal. 5077 /// 5078 /// NOTE: The function is only called when the select and compare constants 5079 /// are equal, the optimization can work only for EQ predicates. This is not a 5080 /// major restriction since a NE compare should be 'normalized' to an equal 5081 /// compare, which usually happens in the combiner and test case 5082 /// select-cmp-br.ll checks for it. 5083 bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI, 5084 const ICmpInst *Icmp, 5085 const unsigned SIOpd) { 5086 assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!"); 5087 if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) { 5088 BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1); 5089 // The check for the single predecessor is not the best that can be 5090 // done. But it protects efficiently against cases like when SI's 5091 // home block has two successors, Succ and Succ1, and Succ1 predecessor 5092 // of Succ. Then SI can't be replaced by SIOpd because the use that gets 5093 // replaced can be reached on either path. So the uniqueness check 5094 // guarantees that the path all uses of SI (outside SI's parent) are on 5095 // is disjoint from all other paths out of SI. But that information 5096 // is more expensive to compute, and the trade-off here is in favor 5097 // of compile-time. It should also be noticed that we check for a single 5098 // predecessor and not only uniqueness. This to handle the situation when 5099 // Succ and Succ1 points to the same basic block. 5100 if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) { 5101 NumSel++; 5102 SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent()); 5103 return true; 5104 } 5105 } 5106 return false; 5107 } 5108 5109 /// Try to fold the comparison based on range information we can get by checking 5110 /// whether bits are known to be zero or one in the inputs. 5111 Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) { 5112 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 5113 Type *Ty = Op0->getType(); 5114 ICmpInst::Predicate Pred = I.getPredicate(); 5115 5116 // Get scalar or pointer size. 5117 unsigned BitWidth = Ty->isIntOrIntVectorTy() 5118 ? Ty->getScalarSizeInBits() 5119 : DL.getPointerTypeSizeInBits(Ty->getScalarType()); 5120 5121 if (!BitWidth) 5122 return nullptr; 5123 5124 KnownBits Op0Known(BitWidth); 5125 KnownBits Op1Known(BitWidth); 5126 5127 if (SimplifyDemandedBits(&I, 0, 5128 getDemandedBitsLHSMask(I, BitWidth), 5129 Op0Known, 0)) 5130 return &I; 5131 5132 if (SimplifyDemandedBits(&I, 1, APInt::getAllOnesValue(BitWidth), 5133 Op1Known, 0)) 5134 return &I; 5135 5136 // Given the known and unknown bits, compute a range that the LHS could be 5137 // in. Compute the Min, Max and RHS values based on the known bits. For the 5138 // EQ and NE we use unsigned values. 5139 APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0); 5140 APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0); 5141 if (I.isSigned()) { 5142 Op0Min = Op0Known.getSignedMinValue(); 5143 Op0Max = Op0Known.getSignedMaxValue(); 5144 Op1Min = Op1Known.getSignedMinValue(); 5145 Op1Max = Op1Known.getSignedMaxValue(); 5146 } else { 5147 Op0Min = Op0Known.getMinValue(); 5148 Op0Max = Op0Known.getMaxValue(); 5149 Op1Min = Op1Known.getMinValue(); 5150 Op1Max = Op1Known.getMaxValue(); 5151 } 5152 5153 // If Min and Max are known to be the same, then SimplifyDemandedBits figured 5154 // out that the LHS or RHS is a constant. Constant fold this now, so that 5155 // code below can assume that Min != Max. 5156 if (!isa<Constant>(Op0) && Op0Min == Op0Max) 5157 return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, Op0Min), Op1); 5158 if (!isa<Constant>(Op1) && Op1Min == Op1Max) 5159 return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Min)); 5160 5161 // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a 5162 // min/max canonical compare with some other compare. That could lead to 5163 // conflict with select canonicalization and infinite looping. 5164 // FIXME: This constraint may go away if min/max intrinsics are canonical. 5165 auto isMinMaxCmp = [&](Instruction &Cmp) { 5166 if (!Cmp.hasOneUse()) 5167 return false; 5168 Value *A, *B; 5169 SelectPatternFlavor SPF = matchSelectPattern(Cmp.user_back(), A, B).Flavor; 5170 if (!SelectPatternResult::isMinOrMax(SPF)) 5171 return false; 5172 return match(Op0, m_MaxOrMin(m_Value(), m_Value())) || 5173 match(Op1, m_MaxOrMin(m_Value(), m_Value())); 5174 }; 5175 if (!isMinMaxCmp(I)) { 5176 switch (Pred) { 5177 default: 5178 break; 5179 case ICmpInst::ICMP_ULT: { 5180 if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B) 5181 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5182 const APInt *CmpC; 5183 if (match(Op1, m_APInt(CmpC))) { 5184 // A <u C -> A == C-1 if min(A)+1 == C 5185 if (*CmpC == Op0Min + 1) 5186 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5187 ConstantInt::get(Op1->getType(), *CmpC - 1)); 5188 // X <u C --> X == 0, if the number of zero bits in the bottom of X 5189 // exceeds the log2 of C. 5190 if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2()) 5191 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5192 Constant::getNullValue(Op1->getType())); 5193 } 5194 break; 5195 } 5196 case ICmpInst::ICMP_UGT: { 5197 if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B) 5198 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5199 const APInt *CmpC; 5200 if (match(Op1, m_APInt(CmpC))) { 5201 // A >u C -> A == C+1 if max(a)-1 == C 5202 if (*CmpC == Op0Max - 1) 5203 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5204 ConstantInt::get(Op1->getType(), *CmpC + 1)); 5205 // X >u C --> X != 0, if the number of zero bits in the bottom of X 5206 // exceeds the log2 of C. 5207 if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits()) 5208 return new ICmpInst(ICmpInst::ICMP_NE, Op0, 5209 Constant::getNullValue(Op1->getType())); 5210 } 5211 break; 5212 } 5213 case ICmpInst::ICMP_SLT: { 5214 if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B) 5215 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5216 const APInt *CmpC; 5217 if (match(Op1, m_APInt(CmpC))) { 5218 if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C 5219 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5220 ConstantInt::get(Op1->getType(), *CmpC - 1)); 5221 } 5222 break; 5223 } 5224 case ICmpInst::ICMP_SGT: { 5225 if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B) 5226 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); 5227 const APInt *CmpC; 5228 if (match(Op1, m_APInt(CmpC))) { 5229 if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C 5230 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, 5231 ConstantInt::get(Op1->getType(), *CmpC + 1)); 5232 } 5233 break; 5234 } 5235 } 5236 } 5237 5238 // Based on the range information we know about the LHS, see if we can 5239 // simplify this comparison. For example, (x&4) < 8 is always true. 5240 switch (Pred) { 5241 default: 5242 llvm_unreachable("Unknown icmp opcode!"); 5243 case ICmpInst::ICMP_EQ: 5244 case ICmpInst::ICMP_NE: { 5245 if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max)) 5246 return replaceInstUsesWith( 5247 I, ConstantInt::getBool(I.getType(), Pred == CmpInst::ICMP_NE)); 5248 5249 // If all bits are known zero except for one, then we know at most one bit 5250 // is set. If the comparison is against zero, then this is a check to see if 5251 // *that* bit is set. 5252 APInt Op0KnownZeroInverted = ~Op0Known.Zero; 5253 if (Op1Known.isZero()) { 5254 // If the LHS is an AND with the same constant, look through it. 5255 Value *LHS = nullptr; 5256 const APInt *LHSC; 5257 if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) || 5258 *LHSC != Op0KnownZeroInverted) 5259 LHS = Op0; 5260 5261 Value *X; 5262 if (match(LHS, m_Shl(m_One(), m_Value(X)))) { 5263 APInt ValToCheck = Op0KnownZeroInverted; 5264 Type *XTy = X->getType(); 5265 if (ValToCheck.isPowerOf2()) { 5266 // ((1 << X) & 8) == 0 -> X != 3 5267 // ((1 << X) & 8) != 0 -> X == 3 5268 auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); 5269 auto NewPred = ICmpInst::getInversePredicate(Pred); 5270 return new ICmpInst(NewPred, X, CmpC); 5271 } else if ((++ValToCheck).isPowerOf2()) { 5272 // ((1 << X) & 7) == 0 -> X >= 3 5273 // ((1 << X) & 7) != 0 -> X < 3 5274 auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); 5275 auto NewPred = 5276 Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; 5277 return new ICmpInst(NewPred, X, CmpC); 5278 } 5279 } 5280 5281 // Check if the LHS is 8 >>u x and the result is a power of 2 like 1. 5282 const APInt *CI; 5283 if (Op0KnownZeroInverted.isOneValue() && 5284 match(LHS, m_LShr(m_Power2(CI), m_Value(X)))) { 5285 // ((8 >>u X) & 1) == 0 -> X != 3 5286 // ((8 >>u X) & 1) != 0 -> X == 3 5287 unsigned CmpVal = CI->countTrailingZeros(); 5288 auto NewPred = ICmpInst::getInversePredicate(Pred); 5289 return new ICmpInst(NewPred, X, ConstantInt::get(X->getType(), CmpVal)); 5290 } 5291 } 5292 break; 5293 } 5294 case ICmpInst::ICMP_ULT: { 5295 if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B) 5296 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5297 if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B) 5298 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5299 break; 5300 } 5301 case ICmpInst::ICMP_UGT: { 5302 if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B) 5303 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5304 if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B) 5305 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5306 break; 5307 } 5308 case ICmpInst::ICMP_SLT: { 5309 if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C) 5310 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5311 if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C) 5312 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5313 break; 5314 } 5315 case ICmpInst::ICMP_SGT: { 5316 if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B) 5317 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5318 if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B) 5319 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5320 break; 5321 } 5322 case ICmpInst::ICMP_SGE: 5323 assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!"); 5324 if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B) 5325 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5326 if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B) 5327 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5328 if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B) 5329 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5330 break; 5331 case ICmpInst::ICMP_SLE: 5332 assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!"); 5333 if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B) 5334 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5335 if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B) 5336 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5337 if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B) 5338 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5339 break; 5340 case ICmpInst::ICMP_UGE: 5341 assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!"); 5342 if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B) 5343 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5344 if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B) 5345 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5346 if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B) 5347 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5348 break; 5349 case ICmpInst::ICMP_ULE: 5350 assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!"); 5351 if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B) 5352 return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); 5353 if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B) 5354 return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); 5355 if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B) 5356 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); 5357 break; 5358 } 5359 5360 // Turn a signed comparison into an unsigned one if both operands are known to 5361 // have the same sign. 5362 if (I.isSigned() && 5363 ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) || 5364 (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) 5365 return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1); 5366 5367 return nullptr; 5368 } 5369 5370 llvm::Optional<std::pair<CmpInst::Predicate, Constant *>> 5371 InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, 5372 Constant *C) { 5373 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && 5374 "Only for relational integer predicates."); 5375 5376 Type *Type = C->getType(); 5377 bool IsSigned = ICmpInst::isSigned(Pred); 5378 5379 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred); 5380 bool WillIncrement = 5381 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT; 5382 5383 // Check if the constant operand can be safely incremented/decremented 5384 // without overflowing/underflowing. 5385 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) { 5386 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned); 5387 }; 5388 5389 Constant *SafeReplacementConstant = nullptr; 5390 if (auto *CI = dyn_cast<ConstantInt>(C)) { 5391 // Bail out if the constant can't be safely incremented/decremented. 5392 if (!ConstantIsOk(CI)) 5393 return llvm::None; 5394 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) { 5395 unsigned NumElts = FVTy->getNumElements(); 5396 for (unsigned i = 0; i != NumElts; ++i) { 5397 Constant *Elt = C->getAggregateElement(i); 5398 if (!Elt) 5399 return llvm::None; 5400 5401 if (isa<UndefValue>(Elt)) 5402 continue; 5403 5404 // Bail out if we can't determine if this constant is min/max or if we 5405 // know that this constant is min/max. 5406 auto *CI = dyn_cast<ConstantInt>(Elt); 5407 if (!CI || !ConstantIsOk(CI)) 5408 return llvm::None; 5409 5410 if (!SafeReplacementConstant) 5411 SafeReplacementConstant = CI; 5412 } 5413 } else { 5414 // ConstantExpr? 5415 return llvm::None; 5416 } 5417 5418 // It may not be safe to change a compare predicate in the presence of 5419 // undefined elements, so replace those elements with the first safe constant 5420 // that we found. 5421 // TODO: in case of poison, it is safe; let's replace undefs only. 5422 if (C->containsUndefOrPoisonElement()) { 5423 assert(SafeReplacementConstant && "Replacement constant not set"); 5424 C = Constant::replaceUndefsWith(C, SafeReplacementConstant); 5425 } 5426 5427 CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(Pred); 5428 5429 // Increment or decrement the constant. 5430 Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true); 5431 Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne); 5432 5433 return std::make_pair(NewPred, NewC); 5434 } 5435 5436 /// If we have an icmp le or icmp ge instruction with a constant operand, turn 5437 /// it into the appropriate icmp lt or icmp gt instruction. This transform 5438 /// allows them to be folded in visitICmpInst. 5439 static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { 5440 ICmpInst::Predicate Pred = I.getPredicate(); 5441 if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) || 5442 InstCombiner::isCanonicalPredicate(Pred)) 5443 return nullptr; 5444 5445 Value *Op0 = I.getOperand(0); 5446 Value *Op1 = I.getOperand(1); 5447 auto *Op1C = dyn_cast<Constant>(Op1); 5448 if (!Op1C) 5449 return nullptr; 5450 5451 auto FlippedStrictness = 5452 InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, Op1C); 5453 if (!FlippedStrictness) 5454 return nullptr; 5455 5456 return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second); 5457 } 5458 5459 /// If we have a comparison with a non-canonical predicate, if we can update 5460 /// all the users, invert the predicate and adjust all the users. 5461 CmpInst *InstCombinerImpl::canonicalizeICmpPredicate(CmpInst &I) { 5462 // Is the predicate already canonical? 5463 CmpInst::Predicate Pred = I.getPredicate(); 5464 if (InstCombiner::isCanonicalPredicate(Pred)) 5465 return nullptr; 5466 5467 // Can all users be adjusted to predicate inversion? 5468 if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr)) 5469 return nullptr; 5470 5471 // Ok, we can canonicalize comparison! 5472 // Let's first invert the comparison's predicate. 5473 I.setPredicate(CmpInst::getInversePredicate(Pred)); 5474 I.setName(I.getName() + ".not"); 5475 5476 // And, adapt users. 5477 freelyInvertAllUsersOf(&I); 5478 5479 return &I; 5480 } 5481 5482 /// Integer compare with boolean values can always be turned into bitwise ops. 5483 static Instruction *canonicalizeICmpBool(ICmpInst &I, 5484 InstCombiner::BuilderTy &Builder) { 5485 Value *A = I.getOperand(0), *B = I.getOperand(1); 5486 assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only"); 5487 5488 // A boolean compared to true/false can be simplified to Op0/true/false in 5489 // 14 out of the 20 (10 predicates * 2 constants) possible combinations. 5490 // Cases not handled by InstSimplify are always 'not' of Op0. 5491 if (match(B, m_Zero())) { 5492 switch (I.getPredicate()) { 5493 case CmpInst::ICMP_EQ: // A == 0 -> !A 5494 case CmpInst::ICMP_ULE: // A <=u 0 -> !A 5495 case CmpInst::ICMP_SGE: // A >=s 0 -> !A 5496 return BinaryOperator::CreateNot(A); 5497 default: 5498 llvm_unreachable("ICmp i1 X, C not simplified as expected."); 5499 } 5500 } else if (match(B, m_One())) { 5501 switch (I.getPredicate()) { 5502 case CmpInst::ICMP_NE: // A != 1 -> !A 5503 case CmpInst::ICMP_ULT: // A <u 1 -> !A 5504 case CmpInst::ICMP_SGT: // A >s -1 -> !A 5505 return BinaryOperator::CreateNot(A); 5506 default: 5507 llvm_unreachable("ICmp i1 X, C not simplified as expected."); 5508 } 5509 } 5510 5511 switch (I.getPredicate()) { 5512 default: 5513 llvm_unreachable("Invalid icmp instruction!"); 5514 case ICmpInst::ICMP_EQ: 5515 // icmp eq i1 A, B -> ~(A ^ B) 5516 return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); 5517 5518 case ICmpInst::ICMP_NE: 5519 // icmp ne i1 A, B -> A ^ B 5520 return BinaryOperator::CreateXor(A, B); 5521 5522 case ICmpInst::ICMP_UGT: 5523 // icmp ugt -> icmp ult 5524 std::swap(A, B); 5525 LLVM_FALLTHROUGH; 5526 case ICmpInst::ICMP_ULT: 5527 // icmp ult i1 A, B -> ~A & B 5528 return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); 5529 5530 case ICmpInst::ICMP_SGT: 5531 // icmp sgt -> icmp slt 5532 std::swap(A, B); 5533 LLVM_FALLTHROUGH; 5534 case ICmpInst::ICMP_SLT: 5535 // icmp slt i1 A, B -> A & ~B 5536 return BinaryOperator::CreateAnd(Builder.CreateNot(B), A); 5537 5538 case ICmpInst::ICMP_UGE: 5539 // icmp uge -> icmp ule 5540 std::swap(A, B); 5541 LLVM_FALLTHROUGH; 5542 case ICmpInst::ICMP_ULE: 5543 // icmp ule i1 A, B -> ~A | B 5544 return BinaryOperator::CreateOr(Builder.CreateNot(A), B); 5545 5546 case ICmpInst::ICMP_SGE: 5547 // icmp sge -> icmp sle 5548 std::swap(A, B); 5549 LLVM_FALLTHROUGH; 5550 case ICmpInst::ICMP_SLE: 5551 // icmp sle i1 A, B -> A | ~B 5552 return BinaryOperator::CreateOr(Builder.CreateNot(B), A); 5553 } 5554 } 5555 5556 // Transform pattern like: 5557 // (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X 5558 // (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X 5559 // Into: 5560 // (X l>> Y) != 0 5561 // (X l>> Y) == 0 5562 static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp, 5563 InstCombiner::BuilderTy &Builder) { 5564 ICmpInst::Predicate Pred, NewPred; 5565 Value *X, *Y; 5566 if (match(&Cmp, 5567 m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) { 5568 switch (Pred) { 5569 case ICmpInst::ICMP_ULE: 5570 NewPred = ICmpInst::ICMP_NE; 5571 break; 5572 case ICmpInst::ICMP_UGT: 5573 NewPred = ICmpInst::ICMP_EQ; 5574 break; 5575 default: 5576 return nullptr; 5577 } 5578 } else if (match(&Cmp, m_c_ICmp(Pred, 5579 m_OneUse(m_CombineOr( 5580 m_Not(m_Shl(m_AllOnes(), m_Value(Y))), 5581 m_Add(m_Shl(m_One(), m_Value(Y)), 5582 m_AllOnes()))), 5583 m_Value(X)))) { 5584 // The variant with 'add' is not canonical, (the variant with 'not' is) 5585 // we only get it because it has extra uses, and can't be canonicalized, 5586 5587 switch (Pred) { 5588 case ICmpInst::ICMP_ULT: 5589 NewPred = ICmpInst::ICMP_NE; 5590 break; 5591 case ICmpInst::ICMP_UGE: 5592 NewPred = ICmpInst::ICMP_EQ; 5593 break; 5594 default: 5595 return nullptr; 5596 } 5597 } else 5598 return nullptr; 5599 5600 Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits"); 5601 Constant *Zero = Constant::getNullValue(NewX->getType()); 5602 return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero); 5603 } 5604 5605 static Instruction *foldVectorCmp(CmpInst &Cmp, 5606 InstCombiner::BuilderTy &Builder) { 5607 const CmpInst::Predicate Pred = Cmp.getPredicate(); 5608 Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1); 5609 Value *V1, *V2; 5610 ArrayRef<int> M; 5611 if (!match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(M)))) 5612 return nullptr; 5613 5614 // If both arguments of the cmp are shuffles that use the same mask and 5615 // shuffle within a single vector, move the shuffle after the cmp: 5616 // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M 5617 Type *V1Ty = V1->getType(); 5618 if (match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(M))) && 5619 V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) { 5620 Value *NewCmp = Builder.CreateCmp(Pred, V1, V2); 5621 return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M); 5622 } 5623 5624 // Try to canonicalize compare with splatted operand and splat constant. 5625 // TODO: We could generalize this for more than splats. See/use the code in 5626 // InstCombiner::foldVectorBinop(). 5627 Constant *C; 5628 if (!LHS->hasOneUse() || !match(RHS, m_Constant(C))) 5629 return nullptr; 5630 5631 // Length-changing splats are ok, so adjust the constants as needed: 5632 // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M 5633 Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true); 5634 int MaskSplatIndex; 5635 if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) { 5636 // We allow undefs in matching, but this transform removes those for safety. 5637 // Demanded elements analysis should be able to recover some/all of that. 5638 C = ConstantVector::getSplat(cast<VectorType>(V1Ty)->getElementCount(), 5639 ScalarC); 5640 SmallVector<int, 8> NewM(M.size(), MaskSplatIndex); 5641 Value *NewCmp = Builder.CreateCmp(Pred, V1, C); 5642 return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), 5643 NewM); 5644 } 5645 5646 return nullptr; 5647 } 5648 5649 // extract(uadd.with.overflow(A, B), 0) ult A 5650 // -> extract(uadd.with.overflow(A, B), 1) 5651 static Instruction *foldICmpOfUAddOv(ICmpInst &I) { 5652 CmpInst::Predicate Pred = I.getPredicate(); 5653 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 5654 5655 Value *UAddOv; 5656 Value *A, *B; 5657 auto UAddOvResultPat = m_ExtractValue<0>( 5658 m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B))); 5659 if (match(Op0, UAddOvResultPat) && 5660 ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) || 5661 (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) && 5662 (match(A, m_One()) || match(B, m_One()))) || 5663 (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) && 5664 (match(A, m_AllOnes()) || match(B, m_AllOnes()))))) 5665 // extract(uadd.with.overflow(A, B), 0) < A 5666 // extract(uadd.with.overflow(A, 1), 0) == 0 5667 // extract(uadd.with.overflow(A, -1), 0) != -1 5668 UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand(); 5669 else if (match(Op1, UAddOvResultPat) && 5670 Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B)) 5671 // A > extract(uadd.with.overflow(A, B), 0) 5672 UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand(); 5673 else 5674 return nullptr; 5675 5676 return ExtractValueInst::Create(UAddOv, 1); 5677 } 5678 5679 Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { 5680 bool Changed = false; 5681 const SimplifyQuery Q = SQ.getWithInstruction(&I); 5682 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 5683 unsigned Op0Cplxity = getComplexity(Op0); 5684 unsigned Op1Cplxity = getComplexity(Op1); 5685 5686 /// Orders the operands of the compare so that they are listed from most 5687 /// complex to least complex. This puts constants before unary operators, 5688 /// before binary operators. 5689 if (Op0Cplxity < Op1Cplxity || 5690 (Op0Cplxity == Op1Cplxity && swapMayExposeCSEOpportunities(Op0, Op1))) { 5691 I.swapOperands(); 5692 std::swap(Op0, Op1); 5693 Changed = true; 5694 } 5695 5696 if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, Q)) 5697 return replaceInstUsesWith(I, V); 5698 5699 // Comparing -val or val with non-zero is the same as just comparing val 5700 // ie, abs(val) != 0 -> val != 0 5701 if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) { 5702 Value *Cond, *SelectTrue, *SelectFalse; 5703 if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue), 5704 m_Value(SelectFalse)))) { 5705 if (Value *V = dyn_castNegVal(SelectTrue)) { 5706 if (V == SelectFalse) 5707 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); 5708 } 5709 else if (Value *V = dyn_castNegVal(SelectFalse)) { 5710 if (V == SelectTrue) 5711 return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); 5712 } 5713 } 5714 } 5715 5716 if (Op0->getType()->isIntOrIntVectorTy(1)) 5717 if (Instruction *Res = canonicalizeICmpBool(I, Builder)) 5718 return Res; 5719 5720 if (Instruction *Res = canonicalizeCmpWithConstant(I)) 5721 return Res; 5722 5723 if (Instruction *Res = canonicalizeICmpPredicate(I)) 5724 return Res; 5725 5726 if (Instruction *Res = foldICmpWithConstant(I)) 5727 return Res; 5728 5729 if (Instruction *Res = foldICmpWithDominatingICmp(I)) 5730 return Res; 5731 5732 if (Instruction *Res = foldICmpBinOp(I, Q)) 5733 return Res; 5734 5735 if (Instruction *Res = foldICmpUsingKnownBits(I)) 5736 return Res; 5737 5738 // Test if the ICmpInst instruction is used exclusively by a select as 5739 // part of a minimum or maximum operation. If so, refrain from doing 5740 // any other folding. This helps out other analyses which understand 5741 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution 5742 // and CodeGen. And in this case, at least one of the comparison 5743 // operands has at least one user besides the compare (the select), 5744 // which would often largely negate the benefit of folding anyway. 5745 // 5746 // Do the same for the other patterns recognized by matchSelectPattern. 5747 if (I.hasOneUse()) 5748 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { 5749 Value *A, *B; 5750 SelectPatternResult SPR = matchSelectPattern(SI, A, B); 5751 if (SPR.Flavor != SPF_UNKNOWN) 5752 return nullptr; 5753 } 5754 5755 // Do this after checking for min/max to prevent infinite looping. 5756 if (Instruction *Res = foldICmpWithZero(I)) 5757 return Res; 5758 5759 // FIXME: We only do this after checking for min/max to prevent infinite 5760 // looping caused by a reverse canonicalization of these patterns for min/max. 5761 // FIXME: The organization of folds is a mess. These would naturally go into 5762 // canonicalizeCmpWithConstant(), but we can't move all of the above folds 5763 // down here after the min/max restriction. 5764 ICmpInst::Predicate Pred = I.getPredicate(); 5765 const APInt *C; 5766 if (match(Op1, m_APInt(C))) { 5767 // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set 5768 if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) { 5769 Constant *Zero = Constant::getNullValue(Op0->getType()); 5770 return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero); 5771 } 5772 5773 // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear 5774 if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) { 5775 Constant *AllOnes = Constant::getAllOnesValue(Op0->getType()); 5776 return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes); 5777 } 5778 } 5779 5780 if (Instruction *Res = foldICmpInstWithConstant(I)) 5781 return Res; 5782 5783 // Try to match comparison as a sign bit test. Intentionally do this after 5784 // foldICmpInstWithConstant() to potentially let other folds to happen first. 5785 if (Instruction *New = foldSignBitTest(I)) 5786 return New; 5787 5788 if (Instruction *Res = foldICmpInstWithConstantNotInt(I)) 5789 return Res; 5790 5791 // If we can optimize a 'icmp GEP, P' or 'icmp P, GEP', do so now. 5792 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op0)) 5793 if (Instruction *NI = foldGEPICmp(GEP, Op1, I.getPredicate(), I)) 5794 return NI; 5795 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op1)) 5796 if (Instruction *NI = foldGEPICmp(GEP, Op0, 5797 ICmpInst::getSwappedPredicate(I.getPredicate()), I)) 5798 return NI; 5799 5800 // Try to optimize equality comparisons against alloca-based pointers. 5801 if (Op0->getType()->isPointerTy() && I.isEquality()) { 5802 assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?"); 5803 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0))) 5804 if (Instruction *New = foldAllocaCmp(I, Alloca, Op1)) 5805 return New; 5806 if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1))) 5807 if (Instruction *New = foldAllocaCmp(I, Alloca, Op0)) 5808 return New; 5809 } 5810 5811 if (Instruction *Res = foldICmpBitCast(I, Builder)) 5812 return Res; 5813 5814 // TODO: Hoist this above the min/max bailout. 5815 if (Instruction *R = foldICmpWithCastOp(I)) 5816 return R; 5817 5818 if (Instruction *Res = foldICmpWithMinMax(I)) 5819 return Res; 5820 5821 { 5822 Value *A, *B; 5823 // Transform (A & ~B) == 0 --> (A & B) != 0 5824 // and (A & ~B) != 0 --> (A & B) == 0 5825 // if A is a power of 2. 5826 if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) && 5827 match(Op1, m_Zero()) && 5828 isKnownToBeAPowerOfTwo(A, false, 0, &I) && I.isEquality()) 5829 return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(A, B), 5830 Op1); 5831 5832 // ~X < ~Y --> Y < X 5833 // ~X < C --> X > ~C 5834 if (match(Op0, m_Not(m_Value(A)))) { 5835 if (match(Op1, m_Not(m_Value(B)))) 5836 return new ICmpInst(I.getPredicate(), B, A); 5837 5838 const APInt *C; 5839 if (match(Op1, m_APInt(C))) 5840 return new ICmpInst(I.getSwappedPredicate(), A, 5841 ConstantInt::get(Op1->getType(), ~(*C))); 5842 } 5843 5844 Instruction *AddI = nullptr; 5845 if (match(&I, m_UAddWithOverflow(m_Value(A), m_Value(B), 5846 m_Instruction(AddI))) && 5847 isa<IntegerType>(A->getType())) { 5848 Value *Result; 5849 Constant *Overflow; 5850 // m_UAddWithOverflow can match patterns that do not include an explicit 5851 // "add" instruction, so check the opcode of the matched op. 5852 if (AddI->getOpcode() == Instruction::Add && 5853 OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, A, B, *AddI, 5854 Result, Overflow)) { 5855 replaceInstUsesWith(*AddI, Result); 5856 eraseInstFromFunction(*AddI); 5857 return replaceInstUsesWith(I, Overflow); 5858 } 5859 } 5860 5861 // (zext a) * (zext b) --> llvm.umul.with.overflow. 5862 if (match(Op0, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { 5863 if (Instruction *R = processUMulZExtIdiom(I, Op0, Op1, *this)) 5864 return R; 5865 } 5866 if (match(Op1, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { 5867 if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this)) 5868 return R; 5869 } 5870 } 5871 5872 if (Instruction *Res = foldICmpEquality(I)) 5873 return Res; 5874 5875 if (Instruction *Res = foldICmpOfUAddOv(I)) 5876 return Res; 5877 5878 // The 'cmpxchg' instruction returns an aggregate containing the old value and 5879 // an i1 which indicates whether or not we successfully did the swap. 5880 // 5881 // Replace comparisons between the old value and the expected value with the 5882 // indicator that 'cmpxchg' returns. 5883 // 5884 // N.B. This transform is only valid when the 'cmpxchg' is not permitted to 5885 // spuriously fail. In those cases, the old value may equal the expected 5886 // value but it is possible for the swap to not occur. 5887 if (I.getPredicate() == ICmpInst::ICMP_EQ) 5888 if (auto *EVI = dyn_cast<ExtractValueInst>(Op0)) 5889 if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand())) 5890 if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 && 5891 !ACXI->isWeak()) 5892 return ExtractValueInst::Create(ACXI, 1); 5893 5894 { 5895 Value *X; 5896 const APInt *C; 5897 // icmp X+Cst, X 5898 if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X) 5899 return foldICmpAddOpConst(X, *C, I.getPredicate()); 5900 5901 // icmp X, X+Cst 5902 if (match(Op1, m_Add(m_Value(X), m_APInt(C))) && Op0 == X) 5903 return foldICmpAddOpConst(X, *C, I.getSwappedPredicate()); 5904 } 5905 5906 if (Instruction *Res = foldICmpWithHighBitMask(I, Builder)) 5907 return Res; 5908 5909 if (I.getType()->isVectorTy()) 5910 if (Instruction *Res = foldVectorCmp(I, Builder)) 5911 return Res; 5912 5913 return Changed ? &I : nullptr; 5914 } 5915 5916 /// Fold fcmp ([us]itofp x, cst) if possible. 5917 Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, 5918 Instruction *LHSI, 5919 Constant *RHSC) { 5920 if (!isa<ConstantFP>(RHSC)) return nullptr; 5921 const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF(); 5922 5923 // Get the width of the mantissa. We don't want to hack on conversions that 5924 // might lose information from the integer, e.g. "i64 -> float" 5925 int MantissaWidth = LHSI->getType()->getFPMantissaWidth(); 5926 if (MantissaWidth == -1) return nullptr; // Unknown. 5927 5928 IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType()); 5929 5930 bool LHSUnsigned = isa<UIToFPInst>(LHSI); 5931 5932 if (I.isEquality()) { 5933 FCmpInst::Predicate P = I.getPredicate(); 5934 bool IsExact = false; 5935 APSInt RHSCvt(IntTy->getBitWidth(), LHSUnsigned); 5936 RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); 5937 5938 // If the floating point constant isn't an integer value, we know if we will 5939 // ever compare equal / not equal to it. 5940 if (!IsExact) { 5941 // TODO: Can never be -0.0 and other non-representable values 5942 APFloat RHSRoundInt(RHS); 5943 RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); 5944 if (RHS != RHSRoundInt) { 5945 if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) 5946 return replaceInstUsesWith(I, Builder.getFalse()); 5947 5948 assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE); 5949 return replaceInstUsesWith(I, Builder.getTrue()); 5950 } 5951 } 5952 5953 // TODO: If the constant is exactly representable, is it always OK to do 5954 // equality compares as integer? 5955 } 5956 5957 // Check to see that the input is converted from an integer type that is small 5958 // enough that preserves all bits. TODO: check here for "known" sign bits. 5959 // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e. 5960 unsigned InputSize = IntTy->getScalarSizeInBits(); 5961 5962 // Following test does NOT adjust InputSize downwards for signed inputs, 5963 // because the most negative value still requires all the mantissa bits 5964 // to distinguish it from one less than that value. 5965 if ((int)InputSize > MantissaWidth) { 5966 // Conversion would lose accuracy. Check if loss can impact comparison. 5967 int Exp = ilogb(RHS); 5968 if (Exp == APFloat::IEK_Inf) { 5969 int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics())); 5970 if (MaxExponent < (int)InputSize - !LHSUnsigned) 5971 // Conversion could create infinity. 5972 return nullptr; 5973 } else { 5974 // Note that if RHS is zero or NaN, then Exp is negative 5975 // and first condition is trivially false. 5976 if (MantissaWidth <= Exp && Exp <= (int)InputSize - !LHSUnsigned) 5977 // Conversion could affect comparison. 5978 return nullptr; 5979 } 5980 } 5981 5982 // Otherwise, we can potentially simplify the comparison. We know that it 5983 // will always come through as an integer value and we know the constant is 5984 // not a NAN (it would have been previously simplified). 5985 assert(!RHS.isNaN() && "NaN comparison not already folded!"); 5986 5987 ICmpInst::Predicate Pred; 5988 switch (I.getPredicate()) { 5989 default: llvm_unreachable("Unexpected predicate!"); 5990 case FCmpInst::FCMP_UEQ: 5991 case FCmpInst::FCMP_OEQ: 5992 Pred = ICmpInst::ICMP_EQ; 5993 break; 5994 case FCmpInst::FCMP_UGT: 5995 case FCmpInst::FCMP_OGT: 5996 Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT; 5997 break; 5998 case FCmpInst::FCMP_UGE: 5999 case FCmpInst::FCMP_OGE: 6000 Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE; 6001 break; 6002 case FCmpInst::FCMP_ULT: 6003 case FCmpInst::FCMP_OLT: 6004 Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT; 6005 break; 6006 case FCmpInst::FCMP_ULE: 6007 case FCmpInst::FCMP_OLE: 6008 Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE; 6009 break; 6010 case FCmpInst::FCMP_UNE: 6011 case FCmpInst::FCMP_ONE: 6012 Pred = ICmpInst::ICMP_NE; 6013 break; 6014 case FCmpInst::FCMP_ORD: 6015 return replaceInstUsesWith(I, Builder.getTrue()); 6016 case FCmpInst::FCMP_UNO: 6017 return replaceInstUsesWith(I, Builder.getFalse()); 6018 } 6019 6020 // Now we know that the APFloat is a normal number, zero or inf. 6021 6022 // See if the FP constant is too large for the integer. For example, 6023 // comparing an i8 to 300.0. 6024 unsigned IntWidth = IntTy->getScalarSizeInBits(); 6025 6026 if (!LHSUnsigned) { 6027 // If the RHS value is > SignedMax, fold the comparison. This handles +INF 6028 // and large values. 6029 APFloat SMax(RHS.getSemantics()); 6030 SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true, 6031 APFloat::rmNearestTiesToEven); 6032 if (SMax < RHS) { // smax < 13123.0 6033 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || 6034 Pred == ICmpInst::ICMP_SLE) 6035 return replaceInstUsesWith(I, Builder.getTrue()); 6036 return replaceInstUsesWith(I, Builder.getFalse()); 6037 } 6038 } else { 6039 // If the RHS value is > UnsignedMax, fold the comparison. This handles 6040 // +INF and large values. 6041 APFloat UMax(RHS.getSemantics()); 6042 UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false, 6043 APFloat::rmNearestTiesToEven); 6044 if (UMax < RHS) { // umax < 13123.0 6045 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || 6046 Pred == ICmpInst::ICMP_ULE) 6047 return replaceInstUsesWith(I, Builder.getTrue()); 6048 return replaceInstUsesWith(I, Builder.getFalse()); 6049 } 6050 } 6051 6052 if (!LHSUnsigned) { 6053 // See if the RHS value is < SignedMin. 6054 APFloat SMin(RHS.getSemantics()); 6055 SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true, 6056 APFloat::rmNearestTiesToEven); 6057 if (SMin > RHS) { // smin > 12312.0 6058 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || 6059 Pred == ICmpInst::ICMP_SGE) 6060 return replaceInstUsesWith(I, Builder.getTrue()); 6061 return replaceInstUsesWith(I, Builder.getFalse()); 6062 } 6063 } else { 6064 // See if the RHS value is < UnsignedMin. 6065 APFloat UMin(RHS.getSemantics()); 6066 UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false, 6067 APFloat::rmNearestTiesToEven); 6068 if (UMin > RHS) { // umin > 12312.0 6069 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || 6070 Pred == ICmpInst::ICMP_UGE) 6071 return replaceInstUsesWith(I, Builder.getTrue()); 6072 return replaceInstUsesWith(I, Builder.getFalse()); 6073 } 6074 } 6075 6076 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or 6077 // [0, UMAX], but it may still be fractional. See if it is fractional by 6078 // casting the FP value to the integer value and back, checking for equality. 6079 // Don't do this for zero, because -0.0 is not fractional. 6080 Constant *RHSInt = LHSUnsigned 6081 ? ConstantExpr::getFPToUI(RHSC, IntTy) 6082 : ConstantExpr::getFPToSI(RHSC, IntTy); 6083 if (!RHS.isZero()) { 6084 bool Equal = LHSUnsigned 6085 ? ConstantExpr::getUIToFP(RHSInt, RHSC->getType()) == RHSC 6086 : ConstantExpr::getSIToFP(RHSInt, RHSC->getType()) == RHSC; 6087 if (!Equal) { 6088 // If we had a comparison against a fractional value, we have to adjust 6089 // the compare predicate and sometimes the value. RHSC is rounded towards 6090 // zero at this point. 6091 switch (Pred) { 6092 default: llvm_unreachable("Unexpected integer comparison!"); 6093 case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true 6094 return replaceInstUsesWith(I, Builder.getTrue()); 6095 case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false 6096 return replaceInstUsesWith(I, Builder.getFalse()); 6097 case ICmpInst::ICMP_ULE: 6098 // (float)int <= 4.4 --> int <= 4 6099 // (float)int <= -4.4 --> false 6100 if (RHS.isNegative()) 6101 return replaceInstUsesWith(I, Builder.getFalse()); 6102 break; 6103 case ICmpInst::ICMP_SLE: 6104 // (float)int <= 4.4 --> int <= 4 6105 // (float)int <= -4.4 --> int < -4 6106 if (RHS.isNegative()) 6107 Pred = ICmpInst::ICMP_SLT; 6108 break; 6109 case ICmpInst::ICMP_ULT: 6110 // (float)int < -4.4 --> false 6111 // (float)int < 4.4 --> int <= 4 6112 if (RHS.isNegative()) 6113 return replaceInstUsesWith(I, Builder.getFalse()); 6114 Pred = ICmpInst::ICMP_ULE; 6115 break; 6116 case ICmpInst::ICMP_SLT: 6117 // (float)int < -4.4 --> int < -4 6118 // (float)int < 4.4 --> int <= 4 6119 if (!RHS.isNegative()) 6120 Pred = ICmpInst::ICMP_SLE; 6121 break; 6122 case ICmpInst::ICMP_UGT: 6123 // (float)int > 4.4 --> int > 4 6124 // (float)int > -4.4 --> true 6125 if (RHS.isNegative()) 6126 return replaceInstUsesWith(I, Builder.getTrue()); 6127 break; 6128 case ICmpInst::ICMP_SGT: 6129 // (float)int > 4.4 --> int > 4 6130 // (float)int > -4.4 --> int >= -4 6131 if (RHS.isNegative()) 6132 Pred = ICmpInst::ICMP_SGE; 6133 break; 6134 case ICmpInst::ICMP_UGE: 6135 // (float)int >= -4.4 --> true 6136 // (float)int >= 4.4 --> int > 4 6137 if (RHS.isNegative()) 6138 return replaceInstUsesWith(I, Builder.getTrue()); 6139 Pred = ICmpInst::ICMP_UGT; 6140 break; 6141 case ICmpInst::ICMP_SGE: 6142 // (float)int >= -4.4 --> int >= -4 6143 // (float)int >= 4.4 --> int > 4 6144 if (!RHS.isNegative()) 6145 Pred = ICmpInst::ICMP_SGT; 6146 break; 6147 } 6148 } 6149 } 6150 6151 // Lower this FP comparison into an appropriate integer version of the 6152 // comparison. 6153 return new ICmpInst(Pred, LHSI->getOperand(0), RHSInt); 6154 } 6155 6156 /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary. 6157 static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, 6158 Constant *RHSC) { 6159 // When C is not 0.0 and infinities are not allowed: 6160 // (C / X) < 0.0 is a sign-bit test of X 6161 // (C / X) < 0.0 --> X < 0.0 (if C is positive) 6162 // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate) 6163 // 6164 // Proof: 6165 // Multiply (C / X) < 0.0 by X * X / C. 6166 // - X is non zero, if it is the flag 'ninf' is violated. 6167 // - C defines the sign of X * X * C. Thus it also defines whether to swap 6168 // the predicate. C is also non zero by definition. 6169 // 6170 // Thus X * X / C is non zero and the transformation is valid. [qed] 6171 6172 FCmpInst::Predicate Pred = I.getPredicate(); 6173 6174 // Check that predicates are valid. 6175 if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) && 6176 (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE)) 6177 return nullptr; 6178 6179 // Check that RHS operand is zero. 6180 if (!match(RHSC, m_AnyZeroFP())) 6181 return nullptr; 6182 6183 // Check fastmath flags ('ninf'). 6184 if (!LHSI->hasNoInfs() || !I.hasNoInfs()) 6185 return nullptr; 6186 6187 // Check the properties of the dividend. It must not be zero to avoid a 6188 // division by zero (see Proof). 6189 const APFloat *C; 6190 if (!match(LHSI->getOperand(0), m_APFloat(C))) 6191 return nullptr; 6192 6193 if (C->isZero()) 6194 return nullptr; 6195 6196 // Get swapped predicate if necessary. 6197 if (C->isNegative()) 6198 Pred = I.getSwappedPredicate(); 6199 6200 return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I); 6201 } 6202 6203 /// Optimize fabs(X) compared with zero. 6204 static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) { 6205 Value *X; 6206 if (!match(I.getOperand(0), m_FAbs(m_Value(X))) || 6207 !match(I.getOperand(1), m_PosZeroFP())) 6208 return nullptr; 6209 6210 auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) { 6211 I->setPredicate(P); 6212 return IC.replaceOperand(*I, 0, X); 6213 }; 6214 6215 switch (I.getPredicate()) { 6216 case FCmpInst::FCMP_UGE: 6217 case FCmpInst::FCMP_OLT: 6218 // fabs(X) >= 0.0 --> true 6219 // fabs(X) < 0.0 --> false 6220 llvm_unreachable("fcmp should have simplified"); 6221 6222 case FCmpInst::FCMP_OGT: 6223 // fabs(X) > 0.0 --> X != 0.0 6224 return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X); 6225 6226 case FCmpInst::FCMP_UGT: 6227 // fabs(X) u> 0.0 --> X u!= 0.0 6228 return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X); 6229 6230 case FCmpInst::FCMP_OLE: 6231 // fabs(X) <= 0.0 --> X == 0.0 6232 return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X); 6233 6234 case FCmpInst::FCMP_ULE: 6235 // fabs(X) u<= 0.0 --> X u== 0.0 6236 return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X); 6237 6238 case FCmpInst::FCMP_OGE: 6239 // fabs(X) >= 0.0 --> !isnan(X) 6240 assert(!I.hasNoNaNs() && "fcmp should have simplified"); 6241 return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X); 6242 6243 case FCmpInst::FCMP_ULT: 6244 // fabs(X) u< 0.0 --> isnan(X) 6245 assert(!I.hasNoNaNs() && "fcmp should have simplified"); 6246 return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X); 6247 6248 case FCmpInst::FCMP_OEQ: 6249 case FCmpInst::FCMP_UEQ: 6250 case FCmpInst::FCMP_ONE: 6251 case FCmpInst::FCMP_UNE: 6252 case FCmpInst::FCMP_ORD: 6253 case FCmpInst::FCMP_UNO: 6254 // Look through the fabs() because it doesn't change anything but the sign. 6255 // fabs(X) == 0.0 --> X == 0.0, 6256 // fabs(X) != 0.0 --> X != 0.0 6257 // isnan(fabs(X)) --> isnan(X) 6258 // !isnan(fabs(X) --> !isnan(X) 6259 return replacePredAndOp0(&I, I.getPredicate(), X); 6260 6261 default: 6262 return nullptr; 6263 } 6264 } 6265 6266 Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) { 6267 bool Changed = false; 6268 6269 /// Orders the operands of the compare so that they are listed from most 6270 /// complex to least complex. This puts constants before unary operators, 6271 /// before binary operators. 6272 if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) { 6273 I.swapOperands(); 6274 Changed = true; 6275 } 6276 6277 const CmpInst::Predicate Pred = I.getPredicate(); 6278 Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); 6279 if (Value *V = SimplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(), 6280 SQ.getWithInstruction(&I))) 6281 return replaceInstUsesWith(I, V); 6282 6283 // Simplify 'fcmp pred X, X' 6284 Type *OpType = Op0->getType(); 6285 assert(OpType == Op1->getType() && "fcmp with different-typed operands?"); 6286 if (Op0 == Op1) { 6287 switch (Pred) { 6288 default: break; 6289 case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y) 6290 case FCmpInst::FCMP_ULT: // True if unordered or less than 6291 case FCmpInst::FCMP_UGT: // True if unordered or greater than 6292 case FCmpInst::FCMP_UNE: // True if unordered or not equal 6293 // Canonicalize these to be 'fcmp uno %X, 0.0'. 6294 I.setPredicate(FCmpInst::FCMP_UNO); 6295 I.setOperand(1, Constant::getNullValue(OpType)); 6296 return &I; 6297 6298 case FCmpInst::FCMP_ORD: // True if ordered (no nans) 6299 case FCmpInst::FCMP_OEQ: // True if ordered and equal 6300 case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal 6301 case FCmpInst::FCMP_OLE: // True if ordered and less than or equal 6302 // Canonicalize these to be 'fcmp ord %X, 0.0'. 6303 I.setPredicate(FCmpInst::FCMP_ORD); 6304 I.setOperand(1, Constant::getNullValue(OpType)); 6305 return &I; 6306 } 6307 } 6308 6309 // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand, 6310 // then canonicalize the operand to 0.0. 6311 if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { 6312 if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI)) 6313 return replaceOperand(I, 0, ConstantFP::getNullValue(OpType)); 6314 6315 if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI)) 6316 return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); 6317 } 6318 6319 // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y 6320 Value *X, *Y; 6321 if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) 6322 return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I); 6323 6324 // Test if the FCmpInst instruction is used exclusively by a select as 6325 // part of a minimum or maximum operation. If so, refrain from doing 6326 // any other folding. This helps out other analyses which understand 6327 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution 6328 // and CodeGen. And in this case, at least one of the comparison 6329 // operands has at least one user besides the compare (the select), 6330 // which would often largely negate the benefit of folding anyway. 6331 if (I.hasOneUse()) 6332 if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { 6333 Value *A, *B; 6334 SelectPatternResult SPR = matchSelectPattern(SI, A, B); 6335 if (SPR.Flavor != SPF_UNKNOWN) 6336 return nullptr; 6337 } 6338 6339 // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0: 6340 // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0 6341 if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP())) 6342 return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); 6343 6344 // Handle fcmp with instruction LHS and constant RHS. 6345 Instruction *LHSI; 6346 Constant *RHSC; 6347 if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) { 6348 switch (LHSI->getOpcode()) { 6349 case Instruction::PHI: 6350 // Only fold fcmp into the PHI if the phi and fcmp are in the same 6351 // block. If in the same block, we're encouraging jump threading. If 6352 // not, we are just pessimizing the code by making an i1 phi. 6353 if (LHSI->getParent() == I.getParent()) 6354 if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) 6355 return NV; 6356 break; 6357 case Instruction::SIToFP: 6358 case Instruction::UIToFP: 6359 if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC)) 6360 return NV; 6361 break; 6362 case Instruction::FDiv: 6363 if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC)) 6364 return NV; 6365 break; 6366 case Instruction::Load: 6367 if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) 6368 if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) 6369 if (GV->isConstant() && GV->hasDefinitiveInitializer() && 6370 !cast<LoadInst>(LHSI)->isVolatile()) 6371 if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) 6372 return Res; 6373 break; 6374 } 6375 } 6376 6377 if (Instruction *R = foldFabsWithFcmpZero(I, *this)) 6378 return R; 6379 6380 if (match(Op0, m_FNeg(m_Value(X)))) { 6381 // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C 6382 Constant *C; 6383 if (match(Op1, m_Constant(C))) { 6384 Constant *NegC = ConstantExpr::getFNeg(C); 6385 return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I); 6386 } 6387 } 6388 6389 if (match(Op0, m_FPExt(m_Value(X)))) { 6390 // fcmp (fpext X), (fpext Y) -> fcmp X, Y 6391 if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType()) 6392 return new FCmpInst(Pred, X, Y, "", &I); 6393 6394 // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless 6395 const APFloat *C; 6396 if (match(Op1, m_APFloat(C))) { 6397 const fltSemantics &FPSem = 6398 X->getType()->getScalarType()->getFltSemantics(); 6399 bool Lossy; 6400 APFloat TruncC = *C; 6401 TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy); 6402 6403 // Avoid lossy conversions and denormals. 6404 // Zero is a special case that's OK to convert. 6405 APFloat Fabs = TruncC; 6406 Fabs.clearSign(); 6407 if (!Lossy && 6408 (!(Fabs < APFloat::getSmallestNormalized(FPSem)) || Fabs.isZero())) { 6409 Constant *NewC = ConstantFP::get(X->getType(), TruncC); 6410 return new FCmpInst(Pred, X, NewC, "", &I); 6411 } 6412 } 6413 } 6414 6415 // Convert a sign-bit test of an FP value into a cast and integer compare. 6416 // TODO: Simplify if the copysign constant is 0.0 or NaN. 6417 // TODO: Handle non-zero compare constants. 6418 // TODO: Handle other predicates. 6419 const APFloat *C; 6420 if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::copysign>(m_APFloat(C), 6421 m_Value(X)))) && 6422 match(Op1, m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) { 6423 Type *IntType = Builder.getIntNTy(X->getType()->getScalarSizeInBits()); 6424 if (auto *VecTy = dyn_cast<VectorType>(OpType)) 6425 IntType = VectorType::get(IntType, VecTy->getElementCount()); 6426 6427 // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0 6428 if (Pred == FCmpInst::FCMP_OLT) { 6429 Value *IntX = Builder.CreateBitCast(X, IntType); 6430 return new ICmpInst(ICmpInst::ICMP_SLT, IntX, 6431 ConstantInt::getNullValue(IntType)); 6432 } 6433 } 6434 6435 if (I.getType()->isVectorTy()) 6436 if (Instruction *Res = foldVectorCmp(I, Builder)) 6437 return Res; 6438 6439 return Changed ? &I : nullptr; 6440 } 6441