1 //===- llvm/Support/GenericFloatingPointPredicateUtils.h -----*- C++-*-----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// Utilities for dealing with flags related to floating point properties and 11 /// mode controls. 12 /// 13 //===----------------------------------------------------------------------===/ 14 15 #ifndef LLVM_ADT_GENERICFLOATINGPOINTPREDICATEUTILS_H 16 #define LLVM_ADT_GENERICFLOATINGPOINTPREDICATEUTILS_H 17 18 #include "llvm/ADT/APFloat.h" 19 #include "llvm/ADT/FloatingPointMode.h" 20 #include "llvm/IR/Instructions.h" 21 #include "llvm/Support/Compiler.h" 22 #include <optional> 23 24 namespace llvm { 25 26 template <typename ContextT> class GenericFloatingPointPredicateUtils { 27 using ValueRefT = typename ContextT::ValueRefT; 28 using FunctionT = typename ContextT::FunctionT; 29 30 constexpr static ValueRefT Invalid = {}; 31 32 private: 33 LLVM_ABI static DenormalMode queryDenormalMode(const FunctionT &F, 34 ValueRefT Val); 35 36 LLVM_ABI static bool lookThroughFAbs(const FunctionT &F, ValueRefT LHS, 37 ValueRefT &Src); 38 39 LLVM_ABI static std::optional<APFloat> matchConstantFloat(const FunctionT &F, 40 ValueRefT Val); 41 42 /// Return the return value for fcmpImpliesClass for a compare that produces 43 /// an exact class test. 44 static std::tuple<ValueRefT, FPClassTest, FPClassTest> exactClass(ValueRefT V,FPClassTest M)45 exactClass(ValueRefT V, FPClassTest M) { 46 return {V, M, ~M}; 47 } 48 49 public: 50 /// Returns a pair of values, which if passed to llvm.is.fpclass, returns the 51 /// same result as an fcmp with the given operands. 52 static std::pair<ValueRefT, FPClassTest> fcmpToClassTest(FCmpInst::Predicate Pred,const FunctionT & F,ValueRefT LHS,ValueRefT RHS,bool LookThroughSrc)53 fcmpToClassTest(FCmpInst::Predicate Pred, const FunctionT &F, ValueRefT LHS, 54 ValueRefT RHS, bool LookThroughSrc) { 55 std::optional<APFloat> ConstRHS = matchConstantFloat(F, RHS); 56 if (!ConstRHS) 57 return {Invalid, fcAllFlags}; 58 59 return fcmpToClassTest(Pred, F, LHS, *ConstRHS, LookThroughSrc); 60 } 61 62 static std::pair<ValueRefT, FPClassTest> fcmpToClassTest(FCmpInst::Predicate Pred,const FunctionT & F,ValueRefT LHS,const APFloat & ConstRHS,bool LookThroughSrc)63 fcmpToClassTest(FCmpInst::Predicate Pred, const FunctionT &F, ValueRefT LHS, 64 const APFloat &ConstRHS, bool LookThroughSrc) { 65 66 auto [Src, ClassIfTrue, ClassIfFalse] = 67 fcmpImpliesClass(Pred, F, LHS, ConstRHS, LookThroughSrc); 68 69 if (Src && ClassIfTrue == ~ClassIfFalse) 70 return {Src, ClassIfTrue}; 71 72 return {Invalid, fcAllFlags}; 73 } 74 75 /// Compute the possible floating-point classes that \p LHS could be based on 76 /// fcmp \Pred \p LHS, \p RHS. 77 /// 78 /// \returns { TestedValue, ClassesIfTrue, ClassesIfFalse } 79 /// 80 /// If the compare returns an exact class test, ClassesIfTrue == 81 /// ~ClassesIfFalse 82 /// 83 /// This is a less exact version of fcmpToClassTest (e.g. fcmpToClassTest will 84 /// only succeed for a test of x > 0 implies positive, but not x > 1). 85 /// 86 /// If \p LookThroughSrc is true, consider the input value when computing the 87 /// mask. This may look through sign bit operations. 88 /// 89 /// If \p LookThroughSrc is false, ignore the source value (i.e. the first 90 /// pair element will always be LHS. 91 /// 92 static std::tuple<ValueRefT, FPClassTest, FPClassTest> fcmpImpliesClass(CmpInst::Predicate Pred,const FunctionT & F,ValueRefT LHS,FPClassTest RHSClass,bool LookThroughSrc)93 fcmpImpliesClass(CmpInst::Predicate Pred, const FunctionT &F, ValueRefT LHS, 94 FPClassTest RHSClass, bool LookThroughSrc) { 95 assert(RHSClass != fcNone); 96 ValueRefT Src = LHS; 97 98 if (Pred == FCmpInst::FCMP_TRUE) 99 return exactClass(Src, fcAllFlags); 100 101 if (Pred == FCmpInst::FCMP_FALSE) 102 return exactClass(Src, fcNone); 103 104 const FPClassTest OrigClass = RHSClass; 105 106 const bool IsNegativeRHS = (RHSClass & fcNegative) == RHSClass; 107 const bool IsPositiveRHS = (RHSClass & fcPositive) == RHSClass; 108 const bool IsNaN = (RHSClass & ~fcNan) == fcNone; 109 110 if (IsNaN) { 111 // fcmp o__ x, nan -> false 112 // fcmp u__ x, nan -> true 113 return exactClass(Src, CmpInst::isOrdered(Pred) ? fcNone : fcAllFlags); 114 } 115 116 // fcmp ord x, zero|normal|subnormal|inf -> ~fcNan 117 if (Pred == FCmpInst::FCMP_ORD) 118 return exactClass(Src, ~fcNan); 119 120 // fcmp uno x, zero|normal|subnormal|inf -> fcNan 121 if (Pred == FCmpInst::FCMP_UNO) 122 return exactClass(Src, fcNan); 123 124 const bool IsFabs = LookThroughSrc && lookThroughFAbs(F, LHS, Src); 125 if (IsFabs) 126 RHSClass = llvm::inverse_fabs(RHSClass); 127 128 const bool IsZero = (OrigClass & fcZero) == OrigClass; 129 if (IsZero) { 130 assert(Pred != FCmpInst::FCMP_ORD && Pred != FCmpInst::FCMP_UNO); 131 // Compares with fcNone are only exactly equal to fcZero if input 132 // denormals are not flushed. 133 // TODO: Handle DAZ by expanding masks to cover subnormal cases. 134 DenormalMode Mode = queryDenormalMode(F, LHS); 135 if (Mode.Input != DenormalMode::IEEE) 136 return {Invalid, fcAllFlags, fcAllFlags}; 137 138 auto ExactClass = [IsFabs, Src](FPClassTest Mask) { 139 if (IsFabs) 140 Mask = llvm::inverse_fabs(Mask); 141 return exactClass(Src, Mask); 142 }; 143 144 switch (Pred) { 145 case FCmpInst::FCMP_OEQ: // Match x == 0.0 146 return exactClass(Src, fcZero); 147 case FCmpInst::FCMP_UEQ: // Match isnan(x) || (x == 0.0) 148 return exactClass(Src, fcZero | fcNan); 149 case FCmpInst::FCMP_UNE: // Match (x != 0.0) 150 return exactClass(Src, ~fcZero); 151 case FCmpInst::FCMP_ONE: // Match !isnan(x) && x != 0.0 152 return exactClass(Src, ~fcNan & ~fcZero); 153 case FCmpInst::FCMP_ORD: 154 // Canonical form of ord/uno is with a zero. We could also handle 155 // non-canonical other non-NaN constants or LHS == RHS. 156 return exactClass(Src, ~fcNan); 157 case FCmpInst::FCMP_UNO: 158 return exactClass(Src, fcNan); 159 case FCmpInst::FCMP_OGT: // x > 0 160 return ExactClass(fcPosSubnormal | fcPosNormal | fcPosInf); 161 case FCmpInst::FCMP_UGT: // isnan(x) || x > 0 162 return ExactClass(fcPosSubnormal | fcPosNormal | fcPosInf | fcNan); 163 case FCmpInst::FCMP_OGE: // x >= 0 164 return ExactClass(fcPositive | fcNegZero); 165 case FCmpInst::FCMP_UGE: // isnan(x) || x >= 0 166 return ExactClass(fcPositive | fcNegZero | fcNan); 167 case FCmpInst::FCMP_OLT: // x < 0 168 return ExactClass(fcNegSubnormal | fcNegNormal | fcNegInf); 169 case FCmpInst::FCMP_ULT: // isnan(x) || x < 0 170 return ExactClass(fcNegSubnormal | fcNegNormal | fcNegInf | fcNan); 171 case FCmpInst::FCMP_OLE: // x <= 0 172 return ExactClass(fcNegative | fcPosZero); 173 case FCmpInst::FCMP_ULE: // isnan(x) || x <= 0 174 return ExactClass(fcNegative | fcPosZero | fcNan); 175 default: 176 llvm_unreachable("all compare types are handled"); 177 } 178 } 179 180 const bool IsDenormalRHS = (OrigClass & fcSubnormal) == OrigClass; 181 182 const bool IsInf = (OrigClass & fcInf) == OrigClass; 183 if (IsInf) { 184 FPClassTest Mask = fcAllFlags; 185 186 switch (Pred) { 187 case FCmpInst::FCMP_OEQ: 188 case FCmpInst::FCMP_UNE: { 189 // Match __builtin_isinf patterns 190 // 191 // fcmp oeq x, +inf -> is_fpclass x, fcPosInf 192 // fcmp oeq fabs(x), +inf -> is_fpclass x, fcInf 193 // fcmp oeq x, -inf -> is_fpclass x, fcNegInf 194 // fcmp oeq fabs(x), -inf -> is_fpclass x, 0 -> false 195 // 196 // fcmp une x, +inf -> is_fpclass x, ~fcPosInf 197 // fcmp une fabs(x), +inf -> is_fpclass x, ~fcInf 198 // fcmp une x, -inf -> is_fpclass x, ~fcNegInf 199 // fcmp une fabs(x), -inf -> is_fpclass x, fcAllFlags -> true 200 if (IsNegativeRHS) { 201 Mask = fcNegInf; 202 if (IsFabs) 203 Mask = fcNone; 204 } else { 205 Mask = fcPosInf; 206 if (IsFabs) 207 Mask |= fcNegInf; 208 } 209 break; 210 } 211 case FCmpInst::FCMP_ONE: 212 case FCmpInst::FCMP_UEQ: { 213 // Match __builtin_isinf patterns 214 // fcmp one x, -inf -> is_fpclass x, fcNegInf 215 // fcmp one fabs(x), -inf -> is_fpclass x, ~fcNegInf & ~fcNan 216 // fcmp one x, +inf -> is_fpclass x, ~fcNegInf & ~fcNan 217 // fcmp one fabs(x), +inf -> is_fpclass x, ~fcInf & fcNan 218 // 219 // fcmp ueq x, +inf -> is_fpclass x, fcPosInf|fcNan 220 // fcmp ueq (fabs x), +inf -> is_fpclass x, fcInf|fcNan 221 // fcmp ueq x, -inf -> is_fpclass x, fcNegInf|fcNan 222 // fcmp ueq fabs(x), -inf -> is_fpclass x, fcNan 223 if (IsNegativeRHS) { 224 Mask = ~fcNegInf & ~fcNan; 225 if (IsFabs) 226 Mask = ~fcNan; 227 } else { 228 Mask = ~fcPosInf & ~fcNan; 229 if (IsFabs) 230 Mask &= ~fcNegInf; 231 } 232 233 break; 234 } 235 case FCmpInst::FCMP_OLT: 236 case FCmpInst::FCMP_UGE: { 237 if (IsNegativeRHS) { 238 // No value is ordered and less than negative infinity. 239 // All values are unordered with or at least negative infinity. 240 // fcmp olt x, -inf -> false 241 // fcmp uge x, -inf -> true 242 Mask = fcNone; 243 break; 244 } 245 246 // fcmp olt fabs(x), +inf -> fcFinite 247 // fcmp uge fabs(x), +inf -> ~fcFinite 248 // fcmp olt x, +inf -> fcFinite|fcNegInf 249 // fcmp uge x, +inf -> ~(fcFinite|fcNegInf) 250 Mask = fcFinite; 251 if (!IsFabs) 252 Mask |= fcNegInf; 253 break; 254 } 255 case FCmpInst::FCMP_OGE: 256 case FCmpInst::FCMP_ULT: { 257 if (IsNegativeRHS) { 258 // fcmp oge x, -inf -> ~fcNan 259 // fcmp oge fabs(x), -inf -> ~fcNan 260 // fcmp ult x, -inf -> fcNan 261 // fcmp ult fabs(x), -inf -> fcNan 262 Mask = ~fcNan; 263 break; 264 } 265 266 // fcmp oge fabs(x), +inf -> fcInf 267 // fcmp oge x, +inf -> fcPosInf 268 // fcmp ult fabs(x), +inf -> ~fcInf 269 // fcmp ult x, +inf -> ~fcPosInf 270 Mask = fcPosInf; 271 if (IsFabs) 272 Mask |= fcNegInf; 273 break; 274 } 275 case FCmpInst::FCMP_OGT: 276 case FCmpInst::FCMP_ULE: { 277 if (IsNegativeRHS) { 278 // fcmp ogt x, -inf -> fcmp one x, -inf 279 // fcmp ogt fabs(x), -inf -> fcmp ord x, x 280 // fcmp ule x, -inf -> fcmp ueq x, -inf 281 // fcmp ule fabs(x), -inf -> fcmp uno x, x 282 Mask = IsFabs ? ~fcNan : ~(fcNegInf | fcNan); 283 break; 284 } 285 286 // No value is ordered and greater than infinity. 287 Mask = fcNone; 288 break; 289 } 290 case FCmpInst::FCMP_OLE: 291 case FCmpInst::FCMP_UGT: { 292 if (IsNegativeRHS) { 293 Mask = IsFabs ? fcNone : fcNegInf; 294 break; 295 } 296 297 // fcmp ole x, +inf -> fcmp ord x, x 298 // fcmp ole fabs(x), +inf -> fcmp ord x, x 299 // fcmp ole x, -inf -> fcmp oeq x, -inf 300 // fcmp ole fabs(x), -inf -> false 301 Mask = ~fcNan; 302 break; 303 } 304 default: 305 llvm_unreachable("all compare types are handled"); 306 } 307 308 // Invert the comparison for the unordered cases. 309 if (FCmpInst::isUnordered(Pred)) 310 Mask = ~Mask; 311 312 return exactClass(Src, Mask); 313 } 314 315 if (Pred == FCmpInst::FCMP_OEQ) 316 return {Src, RHSClass, fcAllFlags}; 317 318 if (Pred == FCmpInst::FCMP_UEQ) { 319 FPClassTest Class = RHSClass | fcNan; 320 return {Src, Class, ~fcNan}; 321 } 322 323 if (Pred == FCmpInst::FCMP_ONE) 324 return {Src, ~fcNan, RHSClass | fcNan}; 325 326 if (Pred == FCmpInst::FCMP_UNE) 327 return {Src, fcAllFlags, RHSClass}; 328 329 assert((RHSClass == fcNone || RHSClass == fcPosNormal || 330 RHSClass == fcNegNormal || RHSClass == fcNormal || 331 RHSClass == fcPosSubnormal || RHSClass == fcNegSubnormal || 332 RHSClass == fcSubnormal) && 333 "should have been recognized as an exact class test"); 334 335 if (IsNegativeRHS) { 336 // TODO: Handle fneg(fabs) 337 if (IsFabs) { 338 // fabs(x) o> -k -> fcmp ord x, x 339 // fabs(x) u> -k -> true 340 // fabs(x) o< -k -> false 341 // fabs(x) u< -k -> fcmp uno x, x 342 switch (Pred) { 343 case FCmpInst::FCMP_OGT: 344 case FCmpInst::FCMP_OGE: 345 return {Src, ~fcNan, fcNan}; 346 case FCmpInst::FCMP_UGT: 347 case FCmpInst::FCMP_UGE: 348 return {Src, fcAllFlags, fcNone}; 349 case FCmpInst::FCMP_OLT: 350 case FCmpInst::FCMP_OLE: 351 return {Src, fcNone, fcAllFlags}; 352 case FCmpInst::FCMP_ULT: 353 case FCmpInst::FCMP_ULE: 354 return {Src, fcNan, ~fcNan}; 355 default: 356 break; 357 } 358 359 return {Invalid, fcAllFlags, fcAllFlags}; 360 } 361 362 FPClassTest ClassesLE = fcNegInf | fcNegNormal; 363 FPClassTest ClassesGE = fcPositive | fcNegZero | fcNegSubnormal; 364 365 if (IsDenormalRHS) 366 ClassesLE |= fcNegSubnormal; 367 else 368 ClassesGE |= fcNegNormal; 369 370 switch (Pred) { 371 case FCmpInst::FCMP_OGT: 372 case FCmpInst::FCMP_OGE: 373 return {Src, ClassesGE, ~ClassesGE | RHSClass}; 374 case FCmpInst::FCMP_UGT: 375 case FCmpInst::FCMP_UGE: 376 return {Src, ClassesGE | fcNan, ~(ClassesGE | fcNan) | RHSClass}; 377 case FCmpInst::FCMP_OLT: 378 case FCmpInst::FCMP_OLE: 379 return {Src, ClassesLE, ~ClassesLE | RHSClass}; 380 case FCmpInst::FCMP_ULT: 381 case FCmpInst::FCMP_ULE: 382 return {Src, ClassesLE | fcNan, ~(ClassesLE | fcNan) | RHSClass}; 383 default: 384 break; 385 } 386 } else if (IsPositiveRHS) { 387 FPClassTest ClassesGE = fcPosNormal | fcPosInf; 388 FPClassTest ClassesLE = fcNegative | fcPosZero | fcPosSubnormal; 389 if (IsDenormalRHS) 390 ClassesGE |= fcPosSubnormal; 391 else 392 ClassesLE |= fcPosNormal; 393 394 if (IsFabs) { 395 ClassesGE = llvm::inverse_fabs(ClassesGE); 396 ClassesLE = llvm::inverse_fabs(ClassesLE); 397 } 398 399 switch (Pred) { 400 case FCmpInst::FCMP_OGT: 401 case FCmpInst::FCMP_OGE: 402 return {Src, ClassesGE, ~ClassesGE | RHSClass}; 403 case FCmpInst::FCMP_UGT: 404 case FCmpInst::FCMP_UGE: 405 return {Src, ClassesGE | fcNan, ~(ClassesGE | fcNan) | RHSClass}; 406 case FCmpInst::FCMP_OLT: 407 case FCmpInst::FCMP_OLE: 408 return {Src, ClassesLE, ~ClassesLE | RHSClass}; 409 case FCmpInst::FCMP_ULT: 410 case FCmpInst::FCMP_ULE: 411 return {Src, ClassesLE | fcNan, ~(ClassesLE | fcNan) | RHSClass}; 412 default: 413 break; 414 } 415 } 416 417 return {Invalid, fcAllFlags, fcAllFlags}; 418 } 419 420 static std::tuple<ValueRefT, FPClassTest, FPClassTest> fcmpImpliesClass(CmpInst::Predicate Pred,const FunctionT & F,ValueRefT LHS,const APFloat & ConstRHS,bool LookThroughSrc)421 fcmpImpliesClass(CmpInst::Predicate Pred, const FunctionT &F, ValueRefT LHS, 422 const APFloat &ConstRHS, bool LookThroughSrc) { 423 // We can refine checks against smallest normal / largest denormal to an 424 // exact class test. 425 if (!ConstRHS.isNegative() && ConstRHS.isSmallestNormalized()) { 426 ValueRefT Src = LHS; 427 const bool IsFabs = LookThroughSrc && lookThroughFAbs(F, LHS, Src); 428 429 FPClassTest Mask; 430 // Match pattern that's used in __builtin_isnormal. 431 switch (Pred) { 432 case FCmpInst::FCMP_OLT: 433 case FCmpInst::FCMP_UGE: { 434 // fcmp olt x, smallest_normal -> 435 // fcNegInf|fcNegNormal|fcSubnormal|fcZero fcmp olt fabs(x), 436 // smallest_normal -> fcSubnormal|fcZero fcmp uge x, smallest_normal -> 437 // fcNan|fcPosNormal|fcPosInf fcmp uge fabs(x), smallest_normal -> 438 // ~(fcSubnormal|fcZero) 439 Mask = fcZero | fcSubnormal; 440 if (!IsFabs) 441 Mask |= fcNegNormal | fcNegInf; 442 443 break; 444 } 445 case FCmpInst::FCMP_OGE: 446 case FCmpInst::FCMP_ULT: { 447 // fcmp oge x, smallest_normal -> fcPosNormal | fcPosInf 448 // fcmp oge fabs(x), smallest_normal -> fcInf | fcNormal 449 // fcmp ult x, smallest_normal -> ~(fcPosNormal | fcPosInf) 450 // fcmp ult fabs(x), smallest_normal -> ~(fcInf | fcNormal) 451 Mask = fcPosInf | fcPosNormal; 452 if (IsFabs) 453 Mask |= fcNegInf | fcNegNormal; 454 break; 455 } 456 default: 457 return fcmpImpliesClass(Pred, F, LHS, ConstRHS.classify(), 458 LookThroughSrc); 459 } 460 461 // Invert the comparison for the unordered cases. 462 if (FCmpInst::isUnordered(Pred)) 463 Mask = ~Mask; 464 465 return exactClass(Src, Mask); 466 } 467 468 return fcmpImpliesClass(Pred, F, LHS, ConstRHS.classify(), LookThroughSrc); 469 } 470 471 static std::tuple<ValueRefT, FPClassTest, FPClassTest> fcmpImpliesClass(CmpInst::Predicate Pred,const FunctionT & F,ValueRefT LHS,ValueRefT RHS,bool LookThroughSrc)472 fcmpImpliesClass(CmpInst::Predicate Pred, const FunctionT &F, ValueRefT LHS, 473 ValueRefT RHS, bool LookThroughSrc) { 474 std::optional<APFloat> ConstRHS = matchConstantFloat(F, RHS); 475 if (!ConstRHS) 476 return {Invalid, fcAllFlags, fcAllFlags}; 477 478 // TODO: Just call computeKnownFPClass for RHS to handle non-constants. 479 return fcmpImpliesClass(Pred, F, LHS, *ConstRHS, LookThroughSrc); 480 } 481 }; 482 483 } // namespace llvm 484 485 #endif // LLVM_ADT_GENERICFLOATINGPOINTPREDICATEUTILS_H 486