1 //===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This is the parent TargetLowering class for hardware code gen 11 /// targets. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "AMDGPUISelLowering.h" 16 #include "AMDGPU.h" 17 #include "AMDGPUInstrInfo.h" 18 #include "AMDGPUMachineFunction.h" 19 #include "SIMachineFunctionInfo.h" 20 #include "llvm/CodeGen/Analysis.h" 21 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" 22 #include "llvm/CodeGen/MachineFrameInfo.h" 23 #include "llvm/IR/DiagnosticInfo.h" 24 #include "llvm/IR/IntrinsicsAMDGPU.h" 25 #include "llvm/IR/PatternMatch.h" 26 #include "llvm/Support/CommandLine.h" 27 #include "llvm/Support/KnownBits.h" 28 #include "llvm/Target/TargetMachine.h" 29 30 using namespace llvm; 31 32 #include "AMDGPUGenCallingConv.inc" 33 34 static cl::opt<bool> AMDGPUBypassSlowDiv( 35 "amdgpu-bypass-slow-div", 36 cl::desc("Skip 64-bit divide for dynamic 32-bit values"), 37 cl::init(true)); 38 39 // Find a larger type to do a load / store of a vector with. 40 EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) { 41 unsigned StoreSize = VT.getStoreSizeInBits(); 42 if (StoreSize <= 32) 43 return EVT::getIntegerVT(Ctx, StoreSize); 44 45 if (StoreSize % 32 == 0) 46 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); 47 48 return VT; 49 } 50 51 unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) { 52 return DAG.computeKnownBits(Op).countMaxActiveBits(); 53 } 54 55 unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) { 56 // In order for this to be a signed 24-bit value, bit 23, must 57 // be a sign bit. 58 return DAG.ComputeMaxSignificantBits(Op); 59 } 60 61 AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, 62 const AMDGPUSubtarget &STI) 63 : TargetLowering(TM), Subtarget(&STI) { 64 // Always lower memset, memcpy, and memmove intrinsics to load/store 65 // instructions, rather then generating calls to memset, mempcy or memmove. 66 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U; 67 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U; 68 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U; 69 70 // Enable ganging up loads and stores in the memcpy DAG lowering. 71 MaxGluedStoresPerMemcpy = 16; 72 73 // Lower floating point store/load to integer store/load to reduce the number 74 // of patterns in tablegen. 75 setOperationAction(ISD::LOAD, MVT::f32, Promote); 76 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); 77 78 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 79 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); 80 81 setOperationAction(ISD::LOAD, MVT::v3f32, Promote); 82 AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32); 83 84 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); 85 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); 86 87 setOperationAction(ISD::LOAD, MVT::v5f32, Promote); 88 AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32); 89 90 setOperationAction(ISD::LOAD, MVT::v6f32, Promote); 91 AddPromotedToType(ISD::LOAD, MVT::v6f32, MVT::v6i32); 92 93 setOperationAction(ISD::LOAD, MVT::v7f32, Promote); 94 AddPromotedToType(ISD::LOAD, MVT::v7f32, MVT::v7i32); 95 96 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); 97 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); 98 99 setOperationAction(ISD::LOAD, MVT::v9f32, Promote); 100 AddPromotedToType(ISD::LOAD, MVT::v9f32, MVT::v9i32); 101 102 setOperationAction(ISD::LOAD, MVT::v10f32, Promote); 103 AddPromotedToType(ISD::LOAD, MVT::v10f32, MVT::v10i32); 104 105 setOperationAction(ISD::LOAD, MVT::v11f32, Promote); 106 AddPromotedToType(ISD::LOAD, MVT::v11f32, MVT::v11i32); 107 108 setOperationAction(ISD::LOAD, MVT::v12f32, Promote); 109 AddPromotedToType(ISD::LOAD, MVT::v12f32, MVT::v12i32); 110 111 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); 112 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); 113 114 setOperationAction(ISD::LOAD, MVT::v32f32, Promote); 115 AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32); 116 117 setOperationAction(ISD::LOAD, MVT::i64, Promote); 118 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32); 119 120 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); 121 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32); 122 123 setOperationAction(ISD::LOAD, MVT::f64, Promote); 124 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32); 125 126 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); 127 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32); 128 129 setOperationAction(ISD::LOAD, MVT::v3i64, Promote); 130 AddPromotedToType(ISD::LOAD, MVT::v3i64, MVT::v6i32); 131 132 setOperationAction(ISD::LOAD, MVT::v4i64, Promote); 133 AddPromotedToType(ISD::LOAD, MVT::v4i64, MVT::v8i32); 134 135 setOperationAction(ISD::LOAD, MVT::v3f64, Promote); 136 AddPromotedToType(ISD::LOAD, MVT::v3f64, MVT::v6i32); 137 138 setOperationAction(ISD::LOAD, MVT::v4f64, Promote); 139 AddPromotedToType(ISD::LOAD, MVT::v4f64, MVT::v8i32); 140 141 setOperationAction(ISD::LOAD, MVT::v8i64, Promote); 142 AddPromotedToType(ISD::LOAD, MVT::v8i64, MVT::v16i32); 143 144 setOperationAction(ISD::LOAD, MVT::v8f64, Promote); 145 AddPromotedToType(ISD::LOAD, MVT::v8f64, MVT::v16i32); 146 147 setOperationAction(ISD::LOAD, MVT::v16i64, Promote); 148 AddPromotedToType(ISD::LOAD, MVT::v16i64, MVT::v32i32); 149 150 setOperationAction(ISD::LOAD, MVT::v16f64, Promote); 151 AddPromotedToType(ISD::LOAD, MVT::v16f64, MVT::v32i32); 152 153 setOperationAction(ISD::LOAD, MVT::i128, Promote); 154 AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32); 155 156 // TODO: Would be better to consume as directly legal 157 setOperationAction(ISD::ATOMIC_LOAD, MVT::f32, Promote); 158 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f32, MVT::i32); 159 160 setOperationAction(ISD::ATOMIC_LOAD, MVT::f64, Promote); 161 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f64, MVT::i64); 162 163 setOperationAction(ISD::ATOMIC_LOAD, MVT::f16, Promote); 164 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f16, MVT::i16); 165 166 setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote); 167 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16); 168 169 setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote); 170 AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32); 171 172 setOperationAction(ISD::ATOMIC_STORE, MVT::f64, Promote); 173 AddPromotedToType(ISD::ATOMIC_STORE, MVT::f64, MVT::i64); 174 175 setOperationAction(ISD::ATOMIC_STORE, MVT::f16, Promote); 176 AddPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16); 177 178 setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote); 179 AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16); 180 181 // There are no 64-bit extloads. These should be done as a 32-bit extload and 182 // an extension to 64-bit. 183 for (MVT VT : MVT::integer_valuetypes()) 184 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i64, VT, 185 Expand); 186 187 for (MVT VT : MVT::integer_valuetypes()) { 188 if (VT == MVT::i64) 189 continue; 190 191 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) { 192 setLoadExtAction(Op, VT, MVT::i1, Promote); 193 setLoadExtAction(Op, VT, MVT::i8, Legal); 194 setLoadExtAction(Op, VT, MVT::i16, Legal); 195 setLoadExtAction(Op, VT, MVT::i32, Expand); 196 } 197 } 198 199 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) 200 for (auto MemVT : 201 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16}) 202 setLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, VT, MemVT, 203 Expand); 204 205 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); 206 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand); 207 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); 208 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2bf16, Expand); 209 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand); 210 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3bf16, Expand); 211 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); 212 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4bf16, Expand); 213 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); 214 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8bf16, Expand); 215 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand); 216 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16bf16, Expand); 217 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand); 218 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32bf16, Expand); 219 220 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); 221 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand); 222 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f32, Expand); 223 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand); 224 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand); 225 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f32, Expand); 226 227 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); 228 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand); 229 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); 230 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2bf16, Expand); 231 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f16, Expand); 232 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3bf16, Expand); 233 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); 234 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4bf16, Expand); 235 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); 236 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8bf16, Expand); 237 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f16, Expand); 238 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16bf16, Expand); 239 240 setOperationAction(ISD::STORE, MVT::f32, Promote); 241 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); 242 243 setOperationAction(ISD::STORE, MVT::v2f32, Promote); 244 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); 245 246 setOperationAction(ISD::STORE, MVT::v3f32, Promote); 247 AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32); 248 249 setOperationAction(ISD::STORE, MVT::v4f32, Promote); 250 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); 251 252 setOperationAction(ISD::STORE, MVT::v5f32, Promote); 253 AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32); 254 255 setOperationAction(ISD::STORE, MVT::v6f32, Promote); 256 AddPromotedToType(ISD::STORE, MVT::v6f32, MVT::v6i32); 257 258 setOperationAction(ISD::STORE, MVT::v7f32, Promote); 259 AddPromotedToType(ISD::STORE, MVT::v7f32, MVT::v7i32); 260 261 setOperationAction(ISD::STORE, MVT::v8f32, Promote); 262 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); 263 264 setOperationAction(ISD::STORE, MVT::v9f32, Promote); 265 AddPromotedToType(ISD::STORE, MVT::v9f32, MVT::v9i32); 266 267 setOperationAction(ISD::STORE, MVT::v10f32, Promote); 268 AddPromotedToType(ISD::STORE, MVT::v10f32, MVT::v10i32); 269 270 setOperationAction(ISD::STORE, MVT::v11f32, Promote); 271 AddPromotedToType(ISD::STORE, MVT::v11f32, MVT::v11i32); 272 273 setOperationAction(ISD::STORE, MVT::v12f32, Promote); 274 AddPromotedToType(ISD::STORE, MVT::v12f32, MVT::v12i32); 275 276 setOperationAction(ISD::STORE, MVT::v16f32, Promote); 277 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); 278 279 setOperationAction(ISD::STORE, MVT::v32f32, Promote); 280 AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32); 281 282 setOperationAction(ISD::STORE, MVT::i64, Promote); 283 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32); 284 285 setOperationAction(ISD::STORE, MVT::v2i64, Promote); 286 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32); 287 288 setOperationAction(ISD::STORE, MVT::f64, Promote); 289 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32); 290 291 setOperationAction(ISD::STORE, MVT::v2f64, Promote); 292 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32); 293 294 setOperationAction(ISD::STORE, MVT::v3i64, Promote); 295 AddPromotedToType(ISD::STORE, MVT::v3i64, MVT::v6i32); 296 297 setOperationAction(ISD::STORE, MVT::v3f64, Promote); 298 AddPromotedToType(ISD::STORE, MVT::v3f64, MVT::v6i32); 299 300 setOperationAction(ISD::STORE, MVT::v4i64, Promote); 301 AddPromotedToType(ISD::STORE, MVT::v4i64, MVT::v8i32); 302 303 setOperationAction(ISD::STORE, MVT::v4f64, Promote); 304 AddPromotedToType(ISD::STORE, MVT::v4f64, MVT::v8i32); 305 306 setOperationAction(ISD::STORE, MVT::v8i64, Promote); 307 AddPromotedToType(ISD::STORE, MVT::v8i64, MVT::v16i32); 308 309 setOperationAction(ISD::STORE, MVT::v8f64, Promote); 310 AddPromotedToType(ISD::STORE, MVT::v8f64, MVT::v16i32); 311 312 setOperationAction(ISD::STORE, MVT::v16i64, Promote); 313 AddPromotedToType(ISD::STORE, MVT::v16i64, MVT::v32i32); 314 315 setOperationAction(ISD::STORE, MVT::v16f64, Promote); 316 AddPromotedToType(ISD::STORE, MVT::v16f64, MVT::v32i32); 317 318 setOperationAction(ISD::STORE, MVT::i128, Promote); 319 AddPromotedToType(ISD::STORE, MVT::i128, MVT::v4i32); 320 321 setTruncStoreAction(MVT::i64, MVT::i1, Expand); 322 setTruncStoreAction(MVT::i64, MVT::i8, Expand); 323 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 324 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 325 326 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand); 327 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand); 328 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand); 329 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand); 330 331 setTruncStoreAction(MVT::f32, MVT::bf16, Expand); 332 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 333 setTruncStoreAction(MVT::v2f32, MVT::v2bf16, Expand); 334 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand); 335 setTruncStoreAction(MVT::v3f32, MVT::v3bf16, Expand); 336 setTruncStoreAction(MVT::v3f32, MVT::v3f16, Expand); 337 setTruncStoreAction(MVT::v4f32, MVT::v4bf16, Expand); 338 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand); 339 setTruncStoreAction(MVT::v8f32, MVT::v8bf16, Expand); 340 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand); 341 setTruncStoreAction(MVT::v16f32, MVT::v16bf16, Expand); 342 setTruncStoreAction(MVT::v16f32, MVT::v16f16, Expand); 343 setTruncStoreAction(MVT::v32f32, MVT::v32bf16, Expand); 344 setTruncStoreAction(MVT::v32f32, MVT::v32f16, Expand); 345 346 setTruncStoreAction(MVT::f64, MVT::bf16, Expand); 347 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 348 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 349 350 setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand); 351 setTruncStoreAction(MVT::v2f64, MVT::v2bf16, Expand); 352 setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand); 353 354 setTruncStoreAction(MVT::v3i32, MVT::v3i8, Expand); 355 356 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); 357 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); 358 setTruncStoreAction(MVT::v3i64, MVT::v3i8, Expand); 359 setTruncStoreAction(MVT::v3i64, MVT::v3i1, Expand); 360 setTruncStoreAction(MVT::v3f64, MVT::v3f32, Expand); 361 setTruncStoreAction(MVT::v3f64, MVT::v3bf16, Expand); 362 setTruncStoreAction(MVT::v3f64, MVT::v3f16, Expand); 363 364 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Expand); 365 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Expand); 366 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand); 367 setTruncStoreAction(MVT::v4f64, MVT::v4bf16, Expand); 368 setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand); 369 370 setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand); 371 setTruncStoreAction(MVT::v8f64, MVT::v8bf16, Expand); 372 setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand); 373 374 setTruncStoreAction(MVT::v16f64, MVT::v16f32, Expand); 375 setTruncStoreAction(MVT::v16f64, MVT::v16bf16, Expand); 376 setTruncStoreAction(MVT::v16f64, MVT::v16f16, Expand); 377 setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); 378 setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); 379 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand); 380 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand); 381 setTruncStoreAction(MVT::v16i64, MVT::v16i1, Expand); 382 383 setOperationAction(ISD::Constant, {MVT::i32, MVT::i64}, Legal); 384 setOperationAction(ISD::ConstantFP, {MVT::f32, MVT::f64}, Legal); 385 386 setOperationAction({ISD::BR_JT, ISD::BRIND}, MVT::Other, Expand); 387 388 // For R600, this is totally unsupported, just custom lower to produce an 389 // error. 390 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 391 392 // Library functions. These default to Expand, but we have instructions 393 // for them. 394 setOperationAction({ISD::FCEIL, ISD::FPOW, ISD::FABS, ISD::FFLOOR, 395 ISD::FROUNDEVEN, ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM}, 396 MVT::f32, Legal); 397 398 setOperationAction(ISD::FLOG2, MVT::f32, Custom); 399 setOperationAction(ISD::FROUND, {MVT::f32, MVT::f64}, Custom); 400 401 setOperationAction( 402 {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, MVT::f32, 403 Custom); 404 405 setOperationAction(ISD::FNEARBYINT, {MVT::f16, MVT::f32, MVT::f64}, Custom); 406 407 setOperationAction(ISD::FRINT, {MVT::f16, MVT::f32, MVT::f64}, Custom); 408 409 setOperationAction(ISD::FREM, {MVT::f16, MVT::f32, MVT::f64}, Custom); 410 411 if (Subtarget->has16BitInsts()) 412 setOperationAction(ISD::IS_FPCLASS, {MVT::f16, MVT::f32, MVT::f64}, Legal); 413 else { 414 setOperationAction(ISD::IS_FPCLASS, {MVT::f32, MVT::f64}, Legal); 415 setOperationAction({ISD::FLOG2, ISD::FEXP2}, MVT::f16, Custom); 416 } 417 418 setOperationAction({ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, MVT::f16, 419 Custom); 420 421 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches 422 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by 423 // default unless marked custom/legal. 424 setOperationAction( 425 ISD::IS_FPCLASS, 426 {MVT::v2f16, MVT::v3f16, MVT::v4f16, MVT::v16f16, MVT::v2f32, MVT::v3f32, 427 MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32, 428 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64, MVT::v16f64}, 429 Custom); 430 431 // Expand to fneg + fadd. 432 setOperationAction(ISD::FSUB, MVT::f64, Expand); 433 434 setOperationAction(ISD::CONCAT_VECTORS, 435 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32, 436 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32, 437 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32, 438 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32, 439 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32}, 440 Custom); 441 442 // FIXME: Why is v8f16/v8bf16 missing? 443 setOperationAction( 444 ISD::EXTRACT_SUBVECTOR, 445 {MVT::v2f16, MVT::v2bf16, MVT::v2i16, MVT::v4f16, MVT::v4bf16, 446 MVT::v4i16, MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, 447 MVT::v4f32, MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, 448 MVT::v6i32, MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, 449 MVT::v9f32, MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, 450 MVT::v11f32, MVT::v12i32, MVT::v12f32, MVT::v16f16, MVT::v16bf16, 451 MVT::v16i16, MVT::v16f32, MVT::v16i32, MVT::v32f32, MVT::v32i32, 452 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64, 453 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64, 454 MVT::v32i16, MVT::v32f16, MVT::v32bf16}, 455 Custom); 456 457 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); 458 setOperationAction(ISD::FP_TO_FP16, {MVT::f64, MVT::f32}, Custom); 459 460 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 461 for (MVT VT : ScalarIntVTs) { 462 // These should use [SU]DIVREM, so set them to expand 463 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT, 464 Expand); 465 466 // GPU does not have divrem function for signed or unsigned. 467 setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, VT, Custom); 468 469 // GPU does not have [S|U]MUL_LOHI functions as a single instruction. 470 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand); 471 472 setOperationAction({ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Expand); 473 474 // AMDGPU uses ADDC/SUBC/ADDE/SUBE 475 setOperationAction({ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT, Legal); 476 } 477 478 // The hardware supports 32-bit FSHR, but not FSHL. 479 setOperationAction(ISD::FSHR, MVT::i32, Legal); 480 481 // The hardware supports 32-bit ROTR, but not ROTL. 482 setOperationAction(ISD::ROTL, {MVT::i32, MVT::i64}, Expand); 483 setOperationAction(ISD::ROTR, MVT::i64, Expand); 484 485 setOperationAction({ISD::MULHU, ISD::MULHS}, MVT::i16, Expand); 486 487 setOperationAction({ISD::MUL, ISD::MULHU, ISD::MULHS}, MVT::i64, Expand); 488 setOperationAction( 489 {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}, 490 MVT::i64, Custom); 491 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); 492 493 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, MVT::i32, 494 Legal); 495 496 setOperationAction( 497 {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, 498 MVT::i64, Custom); 499 500 for (auto VT : {MVT::i8, MVT::i16}) 501 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, VT, Custom); 502 503 static const MVT::SimpleValueType VectorIntTypes[] = { 504 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32, 505 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32}; 506 507 for (MVT VT : VectorIntTypes) { 508 // Expand the following operations for the current type by default. 509 setOperationAction({ISD::ADD, ISD::AND, ISD::FP_TO_SINT, 510 ISD::FP_TO_UINT, ISD::MUL, ISD::MULHU, 511 ISD::MULHS, ISD::OR, ISD::SHL, 512 ISD::SRA, ISD::SRL, ISD::ROTL, 513 ISD::ROTR, ISD::SUB, ISD::SINT_TO_FP, 514 ISD::UINT_TO_FP, ISD::SDIV, ISD::UDIV, 515 ISD::SREM, ISD::UREM, ISD::SMUL_LOHI, 516 ISD::UMUL_LOHI, ISD::SDIVREM, ISD::UDIVREM, 517 ISD::SELECT, ISD::VSELECT, ISD::SELECT_CC, 518 ISD::XOR, ISD::BSWAP, ISD::CTPOP, 519 ISD::CTTZ, ISD::CTLZ, ISD::VECTOR_SHUFFLE, 520 ISD::SETCC}, 521 VT, Expand); 522 } 523 524 static const MVT::SimpleValueType FloatVectorTypes[] = { 525 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32, 526 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32}; 527 528 for (MVT VT : FloatVectorTypes) { 529 setOperationAction( 530 {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM, 531 ISD::FADD, ISD::FCEIL, ISD::FCOS, 532 ISD::FDIV, ISD::FEXP2, ISD::FEXP, 533 ISD::FEXP10, ISD::FLOG2, ISD::FREM, 534 ISD::FLOG, ISD::FLOG10, ISD::FPOW, 535 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL, 536 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT, 537 ISD::FSQRT, ISD::FSIN, ISD::FSUB, 538 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC, 539 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC, 540 ISD::FCANONICALIZE, ISD::FROUNDEVEN}, 541 VT, Expand); 542 } 543 544 // This causes using an unrolled select operation rather than expansion with 545 // bit operations. This is in general better, but the alternative using BFI 546 // instructions may be better if the select sources are SGPRs. 547 setOperationAction(ISD::SELECT, MVT::v2f32, Promote); 548 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32); 549 550 setOperationAction(ISD::SELECT, MVT::v3f32, Promote); 551 AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32); 552 553 setOperationAction(ISD::SELECT, MVT::v4f32, Promote); 554 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32); 555 556 setOperationAction(ISD::SELECT, MVT::v5f32, Promote); 557 AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32); 558 559 setOperationAction(ISD::SELECT, MVT::v6f32, Promote); 560 AddPromotedToType(ISD::SELECT, MVT::v6f32, MVT::v6i32); 561 562 setOperationAction(ISD::SELECT, MVT::v7f32, Promote); 563 AddPromotedToType(ISD::SELECT, MVT::v7f32, MVT::v7i32); 564 565 setOperationAction(ISD::SELECT, MVT::v9f32, Promote); 566 AddPromotedToType(ISD::SELECT, MVT::v9f32, MVT::v9i32); 567 568 setOperationAction(ISD::SELECT, MVT::v10f32, Promote); 569 AddPromotedToType(ISD::SELECT, MVT::v10f32, MVT::v10i32); 570 571 setOperationAction(ISD::SELECT, MVT::v11f32, Promote); 572 AddPromotedToType(ISD::SELECT, MVT::v11f32, MVT::v11i32); 573 574 setOperationAction(ISD::SELECT, MVT::v12f32, Promote); 575 AddPromotedToType(ISD::SELECT, MVT::v12f32, MVT::v12i32); 576 577 setSchedulingPreference(Sched::RegPressure); 578 setJumpIsExpensive(true); 579 580 // FIXME: This is only partially true. If we have to do vector compares, any 581 // SGPR pair can be a condition register. If we have a uniform condition, we 582 // are better off doing SALU operations, where there is only one SCC. For now, 583 // we don't have a way of knowing during instruction selection if a condition 584 // will be uniform and we always use vector compares. Assume we are using 585 // vector compares until that is fixed. 586 setHasMultipleConditionRegisters(true); 587 588 setMinCmpXchgSizeInBits(32); 589 setSupportsUnalignedAtomics(false); 590 591 PredictableSelectIsExpensive = false; 592 593 // We want to find all load dependencies for long chains of stores to enable 594 // merging into very wide vectors. The problem is with vectors with > 4 595 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16 596 // vectors are a legal type, even though we have to split the loads 597 // usually. When we can more precisely specify load legality per address 598 // space, we should be able to make FindBetterChain/MergeConsecutiveStores 599 // smarter so that they can figure out what to do in 2 iterations without all 600 // N > 4 stores on the same chain. 601 GatherAllAliasesMaxDepth = 16; 602 603 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry 604 // about these during lowering. 605 MaxStoresPerMemcpy = 0xffffffff; 606 MaxStoresPerMemmove = 0xffffffff; 607 MaxStoresPerMemset = 0xffffffff; 608 609 // The expansion for 64-bit division is enormous. 610 if (AMDGPUBypassSlowDiv) 611 addBypassSlowDiv(64, 32); 612 613 setTargetDAGCombine({ISD::BITCAST, ISD::SHL, 614 ISD::SRA, ISD::SRL, 615 ISD::TRUNCATE, ISD::MUL, 616 ISD::SMUL_LOHI, ISD::UMUL_LOHI, 617 ISD::MULHU, ISD::MULHS, 618 ISD::SELECT, ISD::SELECT_CC, 619 ISD::STORE, ISD::FADD, 620 ISD::FSUB, ISD::FNEG, 621 ISD::FABS, ISD::AssertZext, 622 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN}); 623 624 setMaxAtomicSizeInBitsSupported(64); 625 setMaxDivRemBitWidthSupported(64); 626 setMaxLargeFPConvertBitWidthSupported(64); 627 } 628 629 bool AMDGPUTargetLowering::mayIgnoreSignedZero(SDValue Op) const { 630 if (getTargetMachine().Options.NoSignedZerosFPMath) 631 return true; 632 633 const auto Flags = Op.getNode()->getFlags(); 634 if (Flags.hasNoSignedZeros()) 635 return true; 636 637 return false; 638 } 639 640 //===----------------------------------------------------------------------===// 641 // Target Information 642 //===----------------------------------------------------------------------===// 643 644 LLVM_READNONE 645 static bool fnegFoldsIntoOpcode(unsigned Opc) { 646 switch (Opc) { 647 case ISD::FADD: 648 case ISD::FSUB: 649 case ISD::FMUL: 650 case ISD::FMA: 651 case ISD::FMAD: 652 case ISD::FMINNUM: 653 case ISD::FMAXNUM: 654 case ISD::FMINNUM_IEEE: 655 case ISD::FMAXNUM_IEEE: 656 case ISD::FMINIMUM: 657 case ISD::FMAXIMUM: 658 case ISD::SELECT: 659 case ISD::FSIN: 660 case ISD::FTRUNC: 661 case ISD::FRINT: 662 case ISD::FNEARBYINT: 663 case ISD::FROUNDEVEN: 664 case ISD::FCANONICALIZE: 665 case AMDGPUISD::RCP: 666 case AMDGPUISD::RCP_LEGACY: 667 case AMDGPUISD::RCP_IFLAG: 668 case AMDGPUISD::SIN_HW: 669 case AMDGPUISD::FMUL_LEGACY: 670 case AMDGPUISD::FMIN_LEGACY: 671 case AMDGPUISD::FMAX_LEGACY: 672 case AMDGPUISD::FMED3: 673 // TODO: handle llvm.amdgcn.fma.legacy 674 return true; 675 case ISD::BITCAST: 676 llvm_unreachable("bitcast is special cased"); 677 default: 678 return false; 679 } 680 } 681 682 static bool fnegFoldsIntoOp(const SDNode *N) { 683 unsigned Opc = N->getOpcode(); 684 if (Opc == ISD::BITCAST) { 685 // TODO: Is there a benefit to checking the conditions performFNegCombine 686 // does? We don't for the other cases. 687 SDValue BCSrc = N->getOperand(0); 688 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) { 689 return BCSrc.getNumOperands() == 2 && 690 BCSrc.getOperand(1).getValueSizeInBits() == 32; 691 } 692 693 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32; 694 } 695 696 return fnegFoldsIntoOpcode(Opc); 697 } 698 699 /// \p returns true if the operation will definitely need to use a 64-bit 700 /// encoding, and thus will use a VOP3 encoding regardless of the source 701 /// modifiers. 702 LLVM_READONLY 703 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) { 704 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) || 705 VT == MVT::f64; 706 } 707 708 /// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the 709 /// type for ISD::SELECT. 710 LLVM_READONLY 711 static bool selectSupportsSourceMods(const SDNode *N) { 712 // TODO: Only applies if select will be vector 713 return N->getValueType(0) == MVT::f32; 714 } 715 716 // Most FP instructions support source modifiers, but this could be refined 717 // slightly. 718 LLVM_READONLY 719 static bool hasSourceMods(const SDNode *N) { 720 if (isa<MemSDNode>(N)) 721 return false; 722 723 switch (N->getOpcode()) { 724 case ISD::CopyToReg: 725 case ISD::FDIV: 726 case ISD::FREM: 727 case ISD::INLINEASM: 728 case ISD::INLINEASM_BR: 729 case AMDGPUISD::DIV_SCALE: 730 case ISD::INTRINSIC_W_CHAIN: 731 732 // TODO: Should really be looking at the users of the bitcast. These are 733 // problematic because bitcasts are used to legalize all stores to integer 734 // types. 735 case ISD::BITCAST: 736 return false; 737 case ISD::INTRINSIC_WO_CHAIN: { 738 switch (N->getConstantOperandVal(0)) { 739 case Intrinsic::amdgcn_interp_p1: 740 case Intrinsic::amdgcn_interp_p2: 741 case Intrinsic::amdgcn_interp_mov: 742 case Intrinsic::amdgcn_interp_p1_f16: 743 case Intrinsic::amdgcn_interp_p2_f16: 744 return false; 745 default: 746 return true; 747 } 748 } 749 case ISD::SELECT: 750 return selectSupportsSourceMods(N); 751 default: 752 return true; 753 } 754 } 755 756 bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N, 757 unsigned CostThreshold) { 758 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus 759 // it is truly free to use a source modifier in all cases. If there are 760 // multiple users but for each one will necessitate using VOP3, there will be 761 // a code size increase. Try to avoid increasing code size unless we know it 762 // will save on the instruction count. 763 unsigned NumMayIncreaseSize = 0; 764 MVT VT = N->getValueType(0).getScalarType().getSimpleVT(); 765 766 assert(!N->use_empty()); 767 768 // XXX - Should this limit number of uses to check? 769 for (const SDNode *U : N->uses()) { 770 if (!hasSourceMods(U)) 771 return false; 772 773 if (!opMustUseVOP3Encoding(U, VT)) { 774 if (++NumMayIncreaseSize > CostThreshold) 775 return false; 776 } 777 } 778 779 return true; 780 } 781 782 EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT, 783 ISD::NodeType ExtendKind) const { 784 assert(!VT.isVector() && "only scalar expected"); 785 786 // Round to the next multiple of 32-bits. 787 unsigned Size = VT.getSizeInBits(); 788 if (Size <= 32) 789 return MVT::i32; 790 return EVT::getIntegerVT(Context, 32 * ((Size + 31) / 32)); 791 } 792 793 MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const { 794 return MVT::i32; 795 } 796 797 bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const { 798 return true; 799 } 800 801 // The backend supports 32 and 64 bit floating point immediates. 802 // FIXME: Why are we reporting vectors of FP immediates as legal? 803 bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, 804 bool ForCodeSize) const { 805 EVT ScalarVT = VT.getScalarType(); 806 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 || 807 (ScalarVT == MVT::f16 && Subtarget->has16BitInsts())); 808 } 809 810 // We don't want to shrink f64 / f32 constants. 811 bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const { 812 EVT ScalarVT = VT.getScalarType(); 813 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64); 814 } 815 816 bool AMDGPUTargetLowering::shouldReduceLoadWidth(SDNode *N, 817 ISD::LoadExtType ExtTy, 818 EVT NewVT) const { 819 // TODO: This may be worth removing. Check regression tests for diffs. 820 if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT)) 821 return false; 822 823 unsigned NewSize = NewVT.getStoreSizeInBits(); 824 825 // If we are reducing to a 32-bit load or a smaller multi-dword load, 826 // this is always better. 827 if (NewSize >= 32) 828 return true; 829 830 EVT OldVT = N->getValueType(0); 831 unsigned OldSize = OldVT.getStoreSizeInBits(); 832 833 MemSDNode *MN = cast<MemSDNode>(N); 834 unsigned AS = MN->getAddressSpace(); 835 // Do not shrink an aligned scalar load to sub-dword. 836 // Scalar engine cannot do sub-dword loads. 837 // TODO: Update this for GFX12 which does have scalar sub-dword loads. 838 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) && 839 (AS == AMDGPUAS::CONSTANT_ADDRESS || 840 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT || 841 (isa<LoadSDNode>(N) && AS == AMDGPUAS::GLOBAL_ADDRESS && 842 MN->isInvariant())) && 843 AMDGPUInstrInfo::isUniformMMO(MN->getMemOperand())) 844 return false; 845 846 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar 847 // extloads, so doing one requires using a buffer_load. In cases where we 848 // still couldn't use a scalar load, using the wider load shouldn't really 849 // hurt anything. 850 851 // If the old size already had to be an extload, there's no harm in continuing 852 // to reduce the width. 853 return (OldSize < 32); 854 } 855 856 bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy, 857 const SelectionDAG &DAG, 858 const MachineMemOperand &MMO) const { 859 860 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits()); 861 862 if (LoadTy.getScalarType() == MVT::i32) 863 return false; 864 865 unsigned LScalarSize = LoadTy.getScalarSizeInBits(); 866 unsigned CastScalarSize = CastTy.getScalarSizeInBits(); 867 868 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32)) 869 return false; 870 871 unsigned Fast = 0; 872 return allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(), 873 CastTy, MMO, &Fast) && 874 Fast; 875 } 876 877 // SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also 878 // profitable with the expansion for 64-bit since it's generally good to 879 // speculate things. 880 bool AMDGPUTargetLowering::isCheapToSpeculateCttz(Type *Ty) const { 881 return true; 882 } 883 884 bool AMDGPUTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const { 885 return true; 886 } 887 888 bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const { 889 switch (N->getOpcode()) { 890 case ISD::EntryToken: 891 case ISD::TokenFactor: 892 return true; 893 case ISD::INTRINSIC_WO_CHAIN: { 894 unsigned IntrID = N->getConstantOperandVal(0); 895 return AMDGPU::isIntrinsicAlwaysUniform(IntrID); 896 } 897 case ISD::LOAD: 898 if (cast<LoadSDNode>(N)->getMemOperand()->getAddrSpace() == 899 AMDGPUAS::CONSTANT_ADDRESS_32BIT) 900 return true; 901 return false; 902 case AMDGPUISD::SETCC: // ballot-style instruction 903 return true; 904 } 905 return false; 906 } 907 908 SDValue AMDGPUTargetLowering::getNegatedExpression( 909 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, 910 NegatibleCost &Cost, unsigned Depth) const { 911 912 switch (Op.getOpcode()) { 913 case ISD::FMA: 914 case ISD::FMAD: { 915 // Negating a fma is not free if it has users without source mods. 916 if (!allUsesHaveSourceMods(Op.getNode())) 917 return SDValue(); 918 break; 919 } 920 case AMDGPUISD::RCP: { 921 SDValue Src = Op.getOperand(0); 922 EVT VT = Op.getValueType(); 923 SDLoc SL(Op); 924 925 SDValue NegSrc = getNegatedExpression(Src, DAG, LegalOperations, 926 ForCodeSize, Cost, Depth + 1); 927 if (NegSrc) 928 return DAG.getNode(AMDGPUISD::RCP, SL, VT, NegSrc, Op->getFlags()); 929 return SDValue(); 930 } 931 default: 932 break; 933 } 934 935 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations, 936 ForCodeSize, Cost, Depth); 937 } 938 939 //===---------------------------------------------------------------------===// 940 // Target Properties 941 //===---------------------------------------------------------------------===// 942 943 bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { 944 assert(VT.isFloatingPoint()); 945 946 // Packed operations do not have a fabs modifier. 947 return VT == MVT::f32 || VT == MVT::f64 || 948 (Subtarget->has16BitInsts() && (VT == MVT::f16 || VT == MVT::bf16)); 949 } 950 951 bool AMDGPUTargetLowering::isFNegFree(EVT VT) const { 952 assert(VT.isFloatingPoint()); 953 // Report this based on the end legalized type. 954 VT = VT.getScalarType(); 955 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16; 956 } 957 958 bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, 959 unsigned NumElem, 960 unsigned AS) const { 961 return true; 962 } 963 964 bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const { 965 // There are few operations which truly have vector input operands. Any vector 966 // operation is going to involve operations on each component, and a 967 // build_vector will be a copy per element, so it always makes sense to use a 968 // build_vector input in place of the extracted element to avoid a copy into a 969 // super register. 970 // 971 // We should probably only do this if all users are extracts only, but this 972 // should be the common case. 973 return true; 974 } 975 976 bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const { 977 // Truncate is just accessing a subregister. 978 979 unsigned SrcSize = Source.getSizeInBits(); 980 unsigned DestSize = Dest.getSizeInBits(); 981 982 return DestSize < SrcSize && DestSize % 32 == 0 ; 983 } 984 985 bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const { 986 // Truncate is just accessing a subregister. 987 988 unsigned SrcSize = Source->getScalarSizeInBits(); 989 unsigned DestSize = Dest->getScalarSizeInBits(); 990 991 if (DestSize== 16 && Subtarget->has16BitInsts()) 992 return SrcSize >= 32; 993 994 return DestSize < SrcSize && DestSize % 32 == 0; 995 } 996 997 bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const { 998 unsigned SrcSize = Src->getScalarSizeInBits(); 999 unsigned DestSize = Dest->getScalarSizeInBits(); 1000 1001 if (SrcSize == 16 && Subtarget->has16BitInsts()) 1002 return DestSize >= 32; 1003 1004 return SrcSize == 32 && DestSize == 64; 1005 } 1006 1007 bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const { 1008 // Any register load of a 64-bit value really requires 2 32-bit moves. For all 1009 // practical purposes, the extra mov 0 to load a 64-bit is free. As used, 1010 // this will enable reducing 64-bit operations the 32-bit, which is always 1011 // good. 1012 1013 if (Src == MVT::i16) 1014 return Dest == MVT::i32 ||Dest == MVT::i64 ; 1015 1016 return Src == MVT::i32 && Dest == MVT::i64; 1017 } 1018 1019 bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const { 1020 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a 1021 // limited number of native 64-bit operations. Shrinking an operation to fit 1022 // in a single 32-bit register should always be helpful. As currently used, 1023 // this is much less general than the name suggests, and is only used in 1024 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is 1025 // not profitable, and may actually be harmful. 1026 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32; 1027 } 1028 1029 bool AMDGPUTargetLowering::isDesirableToCommuteWithShift( 1030 const SDNode* N, CombineLevel Level) const { 1031 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA || 1032 N->getOpcode() == ISD::SRL) && 1033 "Expected shift op"); 1034 // Always commute pre-type legalization and right shifts. 1035 // We're looking for shl(or(x,y),z) patterns. 1036 if (Level < CombineLevel::AfterLegalizeTypes || 1037 N->getOpcode() != ISD::SHL || N->getOperand(0).getOpcode() != ISD::OR) 1038 return true; 1039 1040 // If only user is a i32 right-shift, then don't destroy a BFE pattern. 1041 if (N->getValueType(0) == MVT::i32 && N->use_size() == 1 && 1042 (N->use_begin()->getOpcode() == ISD::SRA || 1043 N->use_begin()->getOpcode() == ISD::SRL)) 1044 return false; 1045 1046 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns. 1047 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) { 1048 if (LHS.getOpcode() != ISD::SHL) 1049 return false; 1050 auto *RHSLd = dyn_cast<LoadSDNode>(RHS); 1051 auto *LHS0 = dyn_cast<LoadSDNode>(LHS.getOperand(0)); 1052 auto *LHS1 = dyn_cast<ConstantSDNode>(LHS.getOperand(1)); 1053 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD && 1054 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() && 1055 RHSLd->getExtensionType() == ISD::ZEXTLOAD; 1056 }; 1057 SDValue LHS = N->getOperand(0).getOperand(0); 1058 SDValue RHS = N->getOperand(0).getOperand(1); 1059 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS)); 1060 } 1061 1062 //===---------------------------------------------------------------------===// 1063 // TargetLowering Callbacks 1064 //===---------------------------------------------------------------------===// 1065 1066 CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC, 1067 bool IsVarArg) { 1068 switch (CC) { 1069 case CallingConv::AMDGPU_VS: 1070 case CallingConv::AMDGPU_GS: 1071 case CallingConv::AMDGPU_PS: 1072 case CallingConv::AMDGPU_CS: 1073 case CallingConv::AMDGPU_HS: 1074 case CallingConv::AMDGPU_ES: 1075 case CallingConv::AMDGPU_LS: 1076 return CC_AMDGPU; 1077 case CallingConv::AMDGPU_CS_Chain: 1078 case CallingConv::AMDGPU_CS_ChainPreserve: 1079 return CC_AMDGPU_CS_CHAIN; 1080 case CallingConv::C: 1081 case CallingConv::Fast: 1082 case CallingConv::Cold: 1083 return CC_AMDGPU_Func; 1084 case CallingConv::AMDGPU_Gfx: 1085 return CC_SI_Gfx; 1086 case CallingConv::AMDGPU_KERNEL: 1087 case CallingConv::SPIR_KERNEL: 1088 default: 1089 report_fatal_error("Unsupported calling convention for call"); 1090 } 1091 } 1092 1093 CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC, 1094 bool IsVarArg) { 1095 switch (CC) { 1096 case CallingConv::AMDGPU_KERNEL: 1097 case CallingConv::SPIR_KERNEL: 1098 llvm_unreachable("kernels should not be handled here"); 1099 case CallingConv::AMDGPU_VS: 1100 case CallingConv::AMDGPU_GS: 1101 case CallingConv::AMDGPU_PS: 1102 case CallingConv::AMDGPU_CS: 1103 case CallingConv::AMDGPU_CS_Chain: 1104 case CallingConv::AMDGPU_CS_ChainPreserve: 1105 case CallingConv::AMDGPU_HS: 1106 case CallingConv::AMDGPU_ES: 1107 case CallingConv::AMDGPU_LS: 1108 return RetCC_SI_Shader; 1109 case CallingConv::AMDGPU_Gfx: 1110 return RetCC_SI_Gfx; 1111 case CallingConv::C: 1112 case CallingConv::Fast: 1113 case CallingConv::Cold: 1114 return RetCC_AMDGPU_Func; 1115 default: 1116 report_fatal_error("Unsupported calling convention."); 1117 } 1118 } 1119 1120 /// The SelectionDAGBuilder will automatically promote function arguments 1121 /// with illegal types. However, this does not work for the AMDGPU targets 1122 /// since the function arguments are stored in memory as these illegal types. 1123 /// In order to handle this properly we need to get the original types sizes 1124 /// from the LLVM IR Function and fixup the ISD:InputArg values before 1125 /// passing them to AnalyzeFormalArguments() 1126 1127 /// When the SelectionDAGBuilder computes the Ins, it takes care of splitting 1128 /// input values across multiple registers. Each item in the Ins array 1129 /// represents a single value that will be stored in registers. Ins[x].VT is 1130 /// the value type of the value that will be stored in the register, so 1131 /// whatever SDNode we lower the argument to needs to be this type. 1132 /// 1133 /// In order to correctly lower the arguments we need to know the size of each 1134 /// argument. Since Ins[x].VT gives us the size of the register that will 1135 /// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type 1136 /// for the original function argument so that we can deduce the correct memory 1137 /// type to use for Ins[x]. In most cases the correct memory type will be 1138 /// Ins[x].ArgVT. However, this will not always be the case. If, for example, 1139 /// we have a kernel argument of type v8i8, this argument will be split into 1140 /// 8 parts and each part will be represented by its own item in the Ins array. 1141 /// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of 1142 /// the argument before it was split. From this, we deduce that the memory type 1143 /// for each individual part is i8. We pass the memory type as LocVT to the 1144 /// calling convention analysis function and the register type (Ins[x].VT) as 1145 /// the ValVT. 1146 void AMDGPUTargetLowering::analyzeFormalArgumentsCompute( 1147 CCState &State, 1148 const SmallVectorImpl<ISD::InputArg> &Ins) const { 1149 const MachineFunction &MF = State.getMachineFunction(); 1150 const Function &Fn = MF.getFunction(); 1151 LLVMContext &Ctx = Fn.getParent()->getContext(); 1152 const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(MF); 1153 const unsigned ExplicitOffset = ST.getExplicitKernelArgOffset(); 1154 CallingConv::ID CC = Fn.getCallingConv(); 1155 1156 Align MaxAlign = Align(1); 1157 uint64_t ExplicitArgOffset = 0; 1158 const DataLayout &DL = Fn.getDataLayout(); 1159 1160 unsigned InIndex = 0; 1161 1162 for (const Argument &Arg : Fn.args()) { 1163 const bool IsByRef = Arg.hasByRefAttr(); 1164 Type *BaseArgTy = Arg.getType(); 1165 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy; 1166 Align Alignment = DL.getValueOrABITypeAlignment( 1167 IsByRef ? Arg.getParamAlign() : std::nullopt, MemArgTy); 1168 MaxAlign = std::max(Alignment, MaxAlign); 1169 uint64_t AllocSize = DL.getTypeAllocSize(MemArgTy); 1170 1171 uint64_t ArgOffset = alignTo(ExplicitArgOffset, Alignment) + ExplicitOffset; 1172 ExplicitArgOffset = alignTo(ExplicitArgOffset, Alignment) + AllocSize; 1173 1174 // We're basically throwing away everything passed into us and starting over 1175 // to get accurate in-memory offsets. The "PartOffset" is completely useless 1176 // to us as computed in Ins. 1177 // 1178 // We also need to figure out what type legalization is trying to do to get 1179 // the correct memory offsets. 1180 1181 SmallVector<EVT, 16> ValueVTs; 1182 SmallVector<uint64_t, 16> Offsets; 1183 ComputeValueVTs(*this, DL, BaseArgTy, ValueVTs, &Offsets, ArgOffset); 1184 1185 for (unsigned Value = 0, NumValues = ValueVTs.size(); 1186 Value != NumValues; ++Value) { 1187 uint64_t BasePartOffset = Offsets[Value]; 1188 1189 EVT ArgVT = ValueVTs[Value]; 1190 EVT MemVT = ArgVT; 1191 MVT RegisterVT = getRegisterTypeForCallingConv(Ctx, CC, ArgVT); 1192 unsigned NumRegs = getNumRegistersForCallingConv(Ctx, CC, ArgVT); 1193 1194 if (NumRegs == 1) { 1195 // This argument is not split, so the IR type is the memory type. 1196 if (ArgVT.isExtended()) { 1197 // We have an extended type, like i24, so we should just use the 1198 // register type. 1199 MemVT = RegisterVT; 1200 } else { 1201 MemVT = ArgVT; 1202 } 1203 } else if (ArgVT.isVector() && RegisterVT.isVector() && 1204 ArgVT.getScalarType() == RegisterVT.getScalarType()) { 1205 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements()); 1206 // We have a vector value which has been split into a vector with 1207 // the same scalar type, but fewer elements. This should handle 1208 // all the floating-point vector types. 1209 MemVT = RegisterVT; 1210 } else if (ArgVT.isVector() && 1211 ArgVT.getVectorNumElements() == NumRegs) { 1212 // This arg has been split so that each element is stored in a separate 1213 // register. 1214 MemVT = ArgVT.getScalarType(); 1215 } else if (ArgVT.isExtended()) { 1216 // We have an extended type, like i65. 1217 MemVT = RegisterVT; 1218 } else { 1219 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs; 1220 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0); 1221 if (RegisterVT.isInteger()) { 1222 MemVT = EVT::getIntegerVT(State.getContext(), MemoryBits); 1223 } else if (RegisterVT.isVector()) { 1224 assert(!RegisterVT.getScalarType().isFloatingPoint()); 1225 unsigned NumElements = RegisterVT.getVectorNumElements(); 1226 assert(MemoryBits % NumElements == 0); 1227 // This vector type has been split into another vector type with 1228 // a different elements size. 1229 EVT ScalarVT = EVT::getIntegerVT(State.getContext(), 1230 MemoryBits / NumElements); 1231 MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements); 1232 } else { 1233 llvm_unreachable("cannot deduce memory type."); 1234 } 1235 } 1236 1237 // Convert one element vectors to scalar. 1238 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1) 1239 MemVT = MemVT.getScalarType(); 1240 1241 // Round up vec3/vec5 argument. 1242 if (MemVT.isVector() && !MemVT.isPow2VectorType()) { 1243 assert(MemVT.getVectorNumElements() == 3 || 1244 MemVT.getVectorNumElements() == 5 || 1245 (MemVT.getVectorNumElements() >= 9 && 1246 MemVT.getVectorNumElements() <= 12)); 1247 MemVT = MemVT.getPow2VectorType(State.getContext()); 1248 } else if (!MemVT.isSimple() && !MemVT.isVector()) { 1249 MemVT = MemVT.getRoundIntegerType(State.getContext()); 1250 } 1251 1252 unsigned PartOffset = 0; 1253 for (unsigned i = 0; i != NumRegs; ++i) { 1254 State.addLoc(CCValAssign::getCustomMem(InIndex++, RegisterVT, 1255 BasePartOffset + PartOffset, 1256 MemVT.getSimpleVT(), 1257 CCValAssign::Full)); 1258 PartOffset += MemVT.getStoreSize(); 1259 } 1260 } 1261 } 1262 } 1263 1264 SDValue AMDGPUTargetLowering::LowerReturn( 1265 SDValue Chain, CallingConv::ID CallConv, 1266 bool isVarArg, 1267 const SmallVectorImpl<ISD::OutputArg> &Outs, 1268 const SmallVectorImpl<SDValue> &OutVals, 1269 const SDLoc &DL, SelectionDAG &DAG) const { 1270 // FIXME: Fails for r600 tests 1271 //assert(!isVarArg && Outs.empty() && OutVals.empty() && 1272 // "wave terminate should not have return values"); 1273 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain); 1274 } 1275 1276 //===---------------------------------------------------------------------===// 1277 // Target specific lowering 1278 //===---------------------------------------------------------------------===// 1279 1280 /// Selects the correct CCAssignFn for a given CallingConvention value. 1281 CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC, 1282 bool IsVarArg) { 1283 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg); 1284 } 1285 1286 CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC, 1287 bool IsVarArg) { 1288 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg); 1289 } 1290 1291 SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain, 1292 SelectionDAG &DAG, 1293 MachineFrameInfo &MFI, 1294 int ClobberedFI) const { 1295 SmallVector<SDValue, 8> ArgChains; 1296 int64_t FirstByte = MFI.getObjectOffset(ClobberedFI); 1297 int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1; 1298 1299 // Include the original chain at the beginning of the list. When this is 1300 // used by target LowerCall hooks, this helps legalize find the 1301 // CALLSEQ_BEGIN node. 1302 ArgChains.push_back(Chain); 1303 1304 // Add a chain value for each stack argument corresponding 1305 for (SDNode *U : DAG.getEntryNode().getNode()->uses()) { 1306 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U)) { 1307 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) { 1308 if (FI->getIndex() < 0) { 1309 int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex()); 1310 int64_t InLastByte = InFirstByte; 1311 InLastByte += MFI.getObjectSize(FI->getIndex()) - 1; 1312 1313 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) || 1314 (FirstByte <= InFirstByte && InFirstByte <= LastByte)) 1315 ArgChains.push_back(SDValue(L, 1)); 1316 } 1317 } 1318 } 1319 } 1320 1321 // Build a tokenfactor for all the chains. 1322 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); 1323 } 1324 1325 SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI, 1326 SmallVectorImpl<SDValue> &InVals, 1327 StringRef Reason) const { 1328 SDValue Callee = CLI.Callee; 1329 SelectionDAG &DAG = CLI.DAG; 1330 1331 const Function &Fn = DAG.getMachineFunction().getFunction(); 1332 1333 StringRef FuncName("<unknown>"); 1334 1335 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee)) 1336 FuncName = G->getSymbol(); 1337 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 1338 FuncName = G->getGlobal()->getName(); 1339 1340 DiagnosticInfoUnsupported NoCalls( 1341 Fn, Reason + FuncName, CLI.DL.getDebugLoc()); 1342 DAG.getContext()->diagnose(NoCalls); 1343 1344 if (!CLI.IsTailCall) { 1345 for (ISD::InputArg &Arg : CLI.Ins) 1346 InVals.push_back(DAG.getUNDEF(Arg.VT)); 1347 } 1348 1349 return DAG.getEntryNode(); 1350 } 1351 1352 SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI, 1353 SmallVectorImpl<SDValue> &InVals) const { 1354 return lowerUnhandledCall(CLI, InVals, "unsupported call to function "); 1355 } 1356 1357 SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 1358 SelectionDAG &DAG) const { 1359 const Function &Fn = DAG.getMachineFunction().getFunction(); 1360 1361 DiagnosticInfoUnsupported NoDynamicAlloca(Fn, "unsupported dynamic alloca", 1362 SDLoc(Op).getDebugLoc()); 1363 DAG.getContext()->diagnose(NoDynamicAlloca); 1364 auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)}; 1365 return DAG.getMergeValues(Ops, SDLoc()); 1366 } 1367 1368 SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, 1369 SelectionDAG &DAG) const { 1370 switch (Op.getOpcode()) { 1371 default: 1372 Op->print(errs(), &DAG); 1373 llvm_unreachable("Custom lowering code for this " 1374 "instruction is not implemented yet!"); 1375 break; 1376 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 1377 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 1378 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); 1379 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); 1380 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG); 1381 case ISD::FREM: return LowerFREM(Op, DAG); 1382 case ISD::FCEIL: return LowerFCEIL(Op, DAG); 1383 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG); 1384 case ISD::FRINT: return LowerFRINT(Op, DAG); 1385 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG); 1386 case ISD::FROUNDEVEN: 1387 return LowerFROUNDEVEN(Op, DAG); 1388 case ISD::FROUND: return LowerFROUND(Op, DAG); 1389 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG); 1390 case ISD::FLOG2: 1391 return LowerFLOG2(Op, DAG); 1392 case ISD::FLOG: 1393 case ISD::FLOG10: 1394 return LowerFLOGCommon(Op, DAG); 1395 case ISD::FEXP: 1396 case ISD::FEXP10: 1397 return lowerFEXP(Op, DAG); 1398 case ISD::FEXP2: 1399 return lowerFEXP2(Op, DAG); 1400 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 1401 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); 1402 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG); 1403 case ISD::FP_TO_SINT: 1404 case ISD::FP_TO_UINT: 1405 return LowerFP_TO_INT(Op, DAG); 1406 case ISD::CTTZ: 1407 case ISD::CTTZ_ZERO_UNDEF: 1408 case ISD::CTLZ: 1409 case ISD::CTLZ_ZERO_UNDEF: 1410 return LowerCTLZ_CTTZ(Op, DAG); 1411 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 1412 } 1413 return Op; 1414 } 1415 1416 void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N, 1417 SmallVectorImpl<SDValue> &Results, 1418 SelectionDAG &DAG) const { 1419 switch (N->getOpcode()) { 1420 case ISD::SIGN_EXTEND_INREG: 1421 // Different parts of legalization seem to interpret which type of 1422 // sign_extend_inreg is the one to check for custom lowering. The extended 1423 // from type is what really matters, but some places check for custom 1424 // lowering of the result type. This results in trying to use 1425 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do 1426 // nothing here and let the illegal result integer be handled normally. 1427 return; 1428 case ISD::FLOG2: 1429 if (SDValue Lowered = LowerFLOG2(SDValue(N, 0), DAG)) 1430 Results.push_back(Lowered); 1431 return; 1432 case ISD::FLOG: 1433 case ISD::FLOG10: 1434 if (SDValue Lowered = LowerFLOGCommon(SDValue(N, 0), DAG)) 1435 Results.push_back(Lowered); 1436 return; 1437 case ISD::FEXP2: 1438 if (SDValue Lowered = lowerFEXP2(SDValue(N, 0), DAG)) 1439 Results.push_back(Lowered); 1440 return; 1441 case ISD::FEXP: 1442 case ISD::FEXP10: 1443 if (SDValue Lowered = lowerFEXP(SDValue(N, 0), DAG)) 1444 Results.push_back(Lowered); 1445 return; 1446 case ISD::CTLZ: 1447 case ISD::CTLZ_ZERO_UNDEF: 1448 if (auto Lowered = lowerCTLZResults(SDValue(N, 0u), DAG)) 1449 Results.push_back(Lowered); 1450 return; 1451 default: 1452 return; 1453 } 1454 } 1455 1456 SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI, 1457 SDValue Op, 1458 SelectionDAG &DAG) const { 1459 1460 const DataLayout &DL = DAG.getDataLayout(); 1461 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op); 1462 const GlobalValue *GV = G->getGlobal(); 1463 1464 if (!MFI->isModuleEntryFunction()) { 1465 if (std::optional<uint32_t> Address = 1466 AMDGPUMachineFunction::getLDSAbsoluteAddress(*GV)) { 1467 return DAG.getConstant(*Address, SDLoc(Op), Op.getValueType()); 1468 } 1469 } 1470 1471 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || 1472 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) { 1473 if (!MFI->isModuleEntryFunction() && 1474 GV->getName() != "llvm.amdgcn.module.lds") { 1475 SDLoc DL(Op); 1476 const Function &Fn = DAG.getMachineFunction().getFunction(); 1477 DiagnosticInfoUnsupported BadLDSDecl( 1478 Fn, "local memory global used by non-kernel function", 1479 DL.getDebugLoc(), DS_Warning); 1480 DAG.getContext()->diagnose(BadLDSDecl); 1481 1482 // We currently don't have a way to correctly allocate LDS objects that 1483 // aren't directly associated with a kernel. We do force inlining of 1484 // functions that use local objects. However, if these dead functions are 1485 // not eliminated, we don't want a compile time error. Just emit a warning 1486 // and a trap, since there should be no callable path here. 1487 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode()); 1488 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 1489 Trap, DAG.getRoot()); 1490 DAG.setRoot(OutputChain); 1491 return DAG.getUNDEF(Op.getValueType()); 1492 } 1493 1494 // XXX: What does the value of G->getOffset() mean? 1495 assert(G->getOffset() == 0 && 1496 "Do not know what to do with an non-zero offset"); 1497 1498 // TODO: We could emit code to handle the initialization somewhere. 1499 // We ignore the initializer for now and legalize it to allow selection. 1500 // The initializer will anyway get errored out during assembly emission. 1501 unsigned Offset = MFI->allocateLDSGlobal(DL, *cast<GlobalVariable>(GV)); 1502 return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType()); 1503 } 1504 return SDValue(); 1505 } 1506 1507 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op, 1508 SelectionDAG &DAG) const { 1509 SmallVector<SDValue, 8> Args; 1510 SDLoc SL(Op); 1511 1512 EVT VT = Op.getValueType(); 1513 if (VT.getVectorElementType().getSizeInBits() < 32) { 1514 unsigned OpBitSize = Op.getOperand(0).getValueType().getSizeInBits(); 1515 if (OpBitSize >= 32 && OpBitSize % 32 == 0) { 1516 unsigned NewNumElt = OpBitSize / 32; 1517 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32 1518 : EVT::getVectorVT(*DAG.getContext(), 1519 MVT::i32, NewNumElt); 1520 for (const SDUse &U : Op->ops()) { 1521 SDValue In = U.get(); 1522 SDValue NewIn = DAG.getNode(ISD::BITCAST, SL, NewEltVT, In); 1523 if (NewNumElt > 1) 1524 DAG.ExtractVectorElements(NewIn, Args); 1525 else 1526 Args.push_back(NewIn); 1527 } 1528 1529 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, 1530 NewNumElt * Op.getNumOperands()); 1531 SDValue BV = DAG.getBuildVector(NewVT, SL, Args); 1532 return DAG.getNode(ISD::BITCAST, SL, VT, BV); 1533 } 1534 } 1535 1536 for (const SDUse &U : Op->ops()) 1537 DAG.ExtractVectorElements(U.get(), Args); 1538 1539 return DAG.getBuildVector(Op.getValueType(), SL, Args); 1540 } 1541 1542 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, 1543 SelectionDAG &DAG) const { 1544 SDLoc SL(Op); 1545 SmallVector<SDValue, 8> Args; 1546 unsigned Start = Op.getConstantOperandVal(1); 1547 EVT VT = Op.getValueType(); 1548 EVT SrcVT = Op.getOperand(0).getValueType(); 1549 1550 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) { 1551 unsigned NumElt = VT.getVectorNumElements(); 1552 unsigned NumSrcElt = SrcVT.getVectorNumElements(); 1553 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types"); 1554 1555 // Extract 32-bit registers at a time. 1556 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumSrcElt / 2); 1557 EVT NewVT = NumElt == 2 1558 ? MVT::i32 1559 : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElt / 2); 1560 SDValue Tmp = DAG.getNode(ISD::BITCAST, SL, NewSrcVT, Op.getOperand(0)); 1561 1562 DAG.ExtractVectorElements(Tmp, Args, Start / 2, NumElt / 2); 1563 if (NumElt == 2) 1564 Tmp = Args[0]; 1565 else 1566 Tmp = DAG.getBuildVector(NewVT, SL, Args); 1567 1568 return DAG.getNode(ISD::BITCAST, SL, VT, Tmp); 1569 } 1570 1571 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start, 1572 VT.getVectorNumElements()); 1573 1574 return DAG.getBuildVector(Op.getValueType(), SL, Args); 1575 } 1576 1577 // TODO: Handle fabs too 1578 static SDValue peekFNeg(SDValue Val) { 1579 if (Val.getOpcode() == ISD::FNEG) 1580 return Val.getOperand(0); 1581 1582 return Val; 1583 } 1584 1585 static SDValue peekFPSignOps(SDValue Val) { 1586 if (Val.getOpcode() == ISD::FNEG) 1587 Val = Val.getOperand(0); 1588 if (Val.getOpcode() == ISD::FABS) 1589 Val = Val.getOperand(0); 1590 if (Val.getOpcode() == ISD::FCOPYSIGN) 1591 Val = Val.getOperand(0); 1592 return Val; 1593 } 1594 1595 SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl( 1596 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, 1597 SDValue False, SDValue CC, DAGCombinerInfo &DCI) const { 1598 SelectionDAG &DAG = DCI.DAG; 1599 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1600 switch (CCOpcode) { 1601 case ISD::SETOEQ: 1602 case ISD::SETONE: 1603 case ISD::SETUNE: 1604 case ISD::SETNE: 1605 case ISD::SETUEQ: 1606 case ISD::SETEQ: 1607 case ISD::SETFALSE: 1608 case ISD::SETFALSE2: 1609 case ISD::SETTRUE: 1610 case ISD::SETTRUE2: 1611 case ISD::SETUO: 1612 case ISD::SETO: 1613 break; 1614 case ISD::SETULE: 1615 case ISD::SETULT: { 1616 if (LHS == True) 1617 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); 1618 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); 1619 } 1620 case ISD::SETOLE: 1621 case ISD::SETOLT: 1622 case ISD::SETLE: 1623 case ISD::SETLT: { 1624 // Ordered. Assume ordered for undefined. 1625 1626 // Only do this after legalization to avoid interfering with other combines 1627 // which might occur. 1628 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG && 1629 !DCI.isCalledByLegalizer()) 1630 return SDValue(); 1631 1632 // We need to permute the operands to get the correct NaN behavior. The 1633 // selected operand is the second one based on the failing compare with NaN, 1634 // so permute it based on the compare type the hardware uses. 1635 if (LHS == True) 1636 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); 1637 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); 1638 } 1639 case ISD::SETUGE: 1640 case ISD::SETUGT: { 1641 if (LHS == True) 1642 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); 1643 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); 1644 } 1645 case ISD::SETGT: 1646 case ISD::SETGE: 1647 case ISD::SETOGE: 1648 case ISD::SETOGT: { 1649 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG && 1650 !DCI.isCalledByLegalizer()) 1651 return SDValue(); 1652 1653 if (LHS == True) 1654 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); 1655 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); 1656 } 1657 case ISD::SETCC_INVALID: 1658 llvm_unreachable("Invalid setcc condcode!"); 1659 } 1660 return SDValue(); 1661 } 1662 1663 /// Generate Min/Max node 1664 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT, 1665 SDValue LHS, SDValue RHS, 1666 SDValue True, SDValue False, 1667 SDValue CC, 1668 DAGCombinerInfo &DCI) const { 1669 if ((LHS == True && RHS == False) || (LHS == False && RHS == True)) 1670 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, DCI); 1671 1672 SelectionDAG &DAG = DCI.DAG; 1673 1674 // If we can't directly match this, try to see if we can fold an fneg to 1675 // match. 1676 1677 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS); 1678 ConstantFPSDNode *CFalse = dyn_cast<ConstantFPSDNode>(False); 1679 SDValue NegTrue = peekFNeg(True); 1680 1681 // Undo the combine foldFreeOpFromSelect does if it helps us match the 1682 // fmin/fmax. 1683 // 1684 // select (fcmp olt (lhs, K)), (fneg lhs), -K 1685 // -> fneg (fmin_legacy lhs, K) 1686 // 1687 // TODO: Use getNegatedExpression 1688 if (LHS == NegTrue && CFalse && CRHS) { 1689 APFloat NegRHS = neg(CRHS->getValueAPF()); 1690 if (NegRHS == CFalse->getValueAPF()) { 1691 SDValue Combined = 1692 combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, NegTrue, False, CC, DCI); 1693 if (Combined) 1694 return DAG.getNode(ISD::FNEG, DL, VT, Combined); 1695 return SDValue(); 1696 } 1697 } 1698 1699 return SDValue(); 1700 } 1701 1702 std::pair<SDValue, SDValue> 1703 AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const { 1704 SDLoc SL(Op); 1705 1706 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); 1707 1708 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); 1709 const SDValue One = DAG.getConstant(1, SL, MVT::i32); 1710 1711 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); 1712 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); 1713 1714 return std::pair(Lo, Hi); 1715 } 1716 1717 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const { 1718 SDLoc SL(Op); 1719 1720 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); 1721 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); 1722 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); 1723 } 1724 1725 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const { 1726 SDLoc SL(Op); 1727 1728 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); 1729 const SDValue One = DAG.getConstant(1, SL, MVT::i32); 1730 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); 1731 } 1732 1733 // Split a vector type into two parts. The first part is a power of two vector. 1734 // The second part is whatever is left over, and is a scalar if it would 1735 // otherwise be a 1-vector. 1736 std::pair<EVT, EVT> 1737 AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const { 1738 EVT LoVT, HiVT; 1739 EVT EltVT = VT.getVectorElementType(); 1740 unsigned NumElts = VT.getVectorNumElements(); 1741 unsigned LoNumElts = PowerOf2Ceil((NumElts + 1) / 2); 1742 LoVT = EVT::getVectorVT(*DAG.getContext(), EltVT, LoNumElts); 1743 HiVT = NumElts - LoNumElts == 1 1744 ? EltVT 1745 : EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts - LoNumElts); 1746 return std::pair(LoVT, HiVT); 1747 } 1748 1749 // Split a vector value into two parts of types LoVT and HiVT. HiVT could be 1750 // scalar. 1751 std::pair<SDValue, SDValue> 1752 AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL, 1753 const EVT &LoVT, const EVT &HiVT, 1754 SelectionDAG &DAG) const { 1755 assert(LoVT.getVectorNumElements() + 1756 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <= 1757 N.getValueType().getVectorNumElements() && 1758 "More vector elements requested than available!"); 1759 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, 1760 DAG.getVectorIdxConstant(0, DL)); 1761 SDValue Hi = DAG.getNode( 1762 HiVT.isVector() ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL, 1763 HiVT, N, DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), DL)); 1764 return std::pair(Lo, Hi); 1765 } 1766 1767 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op, 1768 SelectionDAG &DAG) const { 1769 LoadSDNode *Load = cast<LoadSDNode>(Op); 1770 EVT VT = Op.getValueType(); 1771 SDLoc SL(Op); 1772 1773 1774 // If this is a 2 element vector, we really want to scalarize and not create 1775 // weird 1 element vectors. 1776 if (VT.getVectorNumElements() == 2) { 1777 SDValue Ops[2]; 1778 std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG); 1779 return DAG.getMergeValues(Ops, SL); 1780 } 1781 1782 SDValue BasePtr = Load->getBasePtr(); 1783 EVT MemVT = Load->getMemoryVT(); 1784 1785 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo(); 1786 1787 EVT LoVT, HiVT; 1788 EVT LoMemVT, HiMemVT; 1789 SDValue Lo, Hi; 1790 1791 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG); 1792 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG); 1793 std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG); 1794 1795 unsigned Size = LoMemVT.getStoreSize(); 1796 Align BaseAlign = Load->getAlign(); 1797 Align HiAlign = commonAlignment(BaseAlign, Size); 1798 1799 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT, 1800 Load->getChain(), BasePtr, SrcValue, LoMemVT, 1801 BaseAlign, Load->getMemOperand()->getFlags()); 1802 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Size)); 1803 SDValue HiLoad = 1804 DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(), 1805 HiPtr, SrcValue.getWithOffset(LoMemVT.getStoreSize()), 1806 HiMemVT, HiAlign, Load->getMemOperand()->getFlags()); 1807 1808 SDValue Join; 1809 if (LoVT == HiVT) { 1810 // This is the case that the vector is power of two so was evenly split. 1811 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad); 1812 } else { 1813 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad, 1814 DAG.getVectorIdxConstant(0, SL)); 1815 Join = DAG.getNode( 1816 HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, SL, 1817 VT, Join, HiLoad, 1818 DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), SL)); 1819 } 1820 1821 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other, 1822 LoLoad.getValue(1), HiLoad.getValue(1))}; 1823 1824 return DAG.getMergeValues(Ops, SL); 1825 } 1826 1827 SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op, 1828 SelectionDAG &DAG) const { 1829 LoadSDNode *Load = cast<LoadSDNode>(Op); 1830 EVT VT = Op.getValueType(); 1831 SDValue BasePtr = Load->getBasePtr(); 1832 EVT MemVT = Load->getMemoryVT(); 1833 SDLoc SL(Op); 1834 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo(); 1835 Align BaseAlign = Load->getAlign(); 1836 unsigned NumElements = MemVT.getVectorNumElements(); 1837 1838 // Widen from vec3 to vec4 when the load is at least 8-byte aligned 1839 // or 16-byte fully dereferenceable. Otherwise, split the vector load. 1840 if (NumElements != 3 || 1841 (BaseAlign < Align(8) && 1842 !SrcValue.isDereferenceable(16, *DAG.getContext(), DAG.getDataLayout()))) 1843 return SplitVectorLoad(Op, DAG); 1844 1845 assert(NumElements == 3); 1846 1847 EVT WideVT = 1848 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4); 1849 EVT WideMemVT = 1850 EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), 4); 1851 SDValue WideLoad = DAG.getExtLoad( 1852 Load->getExtensionType(), SL, WideVT, Load->getChain(), BasePtr, SrcValue, 1853 WideMemVT, BaseAlign, Load->getMemOperand()->getFlags()); 1854 return DAG.getMergeValues( 1855 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad, 1856 DAG.getVectorIdxConstant(0, SL)), 1857 WideLoad.getValue(1)}, 1858 SL); 1859 } 1860 1861 SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op, 1862 SelectionDAG &DAG) const { 1863 StoreSDNode *Store = cast<StoreSDNode>(Op); 1864 SDValue Val = Store->getValue(); 1865 EVT VT = Val.getValueType(); 1866 1867 // If this is a 2 element vector, we really want to scalarize and not create 1868 // weird 1 element vectors. 1869 if (VT.getVectorNumElements() == 2) 1870 return scalarizeVectorStore(Store, DAG); 1871 1872 EVT MemVT = Store->getMemoryVT(); 1873 SDValue Chain = Store->getChain(); 1874 SDValue BasePtr = Store->getBasePtr(); 1875 SDLoc SL(Op); 1876 1877 EVT LoVT, HiVT; 1878 EVT LoMemVT, HiMemVT; 1879 SDValue Lo, Hi; 1880 1881 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG); 1882 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG); 1883 std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG); 1884 1885 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize()); 1886 1887 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo(); 1888 Align BaseAlign = Store->getAlign(); 1889 unsigned Size = LoMemVT.getStoreSize(); 1890 Align HiAlign = commonAlignment(BaseAlign, Size); 1891 1892 SDValue LoStore = 1893 DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign, 1894 Store->getMemOperand()->getFlags()); 1895 SDValue HiStore = 1896 DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size), 1897 HiMemVT, HiAlign, Store->getMemOperand()->getFlags()); 1898 1899 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); 1900 } 1901 1902 // This is a shortcut for integer division because we have fast i32<->f32 1903 // conversions, and fast f32 reciprocal instructions. The fractional part of a 1904 // float is enough to accurately represent up to a 24-bit signed integer. 1905 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, 1906 bool Sign) const { 1907 SDLoc DL(Op); 1908 EVT VT = Op.getValueType(); 1909 SDValue LHS = Op.getOperand(0); 1910 SDValue RHS = Op.getOperand(1); 1911 MVT IntVT = MVT::i32; 1912 MVT FltVT = MVT::f32; 1913 1914 unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS); 1915 if (LHSSignBits < 9) 1916 return SDValue(); 1917 1918 unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS); 1919 if (RHSSignBits < 9) 1920 return SDValue(); 1921 1922 unsigned BitSize = VT.getSizeInBits(); 1923 unsigned SignBits = std::min(LHSSignBits, RHSSignBits); 1924 unsigned DivBits = BitSize - SignBits; 1925 if (Sign) 1926 ++DivBits; 1927 1928 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; 1929 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT; 1930 1931 SDValue jq = DAG.getConstant(1, DL, IntVT); 1932 1933 if (Sign) { 1934 // char|short jq = ia ^ ib; 1935 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); 1936 1937 // jq = jq >> (bitsize - 2) 1938 jq = DAG.getNode(ISD::SRA, DL, VT, jq, 1939 DAG.getConstant(BitSize - 2, DL, VT)); 1940 1941 // jq = jq | 0x1 1942 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT)); 1943 } 1944 1945 // int ia = (int)LHS; 1946 SDValue ia = LHS; 1947 1948 // int ib, (int)RHS; 1949 SDValue ib = RHS; 1950 1951 // float fa = (float)ia; 1952 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia); 1953 1954 // float fb = (float)ib; 1955 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib); 1956 1957 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT, 1958 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb)); 1959 1960 // fq = trunc(fq); 1961 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq); 1962 1963 // float fqneg = -fq; 1964 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq); 1965 1966 MachineFunction &MF = DAG.getMachineFunction(); 1967 1968 bool UseFmadFtz = false; 1969 if (Subtarget->isGCN()) { 1970 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1971 UseFmadFtz = 1972 MFI->getMode().FP32Denormals != DenormalMode::getPreserveSign(); 1973 } 1974 1975 // float fr = mad(fqneg, fb, fa); 1976 unsigned OpCode = !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA 1977 : UseFmadFtz ? (unsigned)AMDGPUISD::FMAD_FTZ 1978 : (unsigned)ISD::FMAD; 1979 SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa); 1980 1981 // int iq = (int)fq; 1982 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq); 1983 1984 // fr = fabs(fr); 1985 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr); 1986 1987 // fb = fabs(fb); 1988 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb); 1989 1990 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 1991 1992 // int cv = fr >= fb; 1993 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE); 1994 1995 // jq = (cv ? jq : 0); 1996 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT)); 1997 1998 // dst = iq + jq; 1999 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq); 2000 2001 // Rem needs compensation, it's easier to recompute it 2002 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS); 2003 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem); 2004 2005 // Truncate to number of bits this divide really is. 2006 if (Sign) { 2007 SDValue InRegSize 2008 = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits)); 2009 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize); 2010 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize); 2011 } else { 2012 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT); 2013 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask); 2014 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask); 2015 } 2016 2017 return DAG.getMergeValues({ Div, Rem }, DL); 2018 } 2019 2020 void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op, 2021 SelectionDAG &DAG, 2022 SmallVectorImpl<SDValue> &Results) const { 2023 SDLoc DL(Op); 2024 EVT VT = Op.getValueType(); 2025 2026 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64"); 2027 2028 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); 2029 2030 SDValue One = DAG.getConstant(1, DL, HalfVT); 2031 SDValue Zero = DAG.getConstant(0, DL, HalfVT); 2032 2033 //HiLo split 2034 SDValue LHS_Lo, LHS_Hi; 2035 SDValue LHS = Op.getOperand(0); 2036 std::tie(LHS_Lo, LHS_Hi) = DAG.SplitScalar(LHS, DL, HalfVT, HalfVT); 2037 2038 SDValue RHS_Lo, RHS_Hi; 2039 SDValue RHS = Op.getOperand(1); 2040 std::tie(RHS_Lo, RHS_Hi) = DAG.SplitScalar(RHS, DL, HalfVT, HalfVT); 2041 2042 if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) && 2043 DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) { 2044 2045 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), 2046 LHS_Lo, RHS_Lo); 2047 2048 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero}); 2049 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero}); 2050 2051 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV)); 2052 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM)); 2053 return; 2054 } 2055 2056 if (isTypeLegal(MVT::i64)) { 2057 // The algorithm here is based on ideas from "Software Integer Division", 2058 // Tom Rodeheffer, August 2008. 2059 2060 MachineFunction &MF = DAG.getMachineFunction(); 2061 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 2062 2063 // Compute denominator reciprocal. 2064 unsigned FMAD = 2065 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA 2066 : MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign() 2067 ? (unsigned)ISD::FMAD 2068 : (unsigned)AMDGPUISD::FMAD_FTZ; 2069 2070 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo); 2071 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi); 2072 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi, 2073 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32), 2074 Cvt_Lo); 2075 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1); 2076 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp, 2077 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32)); 2078 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1, 2079 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32)); 2080 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2); 2081 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc, 2082 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32), 2083 Mul1); 2084 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2); 2085 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc); 2086 SDValue Rcp64 = DAG.getBitcast(VT, 2087 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi})); 2088 2089 SDValue Zero64 = DAG.getConstant(0, DL, VT); 2090 SDValue One64 = DAG.getConstant(1, DL, VT); 2091 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1); 2092 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1); 2093 2094 // First round of UNR (Unsigned integer Newton-Raphson). 2095 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS); 2096 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64); 2097 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1); 2098 SDValue Mulhi1_Lo, Mulhi1_Hi; 2099 std::tie(Mulhi1_Lo, Mulhi1_Hi) = 2100 DAG.SplitScalar(Mulhi1, DL, HalfVT, HalfVT); 2101 SDValue Add1_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Lo, 2102 Mulhi1_Lo, Zero1); 2103 SDValue Add1_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Hi, 2104 Mulhi1_Hi, Add1_Lo.getValue(1)); 2105 SDValue Add1 = DAG.getBitcast(VT, 2106 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi})); 2107 2108 // Second round of UNR. 2109 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1); 2110 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2); 2111 SDValue Mulhi2_Lo, Mulhi2_Hi; 2112 std::tie(Mulhi2_Lo, Mulhi2_Hi) = 2113 DAG.SplitScalar(Mulhi2, DL, HalfVT, HalfVT); 2114 SDValue Add2_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Lo, 2115 Mulhi2_Lo, Zero1); 2116 SDValue Add2_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Hi, 2117 Mulhi2_Hi, Add2_Lo.getValue(1)); 2118 SDValue Add2 = DAG.getBitcast(VT, 2119 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi})); 2120 2121 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2); 2122 2123 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3); 2124 2125 SDValue Mul3_Lo, Mul3_Hi; 2126 std::tie(Mul3_Lo, Mul3_Hi) = DAG.SplitScalar(Mul3, DL, HalfVT, HalfVT); 2127 SDValue Sub1_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo, 2128 Mul3_Lo, Zero1); 2129 SDValue Sub1_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Hi, 2130 Mul3_Hi, Sub1_Lo.getValue(1)); 2131 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi); 2132 SDValue Sub1 = DAG.getBitcast(VT, 2133 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi})); 2134 2135 SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT); 2136 SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero, 2137 ISD::SETUGE); 2138 SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero, 2139 ISD::SETUGE); 2140 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ); 2141 2142 // TODO: Here and below portions of the code can be enclosed into if/endif. 2143 // Currently control flow is unconditional and we have 4 selects after 2144 // potential endif to substitute PHIs. 2145 2146 // if C3 != 0 ... 2147 SDValue Sub2_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo, 2148 RHS_Lo, Zero1); 2149 SDValue Sub2_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Mi, 2150 RHS_Hi, Sub1_Lo.getValue(1)); 2151 SDValue Sub2_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi, 2152 Zero, Sub2_Lo.getValue(1)); 2153 SDValue Sub2 = DAG.getBitcast(VT, 2154 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi})); 2155 2156 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64); 2157 2158 SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero, 2159 ISD::SETUGE); 2160 SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero, 2161 ISD::SETUGE); 2162 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ); 2163 2164 // if (C6 != 0) 2165 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64); 2166 2167 SDValue Sub3_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo, 2168 RHS_Lo, Zero1); 2169 SDValue Sub3_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi, 2170 RHS_Hi, Sub2_Lo.getValue(1)); 2171 SDValue Sub3_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub3_Mi, 2172 Zero, Sub3_Lo.getValue(1)); 2173 SDValue Sub3 = DAG.getBitcast(VT, 2174 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi})); 2175 2176 // endif C6 2177 // endif C3 2178 2179 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE); 2180 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE); 2181 2182 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE); 2183 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE); 2184 2185 Results.push_back(Div); 2186 Results.push_back(Rem); 2187 2188 return; 2189 } 2190 2191 // r600 expandion. 2192 // Get Speculative values 2193 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo); 2194 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo); 2195 2196 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ); 2197 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero}); 2198 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM); 2199 2200 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ); 2201 SDValue DIV_Lo = Zero; 2202 2203 const unsigned halfBitWidth = HalfVT.getSizeInBits(); 2204 2205 for (unsigned i = 0; i < halfBitWidth; ++i) { 2206 const unsigned bitPos = halfBitWidth - i - 1; 2207 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT); 2208 // Get value of high bit 2209 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS); 2210 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One); 2211 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit); 2212 2213 // Shift 2214 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT)); 2215 // Add LHS high bit 2216 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit); 2217 2218 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT); 2219 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE); 2220 2221 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT); 2222 2223 // Update REM 2224 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); 2225 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE); 2226 } 2227 2228 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi}); 2229 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV); 2230 Results.push_back(DIV); 2231 Results.push_back(REM); 2232 } 2233 2234 SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op, 2235 SelectionDAG &DAG) const { 2236 SDLoc DL(Op); 2237 EVT VT = Op.getValueType(); 2238 2239 if (VT == MVT::i64) { 2240 SmallVector<SDValue, 2> Results; 2241 LowerUDIVREM64(Op, DAG, Results); 2242 return DAG.getMergeValues(Results, DL); 2243 } 2244 2245 if (VT == MVT::i32) { 2246 if (SDValue Res = LowerDIVREM24(Op, DAG, false)) 2247 return Res; 2248 } 2249 2250 SDValue X = Op.getOperand(0); 2251 SDValue Y = Op.getOperand(1); 2252 2253 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the 2254 // algorithm used here. 2255 2256 // Initial estimate of inv(y). 2257 SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y); 2258 2259 // One round of UNR. 2260 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y); 2261 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z); 2262 Z = DAG.getNode(ISD::ADD, DL, VT, Z, 2263 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ)); 2264 2265 // Quotient/remainder estimate. 2266 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z); 2267 SDValue R = 2268 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y)); 2269 2270 // First quotient/remainder refinement. 2271 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 2272 SDValue One = DAG.getConstant(1, DL, VT); 2273 SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); 2274 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, 2275 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); 2276 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, 2277 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); 2278 2279 // Second quotient/remainder refinement. 2280 Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); 2281 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, 2282 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); 2283 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, 2284 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); 2285 2286 return DAG.getMergeValues({Q, R}, DL); 2287 } 2288 2289 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op, 2290 SelectionDAG &DAG) const { 2291 SDLoc DL(Op); 2292 EVT VT = Op.getValueType(); 2293 2294 SDValue LHS = Op.getOperand(0); 2295 SDValue RHS = Op.getOperand(1); 2296 2297 SDValue Zero = DAG.getConstant(0, DL, VT); 2298 SDValue NegOne = DAG.getConstant(-1, DL, VT); 2299 2300 if (VT == MVT::i32) { 2301 if (SDValue Res = LowerDIVREM24(Op, DAG, true)) 2302 return Res; 2303 } 2304 2305 if (VT == MVT::i64 && 2306 DAG.ComputeNumSignBits(LHS) > 32 && 2307 DAG.ComputeNumSignBits(RHS) > 32) { 2308 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); 2309 2310 //HiLo split 2311 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero); 2312 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero); 2313 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), 2314 LHS_Lo, RHS_Lo); 2315 SDValue Res[2] = { 2316 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)), 2317 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1)) 2318 }; 2319 return DAG.getMergeValues(Res, DL); 2320 } 2321 2322 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT); 2323 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT); 2324 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); 2325 SDValue RSign = LHSign; // Remainder sign is the same as LHS 2326 2327 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); 2328 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); 2329 2330 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); 2331 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); 2332 2333 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); 2334 SDValue Rem = Div.getValue(1); 2335 2336 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); 2337 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); 2338 2339 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); 2340 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); 2341 2342 SDValue Res[2] = { 2343 Div, 2344 Rem 2345 }; 2346 return DAG.getMergeValues(Res, DL); 2347 } 2348 2349 // (frem x, y) -> (fma (fneg (ftrunc (fdiv x, y))), y, x) 2350 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const { 2351 SDLoc SL(Op); 2352 EVT VT = Op.getValueType(); 2353 auto Flags = Op->getFlags(); 2354 SDValue X = Op.getOperand(0); 2355 SDValue Y = Op.getOperand(1); 2356 2357 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags); 2358 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags); 2359 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags); 2360 // TODO: For f32 use FMAD instead if !hasFastFMA32? 2361 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); 2362 } 2363 2364 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const { 2365 SDLoc SL(Op); 2366 SDValue Src = Op.getOperand(0); 2367 2368 // result = trunc(src) 2369 // if (src > 0.0 && src != result) 2370 // result += 1.0 2371 2372 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); 2373 2374 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); 2375 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); 2376 2377 EVT SetCCVT = 2378 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); 2379 2380 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT); 2381 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); 2382 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); 2383 2384 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); 2385 // TODO: Should this propagate fast-math-flags? 2386 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); 2387 } 2388 2389 static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, 2390 SelectionDAG &DAG) { 2391 const unsigned FractBits = 52; 2392 const unsigned ExpBits = 11; 2393 2394 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, 2395 Hi, 2396 DAG.getConstant(FractBits - 32, SL, MVT::i32), 2397 DAG.getConstant(ExpBits, SL, MVT::i32)); 2398 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, 2399 DAG.getConstant(1023, SL, MVT::i32)); 2400 2401 return Exp; 2402 } 2403 2404 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const { 2405 SDLoc SL(Op); 2406 SDValue Src = Op.getOperand(0); 2407 2408 assert(Op.getValueType() == MVT::f64); 2409 2410 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); 2411 2412 // Extract the upper half, since this is where we will find the sign and 2413 // exponent. 2414 SDValue Hi = getHiHalf64(Src, DAG); 2415 2416 SDValue Exp = extractF64Exponent(Hi, SL, DAG); 2417 2418 const unsigned FractBits = 52; 2419 2420 // Extract the sign bit. 2421 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); 2422 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); 2423 2424 // Extend back to 64-bits. 2425 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit}); 2426 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); 2427 2428 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); 2429 const SDValue FractMask 2430 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); 2431 2432 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); 2433 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); 2434 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); 2435 2436 EVT SetCCVT = 2437 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); 2438 2439 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); 2440 2441 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); 2442 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); 2443 2444 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); 2445 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); 2446 2447 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); 2448 } 2449 2450 SDValue AMDGPUTargetLowering::LowerFROUNDEVEN(SDValue Op, 2451 SelectionDAG &DAG) const { 2452 SDLoc SL(Op); 2453 SDValue Src = Op.getOperand(0); 2454 2455 assert(Op.getValueType() == MVT::f64); 2456 2457 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52"); 2458 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); 2459 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); 2460 2461 // TODO: Should this propagate fast-math-flags? 2462 2463 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); 2464 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); 2465 2466 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); 2467 2468 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51"); 2469 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); 2470 2471 EVT SetCCVT = 2472 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); 2473 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT); 2474 2475 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); 2476 } 2477 2478 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, 2479 SelectionDAG &DAG) const { 2480 // FNEARBYINT and FRINT are the same, except in their handling of FP 2481 // exceptions. Those aren't really meaningful for us, and OpenCL only has 2482 // rint, so just treat them as equivalent. 2483 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), Op.getValueType(), 2484 Op.getOperand(0)); 2485 } 2486 2487 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const { 2488 auto VT = Op.getValueType(); 2489 auto Arg = Op.getOperand(0u); 2490 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), VT, Arg); 2491 } 2492 2493 // XXX - May require not supporting f32 denormals? 2494 2495 // Don't handle v2f16. The extra instructions to scalarize and repack around the 2496 // compare and vselect end up producing worse code than scalarizing the whole 2497 // operation. 2498 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const { 2499 SDLoc SL(Op); 2500 SDValue X = Op.getOperand(0); 2501 EVT VT = Op.getValueType(); 2502 2503 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); 2504 2505 // TODO: Should this propagate fast-math-flags? 2506 2507 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); 2508 2509 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); 2510 2511 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT); 2512 const SDValue One = DAG.getConstantFP(1.0, SL, VT); 2513 2514 EVT SetCCVT = 2515 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 2516 2517 const SDValue Half = DAG.getConstantFP(0.5, SL, VT); 2518 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE); 2519 SDValue OneOrZeroFP = DAG.getNode(ISD::SELECT, SL, VT, Cmp, One, Zero); 2520 2521 SDValue SignedOffset = DAG.getNode(ISD::FCOPYSIGN, SL, VT, OneOrZeroFP, X); 2522 return DAG.getNode(ISD::FADD, SL, VT, T, SignedOffset); 2523 } 2524 2525 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const { 2526 SDLoc SL(Op); 2527 SDValue Src = Op.getOperand(0); 2528 2529 // result = trunc(src); 2530 // if (src < 0.0 && src != result) 2531 // result += -1.0. 2532 2533 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); 2534 2535 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); 2536 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); 2537 2538 EVT SetCCVT = 2539 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); 2540 2541 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT); 2542 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); 2543 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); 2544 2545 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); 2546 // TODO: Should this propagate fast-math-flags? 2547 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); 2548 } 2549 2550 /// Return true if it's known that \p Src can never be an f32 denormal value. 2551 static bool valueIsKnownNeverF32Denorm(SDValue Src) { 2552 switch (Src.getOpcode()) { 2553 case ISD::FP_EXTEND: 2554 return Src.getOperand(0).getValueType() == MVT::f16; 2555 case ISD::FP16_TO_FP: 2556 case ISD::FFREXP: 2557 return true; 2558 case ISD::INTRINSIC_WO_CHAIN: { 2559 unsigned IntrinsicID = Src.getConstantOperandVal(0); 2560 switch (IntrinsicID) { 2561 case Intrinsic::amdgcn_frexp_mant: 2562 return true; 2563 default: 2564 return false; 2565 } 2566 } 2567 default: 2568 return false; 2569 } 2570 2571 llvm_unreachable("covered opcode switch"); 2572 } 2573 2574 bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG, 2575 SDNodeFlags Flags) { 2576 if (Flags.hasApproximateFuncs()) 2577 return true; 2578 auto &Options = DAG.getTarget().Options; 2579 return Options.UnsafeFPMath || Options.ApproxFuncFPMath; 2580 } 2581 2582 bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG, 2583 SDValue Src, 2584 SDNodeFlags Flags) { 2585 return !valueIsKnownNeverF32Denorm(Src) && 2586 DAG.getMachineFunction() 2587 .getDenormalMode(APFloat::IEEEsingle()) 2588 .Input != DenormalMode::PreserveSign; 2589 } 2590 2591 SDValue AMDGPUTargetLowering::getIsLtSmallestNormal(SelectionDAG &DAG, 2592 SDValue Src, 2593 SDNodeFlags Flags) const { 2594 SDLoc SL(Src); 2595 EVT VT = Src.getValueType(); 2596 const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT); 2597 SDValue SmallestNormal = 2598 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT); 2599 2600 // Want to scale denormals up, but negatives and 0 work just as well on the 2601 // scaled path. 2602 SDValue IsLtSmallestNormal = DAG.getSetCC( 2603 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src, 2604 SmallestNormal, ISD::SETOLT); 2605 2606 return IsLtSmallestNormal; 2607 } 2608 2609 SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src, 2610 SDNodeFlags Flags) const { 2611 SDLoc SL(Src); 2612 EVT VT = Src.getValueType(); 2613 const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT); 2614 SDValue Inf = DAG.getConstantFP(APFloat::getInf(Semantics), SL, VT); 2615 2616 SDValue Fabs = DAG.getNode(ISD::FABS, SL, VT, Src, Flags); 2617 SDValue IsFinite = DAG.getSetCC( 2618 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Fabs, 2619 Inf, ISD::SETOLT); 2620 return IsFinite; 2621 } 2622 2623 /// If denormal handling is required return the scaled input to FLOG2, and the 2624 /// check for denormal range. Otherwise, return null values. 2625 std::pair<SDValue, SDValue> 2626 AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL, 2627 SDValue Src, SDNodeFlags Flags) const { 2628 if (!needsDenormHandlingF32(DAG, Src, Flags)) 2629 return {}; 2630 2631 MVT VT = MVT::f32; 2632 const fltSemantics &Semantics = APFloat::IEEEsingle(); 2633 SDValue SmallestNormal = 2634 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT); 2635 2636 SDValue IsLtSmallestNormal = DAG.getSetCC( 2637 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src, 2638 SmallestNormal, ISD::SETOLT); 2639 2640 SDValue Scale32 = DAG.getConstantFP(0x1.0p+32, SL, VT); 2641 SDValue One = DAG.getConstantFP(1.0, SL, VT); 2642 SDValue ScaleFactor = 2643 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, Scale32, One, Flags); 2644 2645 SDValue ScaledInput = DAG.getNode(ISD::FMUL, SL, VT, Src, ScaleFactor, Flags); 2646 return {ScaledInput, IsLtSmallestNormal}; 2647 } 2648 2649 SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const { 2650 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals. 2651 // If we have to handle denormals, scale up the input and adjust the result. 2652 2653 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0) 2654 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0) 2655 2656 SDLoc SL(Op); 2657 EVT VT = Op.getValueType(); 2658 SDValue Src = Op.getOperand(0); 2659 SDNodeFlags Flags = Op->getFlags(); 2660 2661 if (VT == MVT::f16) { 2662 // Nothing in half is a denormal when promoted to f32. 2663 assert(!Subtarget->has16BitInsts()); 2664 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); 2665 SDValue Log = DAG.getNode(AMDGPUISD::LOG, SL, MVT::f32, Ext, Flags); 2666 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, 2667 DAG.getTargetConstant(0, SL, MVT::i32), Flags); 2668 } 2669 2670 auto [ScaledInput, IsLtSmallestNormal] = 2671 getScaledLogInput(DAG, SL, Src, Flags); 2672 if (!ScaledInput) 2673 return DAG.getNode(AMDGPUISD::LOG, SL, VT, Src, Flags); 2674 2675 SDValue Log2 = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags); 2676 2677 SDValue ThirtyTwo = DAG.getConstantFP(32.0, SL, VT); 2678 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); 2679 SDValue ResultOffset = 2680 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, ThirtyTwo, Zero); 2681 return DAG.getNode(ISD::FSUB, SL, VT, Log2, ResultOffset, Flags); 2682 } 2683 2684 static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, 2685 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) { 2686 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Y, Flags); 2687 return DAG.getNode(ISD::FADD, SL, VT, Mul, C, Flags); 2688 } 2689 2690 SDValue AMDGPUTargetLowering::LowerFLOGCommon(SDValue Op, 2691 SelectionDAG &DAG) const { 2692 SDValue X = Op.getOperand(0); 2693 EVT VT = Op.getValueType(); 2694 SDNodeFlags Flags = Op->getFlags(); 2695 SDLoc DL(Op); 2696 2697 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10; 2698 assert(IsLog10 || Op.getOpcode() == ISD::FLOG); 2699 2700 const auto &Options = getTargetMachine().Options; 2701 if (VT == MVT::f16 || Flags.hasApproximateFuncs() || 2702 Options.ApproxFuncFPMath || Options.UnsafeFPMath) { 2703 2704 if (VT == MVT::f16 && !Subtarget->has16BitInsts()) { 2705 // Log and multiply in f32 is good enough for f16. 2706 X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X, Flags); 2707 } 2708 2709 SDValue Lowered = LowerFLOGUnsafe(X, DL, DAG, IsLog10, Flags); 2710 if (VT == MVT::f16 && !Subtarget->has16BitInsts()) { 2711 return DAG.getNode(ISD::FP_ROUND, DL, VT, Lowered, 2712 DAG.getTargetConstant(0, DL, MVT::i32), Flags); 2713 } 2714 2715 return Lowered; 2716 } 2717 2718 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, DL, X, Flags); 2719 if (ScaledInput) 2720 X = ScaledInput; 2721 2722 SDValue Y = DAG.getNode(AMDGPUISD::LOG, DL, VT, X, Flags); 2723 2724 SDValue R; 2725 if (Subtarget->hasFastFMAF32()) { 2726 // c+cc are ln(2)/ln(10) to more than 49 bits 2727 const float c_log10 = 0x1.344134p-2f; 2728 const float cc_log10 = 0x1.09f79ep-26f; 2729 2730 // c + cc is ln(2) to more than 49 bits 2731 const float c_log = 0x1.62e42ep-1f; 2732 const float cc_log = 0x1.efa39ep-25f; 2733 2734 SDValue C = DAG.getConstantFP(IsLog10 ? c_log10 : c_log, DL, VT); 2735 SDValue CC = DAG.getConstantFP(IsLog10 ? cc_log10 : cc_log, DL, VT); 2736 2737 R = DAG.getNode(ISD::FMUL, DL, VT, Y, C, Flags); 2738 SDValue NegR = DAG.getNode(ISD::FNEG, DL, VT, R, Flags); 2739 SDValue FMA0 = DAG.getNode(ISD::FMA, DL, VT, Y, C, NegR, Flags); 2740 SDValue FMA1 = DAG.getNode(ISD::FMA, DL, VT, Y, CC, FMA0, Flags); 2741 R = DAG.getNode(ISD::FADD, DL, VT, R, FMA1, Flags); 2742 } else { 2743 // ch+ct is ln(2)/ln(10) to more than 36 bits 2744 const float ch_log10 = 0x1.344000p-2f; 2745 const float ct_log10 = 0x1.3509f6p-18f; 2746 2747 // ch + ct is ln(2) to more than 36 bits 2748 const float ch_log = 0x1.62e000p-1f; 2749 const float ct_log = 0x1.0bfbe8p-15f; 2750 2751 SDValue CH = DAG.getConstantFP(IsLog10 ? ch_log10 : ch_log, DL, VT); 2752 SDValue CT = DAG.getConstantFP(IsLog10 ? ct_log10 : ct_log, DL, VT); 2753 2754 SDValue YAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Y); 2755 SDValue MaskConst = DAG.getConstant(0xfffff000, DL, MVT::i32); 2756 SDValue YHInt = DAG.getNode(ISD::AND, DL, MVT::i32, YAsInt, MaskConst); 2757 SDValue YH = DAG.getNode(ISD::BITCAST, DL, MVT::f32, YHInt); 2758 SDValue YT = DAG.getNode(ISD::FSUB, DL, VT, Y, YH, Flags); 2759 2760 SDValue YTCT = DAG.getNode(ISD::FMUL, DL, VT, YT, CT, Flags); 2761 SDValue Mad0 = getMad(DAG, DL, VT, YH, CT, YTCT, Flags); 2762 SDValue Mad1 = getMad(DAG, DL, VT, YT, CH, Mad0, Flags); 2763 R = getMad(DAG, DL, VT, YH, CH, Mad1); 2764 } 2765 2766 const bool IsFiniteOnly = (Flags.hasNoNaNs() || Options.NoNaNsFPMath) && 2767 (Flags.hasNoInfs() || Options.NoInfsFPMath); 2768 2769 // TODO: Check if known finite from source value. 2770 if (!IsFiniteOnly) { 2771 SDValue IsFinite = getIsFinite(DAG, Y, Flags); 2772 R = DAG.getNode(ISD::SELECT, DL, VT, IsFinite, R, Y, Flags); 2773 } 2774 2775 if (IsScaled) { 2776 SDValue Zero = DAG.getConstantFP(0.0f, DL, VT); 2777 SDValue ShiftK = 2778 DAG.getConstantFP(IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT); 2779 SDValue Shift = 2780 DAG.getNode(ISD::SELECT, DL, VT, IsScaled, ShiftK, Zero, Flags); 2781 R = DAG.getNode(ISD::FSUB, DL, VT, R, Shift, Flags); 2782 } 2783 2784 return R; 2785 } 2786 2787 SDValue AMDGPUTargetLowering::LowerFLOG10(SDValue Op, SelectionDAG &DAG) const { 2788 return LowerFLOGCommon(Op, DAG); 2789 } 2790 2791 // Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a 2792 // promote f16 operation. 2793 SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL, 2794 SelectionDAG &DAG, bool IsLog10, 2795 SDNodeFlags Flags) const { 2796 EVT VT = Src.getValueType(); 2797 unsigned LogOp = 2798 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2; 2799 2800 double Log2BaseInverted = 2801 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2; 2802 2803 if (VT == MVT::f32) { 2804 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags); 2805 if (ScaledInput) { 2806 SDValue LogSrc = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags); 2807 SDValue ScaledResultOffset = 2808 DAG.getConstantFP(-32.0 * Log2BaseInverted, SL, VT); 2809 2810 SDValue Zero = DAG.getConstantFP(0.0f, SL, VT); 2811 2812 SDValue ResultOffset = DAG.getNode(ISD::SELECT, SL, VT, IsScaled, 2813 ScaledResultOffset, Zero, Flags); 2814 2815 SDValue Log2Inv = DAG.getConstantFP(Log2BaseInverted, SL, VT); 2816 2817 if (Subtarget->hasFastFMAF32()) 2818 return DAG.getNode(ISD::FMA, SL, VT, LogSrc, Log2Inv, ResultOffset, 2819 Flags); 2820 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, LogSrc, Log2Inv, Flags); 2821 return DAG.getNode(ISD::FADD, SL, VT, Mul, ResultOffset); 2822 } 2823 } 2824 2825 SDValue Log2Operand = DAG.getNode(LogOp, SL, VT, Src, Flags); 2826 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT); 2827 2828 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand, 2829 Flags); 2830 } 2831 2832 SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const { 2833 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals. 2834 // If we have to handle denormals, scale up the input and adjust the result. 2835 2836 SDLoc SL(Op); 2837 EVT VT = Op.getValueType(); 2838 SDValue Src = Op.getOperand(0); 2839 SDNodeFlags Flags = Op->getFlags(); 2840 2841 if (VT == MVT::f16) { 2842 // Nothing in half is a denormal when promoted to f32. 2843 assert(!Subtarget->has16BitInsts()); 2844 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); 2845 SDValue Log = DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Ext, Flags); 2846 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, 2847 DAG.getTargetConstant(0, SL, MVT::i32), Flags); 2848 } 2849 2850 assert(VT == MVT::f32); 2851 2852 if (!needsDenormHandlingF32(DAG, Src, Flags)) 2853 return DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Src, Flags); 2854 2855 // bool needs_scaling = x < -0x1.f80000p+6f; 2856 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f); 2857 2858 // -nextafter(128.0, -1) 2859 SDValue RangeCheckConst = DAG.getConstantFP(-0x1.f80000p+6f, SL, VT); 2860 2861 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 2862 2863 SDValue NeedsScaling = 2864 DAG.getSetCC(SL, SetCCVT, Src, RangeCheckConst, ISD::SETOLT); 2865 2866 SDValue SixtyFour = DAG.getConstantFP(0x1.0p+6f, SL, VT); 2867 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); 2868 2869 SDValue AddOffset = 2870 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, SixtyFour, Zero); 2871 2872 SDValue AddInput = DAG.getNode(ISD::FADD, SL, VT, Src, AddOffset, Flags); 2873 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, AddInput, Flags); 2874 2875 SDValue TwoExpNeg64 = DAG.getConstantFP(0x1.0p-64f, SL, VT); 2876 SDValue One = DAG.getConstantFP(1.0, SL, VT); 2877 SDValue ResultScale = 2878 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, TwoExpNeg64, One); 2879 2880 return DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScale, Flags); 2881 } 2882 2883 SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL, 2884 SelectionDAG &DAG, 2885 SDNodeFlags Flags) const { 2886 EVT VT = X.getValueType(); 2887 const SDValue Log2E = DAG.getConstantFP(numbers::log2e, SL, VT); 2888 2889 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) { 2890 // exp2(M_LOG2E_F * f); 2891 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Log2E, Flags); 2892 return DAG.getNode(VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP 2893 : (unsigned)ISD::FEXP2, 2894 SL, VT, Mul, Flags); 2895 } 2896 2897 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 2898 2899 SDValue Threshold = DAG.getConstantFP(-0x1.5d58a0p+6f, SL, VT); 2900 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); 2901 2902 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+6f, SL, VT); 2903 2904 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); 2905 2906 SDValue AdjustedX = 2907 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); 2908 2909 SDValue ExpInput = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, Log2E, Flags); 2910 2911 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, ExpInput, Flags); 2912 2913 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.969d48p-93f, SL, VT); 2914 SDValue AdjustedResult = 2915 DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScaleFactor, Flags); 2916 2917 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, Exp2, 2918 Flags); 2919 } 2920 2921 /// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be 2922 /// handled correctly. 2923 SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL, 2924 SelectionDAG &DAG, 2925 SDNodeFlags Flags) const { 2926 const EVT VT = X.getValueType(); 2927 const unsigned Exp2Op = VT == MVT::f32 ? AMDGPUISD::EXP : ISD::FEXP2; 2928 2929 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) { 2930 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f); 2931 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT); 2932 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT); 2933 2934 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, X, K0, Flags); 2935 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags); 2936 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, X, K1, Flags); 2937 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags); 2938 return DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1); 2939 } 2940 2941 // bool s = x < -0x1.2f7030p+5f; 2942 // x += s ? 0x1.0p+5f : 0.0f; 2943 // exp10 = exp2(x * 0x1.a92000p+1f) * 2944 // exp2(x * 0x1.4f0978p-11f) * 2945 // (s ? 0x1.9f623ep-107f : 1.0f); 2946 2947 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 2948 2949 SDValue Threshold = DAG.getConstantFP(-0x1.2f7030p+5f, SL, VT); 2950 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); 2951 2952 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+5f, SL, VT); 2953 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); 2954 SDValue AdjustedX = 2955 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); 2956 2957 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT); 2958 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT); 2959 2960 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K0, Flags); 2961 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags); 2962 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K1, Flags); 2963 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags); 2964 2965 SDValue MulExps = DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1, Flags); 2966 2967 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.9f623ep-107f, SL, VT); 2968 SDValue AdjustedResult = 2969 DAG.getNode(ISD::FMUL, SL, VT, MulExps, ResultScaleFactor, Flags); 2970 2971 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, MulExps, 2972 Flags); 2973 } 2974 2975 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const { 2976 EVT VT = Op.getValueType(); 2977 SDLoc SL(Op); 2978 SDValue X = Op.getOperand(0); 2979 SDNodeFlags Flags = Op->getFlags(); 2980 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10; 2981 2982 if (VT.getScalarType() == MVT::f16) { 2983 // v_exp_f16 (fmul x, log2e) 2984 if (allowApproxFunc(DAG, Flags)) // TODO: Does this really require fast? 2985 return lowerFEXPUnsafe(X, SL, DAG, Flags); 2986 2987 if (VT.isVector()) 2988 return SDValue(); 2989 2990 // exp(f16 x) -> 2991 // fptrunc (v_exp_f32 (fmul (fpext x), log2e)) 2992 2993 // Nothing in half is a denormal when promoted to f32. 2994 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, X, Flags); 2995 SDValue Lowered = lowerFEXPUnsafe(Ext, SL, DAG, Flags); 2996 return DAG.getNode(ISD::FP_ROUND, SL, VT, Lowered, 2997 DAG.getTargetConstant(0, SL, MVT::i32), Flags); 2998 } 2999 3000 assert(VT == MVT::f32); 3001 3002 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying 3003 // library behavior. Also, is known-not-daz source sufficient? 3004 if (allowApproxFunc(DAG, Flags)) { 3005 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags) 3006 : lowerFEXPUnsafe(X, SL, DAG, Flags); 3007 } 3008 3009 // Algorithm: 3010 // 3011 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64) 3012 // 3013 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer 3014 // n = 64*m + j, 0 <= j < 64 3015 // 3016 // e^x = 2^((64*m + j + f)/64) 3017 // = (2^m) * (2^(j/64)) * 2^(f/64) 3018 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64)) 3019 // 3020 // f = x*(64/ln(2)) - n 3021 // r = f*(ln(2)/64) = x - n*(ln(2)/64) 3022 // 3023 // e^x = (2^m) * (2^(j/64)) * e^r 3024 // 3025 // (2^(j/64)) is precomputed 3026 // 3027 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5! 3028 // e^r = 1 + q 3029 // 3030 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5! 3031 // 3032 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) ) 3033 SDNodeFlags FlagsNoContract = Flags; 3034 FlagsNoContract.setAllowContract(false); 3035 3036 SDValue PH, PL; 3037 if (Subtarget->hasFastFMAF32()) { 3038 const float c_exp = numbers::log2ef; 3039 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits 3040 const float c_exp10 = 0x1.a934f0p+1f; 3041 const float cc_exp10 = 0x1.2f346ep-24f; 3042 3043 SDValue C = DAG.getConstantFP(IsExp10 ? c_exp10 : c_exp, SL, VT); 3044 SDValue CC = DAG.getConstantFP(IsExp10 ? cc_exp10 : cc_exp, SL, VT); 3045 3046 PH = DAG.getNode(ISD::FMUL, SL, VT, X, C, Flags); 3047 SDValue NegPH = DAG.getNode(ISD::FNEG, SL, VT, PH, Flags); 3048 SDValue FMA0 = DAG.getNode(ISD::FMA, SL, VT, X, C, NegPH, Flags); 3049 PL = DAG.getNode(ISD::FMA, SL, VT, X, CC, FMA0, Flags); 3050 } else { 3051 const float ch_exp = 0x1.714000p+0f; 3052 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits 3053 3054 const float ch_exp10 = 0x1.a92000p+1f; 3055 const float cl_exp10 = 0x1.4f0978p-11f; 3056 3057 SDValue CH = DAG.getConstantFP(IsExp10 ? ch_exp10 : ch_exp, SL, VT); 3058 SDValue CL = DAG.getConstantFP(IsExp10 ? cl_exp10 : cl_exp, SL, VT); 3059 3060 SDValue XAsInt = DAG.getNode(ISD::BITCAST, SL, MVT::i32, X); 3061 SDValue MaskConst = DAG.getConstant(0xfffff000, SL, MVT::i32); 3062 SDValue XHAsInt = DAG.getNode(ISD::AND, SL, MVT::i32, XAsInt, MaskConst); 3063 SDValue XH = DAG.getNode(ISD::BITCAST, SL, VT, XHAsInt); 3064 SDValue XL = DAG.getNode(ISD::FSUB, SL, VT, X, XH, Flags); 3065 3066 PH = DAG.getNode(ISD::FMUL, SL, VT, XH, CH, Flags); 3067 3068 SDValue XLCL = DAG.getNode(ISD::FMUL, SL, VT, XL, CL, Flags); 3069 SDValue Mad0 = getMad(DAG, SL, VT, XL, CH, XLCL, Flags); 3070 PL = getMad(DAG, SL, VT, XH, CL, Mad0, Flags); 3071 } 3072 3073 SDValue E = DAG.getNode(ISD::FROUNDEVEN, SL, VT, PH, Flags); 3074 3075 // It is unsafe to contract this fsub into the PH multiply. 3076 SDValue PHSubE = DAG.getNode(ISD::FSUB, SL, VT, PH, E, FlagsNoContract); 3077 3078 SDValue A = DAG.getNode(ISD::FADD, SL, VT, PHSubE, PL, Flags); 3079 SDValue IntE = DAG.getNode(ISD::FP_TO_SINT, SL, MVT::i32, E); 3080 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, A, Flags); 3081 3082 SDValue R = DAG.getNode(ISD::FLDEXP, SL, VT, Exp2, IntE, Flags); 3083 3084 SDValue UnderflowCheckConst = 3085 DAG.getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT); 3086 3087 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 3088 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); 3089 SDValue Underflow = 3090 DAG.getSetCC(SL, SetCCVT, X, UnderflowCheckConst, ISD::SETOLT); 3091 3092 R = DAG.getNode(ISD::SELECT, SL, VT, Underflow, Zero, R); 3093 const auto &Options = getTargetMachine().Options; 3094 3095 if (!Flags.hasNoInfs() && !Options.NoInfsFPMath) { 3096 SDValue OverflowCheckConst = 3097 DAG.getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT); 3098 SDValue Overflow = 3099 DAG.getSetCC(SL, SetCCVT, X, OverflowCheckConst, ISD::SETOGT); 3100 SDValue Inf = 3101 DAG.getConstantFP(APFloat::getInf(APFloat::IEEEsingle()), SL, VT); 3102 R = DAG.getNode(ISD::SELECT, SL, VT, Overflow, Inf, R); 3103 } 3104 3105 return R; 3106 } 3107 3108 static bool isCtlzOpc(unsigned Opc) { 3109 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF; 3110 } 3111 3112 static bool isCttzOpc(unsigned Opc) { 3113 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_UNDEF; 3114 } 3115 3116 SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op, 3117 SelectionDAG &DAG) const { 3118 auto SL = SDLoc(Op); 3119 auto Opc = Op.getOpcode(); 3120 auto Arg = Op.getOperand(0u); 3121 auto ResultVT = Op.getValueType(); 3122 3123 if (ResultVT != MVT::i8 && ResultVT != MVT::i16) 3124 return {}; 3125 3126 assert(isCtlzOpc(Opc)); 3127 assert(ResultVT == Arg.getValueType()); 3128 3129 const uint64_t NumBits = ResultVT.getFixedSizeInBits(); 3130 SDValue NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32); 3131 SDValue NewOp; 3132 3133 if (Opc == ISD::CTLZ_ZERO_UNDEF) { 3134 NewOp = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Arg); 3135 NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits); 3136 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); 3137 } else { 3138 NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); 3139 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); 3140 NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits); 3141 } 3142 3143 return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp); 3144 } 3145 3146 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const { 3147 SDLoc SL(Op); 3148 SDValue Src = Op.getOperand(0); 3149 3150 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode())); 3151 bool Ctlz = isCtlzOpc(Op.getOpcode()); 3152 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32; 3153 3154 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF || 3155 Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF; 3156 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64; 3157 3158 if (Src.getValueType() == MVT::i32 || Is64BitScalar) { 3159 // (ctlz hi:lo) -> (umin (ffbh src), 32) 3160 // (cttz hi:lo) -> (umin (ffbl src), 32) 3161 // (ctlz_zero_undef src) -> (ffbh src) 3162 // (cttz_zero_undef src) -> (ffbl src) 3163 3164 // 64-bit scalar version produce 32-bit result 3165 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64) 3166 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64) 3167 // (ctlz_zero_undef src) -> (S_FLBIT_I32_B64 src) 3168 // (cttz_zero_undef src) -> (S_FF1_I32_B64 src) 3169 SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src); 3170 if (!ZeroUndef) { 3171 const SDValue ConstVal = DAG.getConstant( 3172 Op.getValueType().getScalarSizeInBits(), SL, MVT::i32); 3173 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal); 3174 } 3175 return DAG.getNode(ISD::ZERO_EXTEND, SL, Src.getValueType(), NewOpr); 3176 } 3177 3178 SDValue Lo, Hi; 3179 std::tie(Lo, Hi) = split64BitValue(Src, DAG); 3180 3181 SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo); 3182 SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi); 3183 3184 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64) 3185 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64) 3186 // (ctlz_zero_undef hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32)) 3187 // (cttz_zero_undef hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo)) 3188 3189 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT; 3190 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32); 3191 if (Ctlz) 3192 OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32); 3193 else 3194 OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32); 3195 3196 SDValue NewOpr; 3197 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi); 3198 if (!ZeroUndef) { 3199 const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32); 3200 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64); 3201 } 3202 3203 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); 3204 } 3205 3206 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, 3207 bool Signed) const { 3208 // The regular method converting a 64-bit integer to float roughly consists of 3209 // 2 steps: normalization and rounding. In fact, after normalization, the 3210 // conversion from a 64-bit integer to a float is essentially the same as the 3211 // one from a 32-bit integer. The only difference is that it has more 3212 // trailing bits to be rounded. To leverage the native 32-bit conversion, a 3213 // 64-bit integer could be preprocessed and fit into a 32-bit integer then 3214 // converted into the correct float number. The basic steps for the unsigned 3215 // conversion are illustrated in the following pseudo code: 3216 // 3217 // f32 uitofp(i64 u) { 3218 // i32 hi, lo = split(u); 3219 // // Only count the leading zeros in hi as we have native support of the 3220 // // conversion from i32 to f32. If hi is all 0s, the conversion is 3221 // // reduced to a 32-bit one automatically. 3222 // i32 shamt = clz(hi); // Return 32 if hi is all 0s. 3223 // u <<= shamt; 3224 // hi, lo = split(u); 3225 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo. 3226 // // convert it as a 32-bit integer and scale the result back. 3227 // return uitofp(hi) * 2^(32 - shamt); 3228 // } 3229 // 3230 // The signed one follows the same principle but uses 'ffbh_i32' to count its 3231 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is 3232 // converted instead followed by negation based its sign bit. 3233 3234 SDLoc SL(Op); 3235 SDValue Src = Op.getOperand(0); 3236 3237 SDValue Lo, Hi; 3238 std::tie(Lo, Hi) = split64BitValue(Src, DAG); 3239 SDValue Sign; 3240 SDValue ShAmt; 3241 if (Signed && Subtarget->isGCN()) { 3242 // We also need to consider the sign bit in Lo if Hi has just sign bits, 3243 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into 3244 // account. That is, the maximal shift is 3245 // - 32 if Lo and Hi have opposite signs; 3246 // - 33 if Lo and Hi have the same sign. 3247 // 3248 // Or, MaxShAmt = 33 + OppositeSign, where 3249 // 3250 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is 3251 // - -1 if Lo and Hi have opposite signs; and 3252 // - 0 otherwise. 3253 // 3254 // All in all, ShAmt is calculated as 3255 // 3256 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1. 3257 // 3258 // or 3259 // 3260 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31). 3261 // 3262 // to reduce the critical path. 3263 SDValue OppositeSign = DAG.getNode( 3264 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi), 3265 DAG.getConstant(31, SL, MVT::i32)); 3266 SDValue MaxShAmt = 3267 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), 3268 OppositeSign); 3269 // Count the leading sign bits. 3270 ShAmt = DAG.getNode(AMDGPUISD::FFBH_I32, SL, MVT::i32, Hi); 3271 // Different from unsigned conversion, the shift should be one bit less to 3272 // preserve the sign bit. 3273 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt, 3274 DAG.getConstant(1, SL, MVT::i32)); 3275 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt); 3276 } else { 3277 if (Signed) { 3278 // Without 'ffbh_i32', only leading zeros could be counted. Take the 3279 // absolute value first. 3280 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src, 3281 DAG.getConstant(63, SL, MVT::i64)); 3282 SDValue Abs = 3283 DAG.getNode(ISD::XOR, SL, MVT::i64, 3284 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign); 3285 std::tie(Lo, Hi) = split64BitValue(Abs, DAG); 3286 } 3287 // Count the leading zeros. 3288 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi); 3289 // The shift amount for signed integers is [0, 32]. 3290 } 3291 // Normalize the given 64-bit integer. 3292 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt); 3293 // Split it again. 3294 std::tie(Lo, Hi) = split64BitValue(Norm, DAG); 3295 // Calculate the adjust bit for rounding. 3296 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo) 3297 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32, 3298 DAG.getConstant(1, SL, MVT::i32), Lo); 3299 // Get the 32-bit normalized integer. 3300 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust); 3301 // Convert the normalized 32-bit integer into f32. 3302 unsigned Opc = 3303 (Signed && Subtarget->isGCN()) ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; 3304 SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm); 3305 3306 // Finally, need to scale back the converted floating number as the original 3307 // 64-bit integer is converted as a 32-bit one. 3308 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), 3309 ShAmt); 3310 // On GCN, use LDEXP directly. 3311 if (Subtarget->isGCN()) 3312 return DAG.getNode(ISD::FLDEXP, SL, MVT::f32, FVal, ShAmt); 3313 3314 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent 3315 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit 3316 // exponent is enough to avoid overflowing into the sign bit. 3317 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt, 3318 DAG.getConstant(23, SL, MVT::i32)); 3319 SDValue IVal = 3320 DAG.getNode(ISD::ADD, SL, MVT::i32, 3321 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp); 3322 if (Signed) { 3323 // Set the sign bit. 3324 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32, 3325 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign), 3326 DAG.getConstant(31, SL, MVT::i32)); 3327 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign); 3328 } 3329 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal); 3330 } 3331 3332 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, 3333 bool Signed) const { 3334 SDLoc SL(Op); 3335 SDValue Src = Op.getOperand(0); 3336 3337 SDValue Lo, Hi; 3338 std::tie(Lo, Hi) = split64BitValue(Src, DAG); 3339 3340 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, 3341 SL, MVT::f64, Hi); 3342 3343 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); 3344 3345 SDValue LdExp = DAG.getNode(ISD::FLDEXP, SL, MVT::f64, CvtHi, 3346 DAG.getConstant(32, SL, MVT::i32)); 3347 // TODO: Should this propagate fast-math-flags? 3348 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); 3349 } 3350 3351 SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op, 3352 SelectionDAG &DAG) const { 3353 // TODO: Factor out code common with LowerSINT_TO_FP. 3354 EVT DestVT = Op.getValueType(); 3355 SDValue Src = Op.getOperand(0); 3356 EVT SrcVT = Src.getValueType(); 3357 3358 if (SrcVT == MVT::i16) { 3359 if (DestVT == MVT::f16) 3360 return Op; 3361 SDLoc DL(Op); 3362 3363 // Promote src to i32 3364 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src); 3365 return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext); 3366 } 3367 3368 if (DestVT == MVT::bf16) { 3369 SDLoc SL(Op); 3370 SDValue ToF32 = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f32, Src); 3371 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true); 3372 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); 3373 } 3374 3375 if (SrcVT != MVT::i64) 3376 return Op; 3377 3378 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { 3379 SDLoc DL(Op); 3380 3381 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); 3382 SDValue FPRoundFlag = 3383 DAG.getIntPtrConstant(0, SDLoc(Op), /*isTarget=*/true); 3384 SDValue FPRound = 3385 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); 3386 3387 return FPRound; 3388 } 3389 3390 if (DestVT == MVT::f32) 3391 return LowerINT_TO_FP32(Op, DAG, false); 3392 3393 assert(DestVT == MVT::f64); 3394 return LowerINT_TO_FP64(Op, DAG, false); 3395 } 3396 3397 SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op, 3398 SelectionDAG &DAG) const { 3399 EVT DestVT = Op.getValueType(); 3400 3401 SDValue Src = Op.getOperand(0); 3402 EVT SrcVT = Src.getValueType(); 3403 3404 if (SrcVT == MVT::i16) { 3405 if (DestVT == MVT::f16) 3406 return Op; 3407 3408 SDLoc DL(Op); 3409 // Promote src to i32 3410 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src); 3411 return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext); 3412 } 3413 3414 if (DestVT == MVT::bf16) { 3415 SDLoc SL(Op); 3416 SDValue ToF32 = DAG.getNode(ISD::SINT_TO_FP, SL, MVT::f32, Src); 3417 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true); 3418 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); 3419 } 3420 3421 if (SrcVT != MVT::i64) 3422 return Op; 3423 3424 // TODO: Factor out code common with LowerUINT_TO_FP. 3425 3426 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { 3427 SDLoc DL(Op); 3428 SDValue Src = Op.getOperand(0); 3429 3430 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); 3431 SDValue FPRoundFlag = 3432 DAG.getIntPtrConstant(0, SDLoc(Op), /*isTarget=*/true); 3433 SDValue FPRound = 3434 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); 3435 3436 return FPRound; 3437 } 3438 3439 if (DestVT == MVT::f32) 3440 return LowerINT_TO_FP32(Op, DAG, true); 3441 3442 assert(DestVT == MVT::f64); 3443 return LowerINT_TO_FP64(Op, DAG, true); 3444 } 3445 3446 SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG, 3447 bool Signed) const { 3448 SDLoc SL(Op); 3449 3450 SDValue Src = Op.getOperand(0); 3451 EVT SrcVT = Src.getValueType(); 3452 3453 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64); 3454 3455 // The basic idea of converting a floating point number into a pair of 32-bit 3456 // integers is illustrated as follows: 3457 // 3458 // tf := trunc(val); 3459 // hif := floor(tf * 2^-32); 3460 // lof := tf - hif * 2^32; // lof is always positive due to floor. 3461 // hi := fptoi(hif); 3462 // lo := fptoi(lof); 3463 // 3464 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src); 3465 SDValue Sign; 3466 if (Signed && SrcVT == MVT::f32) { 3467 // However, a 32-bit floating point number has only 23 bits mantissa and 3468 // it's not enough to hold all the significant bits of `lof` if val is 3469 // negative. To avoid the loss of precision, We need to take the absolute 3470 // value after truncating and flip the result back based on the original 3471 // signedness. 3472 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32, 3473 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc), 3474 DAG.getConstant(31, SL, MVT::i32)); 3475 Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc); 3476 } 3477 3478 SDValue K0, K1; 3479 if (SrcVT == MVT::f64) { 3480 K0 = DAG.getConstantFP( 3481 llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), SL, 3482 SrcVT); 3483 K1 = DAG.getConstantFP( 3484 llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), SL, 3485 SrcVT); 3486 } else { 3487 K0 = DAG.getConstantFP( 3488 llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), SL, SrcVT); 3489 K1 = DAG.getConstantFP( 3490 llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), SL, SrcVT); 3491 } 3492 // TODO: Should this propagate fast-math-flags? 3493 SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0); 3494 3495 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul); 3496 3497 SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc); 3498 3499 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT 3500 : ISD::FP_TO_UINT, 3501 SL, MVT::i32, FloorMul); 3502 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); 3503 3504 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, 3505 DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi})); 3506 3507 if (Signed && SrcVT == MVT::f32) { 3508 assert(Sign); 3509 // Flip the result based on the signedness, which is either all 0s or 1s. 3510 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64, 3511 DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign})); 3512 // r := xor(r, sign) - sign; 3513 Result = 3514 DAG.getNode(ISD::SUB, SL, MVT::i64, 3515 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign); 3516 } 3517 3518 return Result; 3519 } 3520 3521 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const { 3522 SDLoc DL(Op); 3523 SDValue N0 = Op.getOperand(0); 3524 3525 // Convert to target node to get known bits 3526 if (N0.getValueType() == MVT::f32) 3527 return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0); 3528 3529 if (getTargetMachine().Options.UnsafeFPMath) { 3530 // There is a generic expand for FP_TO_FP16 with unsafe fast math. 3531 return SDValue(); 3532 } 3533 3534 assert(N0.getSimpleValueType() == MVT::f64); 3535 3536 // f64 -> f16 conversion using round-to-nearest-even rounding mode. 3537 const unsigned ExpMask = 0x7ff; 3538 const unsigned ExpBiasf64 = 1023; 3539 const unsigned ExpBiasf16 = 15; 3540 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); 3541 SDValue One = DAG.getConstant(1, DL, MVT::i32); 3542 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); 3543 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, 3544 DAG.getConstant(32, DL, MVT::i64)); 3545 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32); 3546 U = DAG.getZExtOrTrunc(U, DL, MVT::i32); 3547 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, 3548 DAG.getConstant(20, DL, MVT::i64)); 3549 E = DAG.getNode(ISD::AND, DL, MVT::i32, E, 3550 DAG.getConstant(ExpMask, DL, MVT::i32)); 3551 // Subtract the fp64 exponent bias (1023) to get the real exponent and 3552 // add the f16 bias (15) to get the biased exponent for the f16 format. 3553 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, 3554 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32)); 3555 3556 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, 3557 DAG.getConstant(8, DL, MVT::i32)); 3558 M = DAG.getNode(ISD::AND, DL, MVT::i32, M, 3559 DAG.getConstant(0xffe, DL, MVT::i32)); 3560 3561 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, 3562 DAG.getConstant(0x1ff, DL, MVT::i32)); 3563 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); 3564 3565 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ); 3566 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); 3567 3568 // (M != 0 ? 0x0200 : 0) | 0x7c00; 3569 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, 3570 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32), 3571 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); 3572 3573 // N = M | (E << 12); 3574 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, 3575 DAG.getNode(ISD::SHL, DL, MVT::i32, E, 3576 DAG.getConstant(12, DL, MVT::i32))); 3577 3578 // B = clamp(1-E, 0, 13); 3579 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, 3580 One, E); 3581 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); 3582 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, 3583 DAG.getConstant(13, DL, MVT::i32)); 3584 3585 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, 3586 DAG.getConstant(0x1000, DL, MVT::i32)); 3587 3588 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); 3589 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); 3590 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE); 3591 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); 3592 3593 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT); 3594 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, 3595 DAG.getConstant(0x7, DL, MVT::i32)); 3596 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, 3597 DAG.getConstant(2, DL, MVT::i32)); 3598 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32), 3599 One, Zero, ISD::SETEQ); 3600 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32), 3601 One, Zero, ISD::SETGT); 3602 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); 3603 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); 3604 3605 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32), 3606 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); 3607 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32), 3608 I, V, ISD::SETEQ); 3609 3610 // Extract the sign bit. 3611 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, 3612 DAG.getConstant(16, DL, MVT::i32)); 3613 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, 3614 DAG.getConstant(0x8000, DL, MVT::i32)); 3615 3616 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); 3617 return DAG.getZExtOrTrunc(V, DL, Op.getValueType()); 3618 } 3619 3620 SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op, 3621 SelectionDAG &DAG) const { 3622 SDValue Src = Op.getOperand(0); 3623 unsigned OpOpcode = Op.getOpcode(); 3624 EVT SrcVT = Src.getValueType(); 3625 EVT DestVT = Op.getValueType(); 3626 3627 // Will be selected natively 3628 if (SrcVT == MVT::f16 && DestVT == MVT::i16) 3629 return Op; 3630 3631 if (SrcVT == MVT::bf16) { 3632 SDLoc DL(Op); 3633 SDValue PromotedSrc = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); 3634 return DAG.getNode(Op.getOpcode(), DL, DestVT, PromotedSrc); 3635 } 3636 3637 // Promote i16 to i32 3638 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) { 3639 SDLoc DL(Op); 3640 3641 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); 3642 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32); 3643 } 3644 3645 if (DestVT != MVT::i64) 3646 return Op; 3647 3648 if (SrcVT == MVT::f16 || 3649 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) { 3650 SDLoc DL(Op); 3651 3652 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); 3653 unsigned Ext = 3654 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 3655 return DAG.getNode(Ext, DL, MVT::i64, FpToInt32); 3656 } 3657 3658 if (SrcVT == MVT::f32 || SrcVT == MVT::f64) 3659 return LowerFP_TO_INT64(Op, DAG, OpOpcode == ISD::FP_TO_SINT); 3660 3661 return SDValue(); 3662 } 3663 3664 SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 3665 SelectionDAG &DAG) const { 3666 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 3667 MVT VT = Op.getSimpleValueType(); 3668 MVT ScalarVT = VT.getScalarType(); 3669 3670 assert(VT.isVector()); 3671 3672 SDValue Src = Op.getOperand(0); 3673 SDLoc DL(Op); 3674 3675 // TODO: Don't scalarize on Evergreen? 3676 unsigned NElts = VT.getVectorNumElements(); 3677 SmallVector<SDValue, 8> Args; 3678 DAG.ExtractVectorElements(Src, Args, 0, NElts); 3679 3680 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType()); 3681 for (unsigned I = 0; I < NElts; ++I) 3682 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp); 3683 3684 return DAG.getBuildVector(VT, DL, Args); 3685 } 3686 3687 //===----------------------------------------------------------------------===// 3688 // Custom DAG optimizations 3689 //===----------------------------------------------------------------------===// 3690 3691 static bool isU24(SDValue Op, SelectionDAG &DAG) { 3692 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24; 3693 } 3694 3695 static bool isI24(SDValue Op, SelectionDAG &DAG) { 3696 EVT VT = Op.getValueType(); 3697 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated 3698 // as unsigned 24-bit values. 3699 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24; 3700 } 3701 3702 static SDValue simplifyMul24(SDNode *Node24, 3703 TargetLowering::DAGCombinerInfo &DCI) { 3704 SelectionDAG &DAG = DCI.DAG; 3705 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3706 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN; 3707 3708 SDValue LHS = IsIntrin ? Node24->getOperand(1) : Node24->getOperand(0); 3709 SDValue RHS = IsIntrin ? Node24->getOperand(2) : Node24->getOperand(1); 3710 unsigned NewOpcode = Node24->getOpcode(); 3711 if (IsIntrin) { 3712 unsigned IID = Node24->getConstantOperandVal(0); 3713 switch (IID) { 3714 case Intrinsic::amdgcn_mul_i24: 3715 NewOpcode = AMDGPUISD::MUL_I24; 3716 break; 3717 case Intrinsic::amdgcn_mul_u24: 3718 NewOpcode = AMDGPUISD::MUL_U24; 3719 break; 3720 case Intrinsic::amdgcn_mulhi_i24: 3721 NewOpcode = AMDGPUISD::MULHI_I24; 3722 break; 3723 case Intrinsic::amdgcn_mulhi_u24: 3724 NewOpcode = AMDGPUISD::MULHI_U24; 3725 break; 3726 default: 3727 llvm_unreachable("Expected 24-bit mul intrinsic"); 3728 } 3729 } 3730 3731 APInt Demanded = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 24); 3732 3733 // First try to simplify using SimplifyMultipleUseDemandedBits which allows 3734 // the operands to have other uses, but will only perform simplifications that 3735 // involve bypassing some nodes for this user. 3736 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(LHS, Demanded, DAG); 3737 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(RHS, Demanded, DAG); 3738 if (DemandedLHS || DemandedRHS) 3739 return DAG.getNode(NewOpcode, SDLoc(Node24), Node24->getVTList(), 3740 DemandedLHS ? DemandedLHS : LHS, 3741 DemandedRHS ? DemandedRHS : RHS); 3742 3743 // Now try SimplifyDemandedBits which can simplify the nodes used by our 3744 // operands if this node is the only user. 3745 if (TLI.SimplifyDemandedBits(LHS, Demanded, DCI)) 3746 return SDValue(Node24, 0); 3747 if (TLI.SimplifyDemandedBits(RHS, Demanded, DCI)) 3748 return SDValue(Node24, 0); 3749 3750 return SDValue(); 3751 } 3752 3753 template <typename IntTy> 3754 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset, 3755 uint32_t Width, const SDLoc &DL) { 3756 if (Width + Offset < 32) { 3757 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width); 3758 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width); 3759 return DAG.getConstant(Result, DL, MVT::i32); 3760 } 3761 3762 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32); 3763 } 3764 3765 static bool hasVolatileUser(SDNode *Val) { 3766 for (SDNode *U : Val->uses()) { 3767 if (MemSDNode *M = dyn_cast<MemSDNode>(U)) { 3768 if (M->isVolatile()) 3769 return true; 3770 } 3771 } 3772 3773 return false; 3774 } 3775 3776 bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const { 3777 // i32 vectors are the canonical memory type. 3778 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT)) 3779 return false; 3780 3781 if (!VT.isByteSized()) 3782 return false; 3783 3784 unsigned Size = VT.getStoreSize(); 3785 3786 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector()) 3787 return false; 3788 3789 if (Size == 3 || (Size > 4 && (Size % 4 != 0))) 3790 return false; 3791 3792 return true; 3793 } 3794 3795 // Replace load of an illegal type with a store of a bitcast to a friendlier 3796 // type. 3797 SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N, 3798 DAGCombinerInfo &DCI) const { 3799 if (!DCI.isBeforeLegalize()) 3800 return SDValue(); 3801 3802 LoadSDNode *LN = cast<LoadSDNode>(N); 3803 if (!LN->isSimple() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN)) 3804 return SDValue(); 3805 3806 SDLoc SL(N); 3807 SelectionDAG &DAG = DCI.DAG; 3808 EVT VT = LN->getMemoryVT(); 3809 3810 unsigned Size = VT.getStoreSize(); 3811 Align Alignment = LN->getAlign(); 3812 if (Alignment < Size && isTypeLegal(VT)) { 3813 unsigned IsFast; 3814 unsigned AS = LN->getAddressSpace(); 3815 3816 // Expand unaligned loads earlier than legalization. Due to visitation order 3817 // problems during legalization, the emitted instructions to pack and unpack 3818 // the bytes again are not eliminated in the case of an unaligned copy. 3819 if (!allowsMisalignedMemoryAccesses( 3820 VT, AS, Alignment, LN->getMemOperand()->getFlags(), &IsFast)) { 3821 if (VT.isVector()) 3822 return SplitVectorLoad(SDValue(LN, 0), DAG); 3823 3824 SDValue Ops[2]; 3825 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG); 3826 3827 return DAG.getMergeValues(Ops, SDLoc(N)); 3828 } 3829 3830 if (!IsFast) 3831 return SDValue(); 3832 } 3833 3834 if (!shouldCombineMemoryType(VT)) 3835 return SDValue(); 3836 3837 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); 3838 3839 SDValue NewLoad 3840 = DAG.getLoad(NewVT, SL, LN->getChain(), 3841 LN->getBasePtr(), LN->getMemOperand()); 3842 3843 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad); 3844 DCI.CombineTo(N, BC, NewLoad.getValue(1)); 3845 return SDValue(N, 0); 3846 } 3847 3848 // Replace store of an illegal type with a store of a bitcast to a friendlier 3849 // type. 3850 SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N, 3851 DAGCombinerInfo &DCI) const { 3852 if (!DCI.isBeforeLegalize()) 3853 return SDValue(); 3854 3855 StoreSDNode *SN = cast<StoreSDNode>(N); 3856 if (!SN->isSimple() || !ISD::isNormalStore(SN)) 3857 return SDValue(); 3858 3859 EVT VT = SN->getMemoryVT(); 3860 unsigned Size = VT.getStoreSize(); 3861 3862 SDLoc SL(N); 3863 SelectionDAG &DAG = DCI.DAG; 3864 Align Alignment = SN->getAlign(); 3865 if (Alignment < Size && isTypeLegal(VT)) { 3866 unsigned IsFast; 3867 unsigned AS = SN->getAddressSpace(); 3868 3869 // Expand unaligned stores earlier than legalization. Due to visitation 3870 // order problems during legalization, the emitted instructions to pack and 3871 // unpack the bytes again are not eliminated in the case of an unaligned 3872 // copy. 3873 if (!allowsMisalignedMemoryAccesses( 3874 VT, AS, Alignment, SN->getMemOperand()->getFlags(), &IsFast)) { 3875 if (VT.isVector()) 3876 return SplitVectorStore(SDValue(SN, 0), DAG); 3877 3878 return expandUnalignedStore(SN, DAG); 3879 } 3880 3881 if (!IsFast) 3882 return SDValue(); 3883 } 3884 3885 if (!shouldCombineMemoryType(VT)) 3886 return SDValue(); 3887 3888 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); 3889 SDValue Val = SN->getValue(); 3890 3891 //DCI.AddToWorklist(Val.getNode()); 3892 3893 bool OtherUses = !Val.hasOneUse(); 3894 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val); 3895 if (OtherUses) { 3896 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal); 3897 DAG.ReplaceAllUsesOfValueWith(Val, CastBack); 3898 } 3899 3900 return DAG.getStore(SN->getChain(), SL, CastVal, 3901 SN->getBasePtr(), SN->getMemOperand()); 3902 } 3903 3904 // FIXME: This should go in generic DAG combiner with an isTruncateFree check, 3905 // but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU 3906 // issues. 3907 SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N, 3908 DAGCombinerInfo &DCI) const { 3909 SelectionDAG &DAG = DCI.DAG; 3910 SDValue N0 = N->getOperand(0); 3911 3912 // (vt2 (assertzext (truncate vt0:x), vt1)) -> 3913 // (vt2 (truncate (assertzext vt0:x, vt1))) 3914 if (N0.getOpcode() == ISD::TRUNCATE) { 3915 SDValue N1 = N->getOperand(1); 3916 EVT ExtVT = cast<VTSDNode>(N1)->getVT(); 3917 SDLoc SL(N); 3918 3919 SDValue Src = N0.getOperand(0); 3920 EVT SrcVT = Src.getValueType(); 3921 if (SrcVT.bitsGE(ExtVT)) { 3922 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1); 3923 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg); 3924 } 3925 } 3926 3927 return SDValue(); 3928 } 3929 3930 SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine( 3931 SDNode *N, DAGCombinerInfo &DCI) const { 3932 unsigned IID = N->getConstantOperandVal(0); 3933 switch (IID) { 3934 case Intrinsic::amdgcn_mul_i24: 3935 case Intrinsic::amdgcn_mul_u24: 3936 case Intrinsic::amdgcn_mulhi_i24: 3937 case Intrinsic::amdgcn_mulhi_u24: 3938 return simplifyMul24(N, DCI); 3939 case Intrinsic::amdgcn_fract: 3940 case Intrinsic::amdgcn_rsq: 3941 case Intrinsic::amdgcn_rcp_legacy: 3942 case Intrinsic::amdgcn_rsq_legacy: 3943 case Intrinsic::amdgcn_rsq_clamp: { 3944 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted 3945 SDValue Src = N->getOperand(1); 3946 return Src.isUndef() ? Src : SDValue(); 3947 } 3948 case Intrinsic::amdgcn_frexp_exp: { 3949 // frexp_exp (fneg x) -> frexp_exp x 3950 // frexp_exp (fabs x) -> frexp_exp x 3951 // frexp_exp (fneg (fabs x)) -> frexp_exp x 3952 SDValue Src = N->getOperand(1); 3953 SDValue PeekSign = peekFPSignOps(Src); 3954 if (PeekSign == Src) 3955 return SDValue(); 3956 return SDValue(DCI.DAG.UpdateNodeOperands(N, N->getOperand(0), PeekSign), 3957 0); 3958 } 3959 default: 3960 return SDValue(); 3961 } 3962 } 3963 3964 /// Split the 64-bit value \p LHS into two 32-bit components, and perform the 3965 /// binary operation \p Opc to it with the corresponding constant operands. 3966 SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl( 3967 DAGCombinerInfo &DCI, const SDLoc &SL, 3968 unsigned Opc, SDValue LHS, 3969 uint32_t ValLo, uint32_t ValHi) const { 3970 SelectionDAG &DAG = DCI.DAG; 3971 SDValue Lo, Hi; 3972 std::tie(Lo, Hi) = split64BitValue(LHS, DAG); 3973 3974 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32); 3975 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32); 3976 3977 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS); 3978 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS); 3979 3980 // Re-visit the ands. It's possible we eliminated one of them and it could 3981 // simplify the vector. 3982 DCI.AddToWorklist(Lo.getNode()); 3983 DCI.AddToWorklist(Hi.getNode()); 3984 3985 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd}); 3986 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); 3987 } 3988 3989 SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N, 3990 DAGCombinerInfo &DCI) const { 3991 EVT VT = N->getValueType(0); 3992 3993 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1)); 3994 if (!RHS) 3995 return SDValue(); 3996 3997 SDValue LHS = N->getOperand(0); 3998 unsigned RHSVal = RHS->getZExtValue(); 3999 if (!RHSVal) 4000 return LHS; 4001 4002 SDLoc SL(N); 4003 SelectionDAG &DAG = DCI.DAG; 4004 4005 switch (LHS->getOpcode()) { 4006 default: 4007 break; 4008 case ISD::ZERO_EXTEND: 4009 case ISD::SIGN_EXTEND: 4010 case ISD::ANY_EXTEND: { 4011 SDValue X = LHS->getOperand(0); 4012 4013 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 && 4014 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) { 4015 // Prefer build_vector as the canonical form if packed types are legal. 4016 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x 4017 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL, 4018 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) }); 4019 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); 4020 } 4021 4022 // shl (ext x) => zext (shl x), if shift does not overflow int 4023 if (VT != MVT::i64) 4024 break; 4025 KnownBits Known = DAG.computeKnownBits(X); 4026 unsigned LZ = Known.countMinLeadingZeros(); 4027 if (LZ < RHSVal) 4028 break; 4029 EVT XVT = X.getValueType(); 4030 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0)); 4031 return DAG.getZExtOrTrunc(Shl, SL, VT); 4032 } 4033 } 4034 4035 if (VT != MVT::i64) 4036 return SDValue(); 4037 4038 // i64 (shl x, C) -> (build_pair 0, (shl x, C -32)) 4039 4040 // On some subtargets, 64-bit shift is a quarter rate instruction. In the 4041 // common case, splitting this into a move and a 32-bit shift is faster and 4042 // the same code size. 4043 if (RHSVal < 32) 4044 return SDValue(); 4045 4046 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32); 4047 4048 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); 4049 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); 4050 4051 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); 4052 4053 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift}); 4054 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); 4055 } 4056 4057 SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N, 4058 DAGCombinerInfo &DCI) const { 4059 if (N->getValueType(0) != MVT::i64) 4060 return SDValue(); 4061 4062 const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1)); 4063 if (!RHS) 4064 return SDValue(); 4065 4066 SelectionDAG &DAG = DCI.DAG; 4067 SDLoc SL(N); 4068 unsigned RHSVal = RHS->getZExtValue(); 4069 4070 // (sra i64:x, 32) -> build_pair x, (sra hi_32(x), 31) 4071 if (RHSVal == 32) { 4072 SDValue Hi = getHiHalf64(N->getOperand(0), DAG); 4073 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, 4074 DAG.getConstant(31, SL, MVT::i32)); 4075 4076 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift}); 4077 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); 4078 } 4079 4080 // (sra i64:x, 63) -> build_pair (sra hi_32(x), 31), (sra hi_32(x), 31) 4081 if (RHSVal == 63) { 4082 SDValue Hi = getHiHalf64(N->getOperand(0), DAG); 4083 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, 4084 DAG.getConstant(31, SL, MVT::i32)); 4085 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift}); 4086 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); 4087 } 4088 4089 return SDValue(); 4090 } 4091 4092 SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N, 4093 DAGCombinerInfo &DCI) const { 4094 auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1)); 4095 if (!RHS) 4096 return SDValue(); 4097 4098 EVT VT = N->getValueType(0); 4099 SDValue LHS = N->getOperand(0); 4100 unsigned ShiftAmt = RHS->getZExtValue(); 4101 SelectionDAG &DAG = DCI.DAG; 4102 SDLoc SL(N); 4103 4104 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1) 4105 // this improves the ability to match BFE patterns in isel. 4106 if (LHS.getOpcode() == ISD::AND) { 4107 if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) { 4108 unsigned MaskIdx, MaskLen; 4109 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) && 4110 MaskIdx == ShiftAmt) { 4111 return DAG.getNode( 4112 ISD::AND, SL, VT, 4113 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)), 4114 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1))); 4115 } 4116 } 4117 } 4118 4119 if (VT != MVT::i64) 4120 return SDValue(); 4121 4122 if (ShiftAmt < 32) 4123 return SDValue(); 4124 4125 // srl i64:x, C for C >= 32 4126 // => 4127 // build_pair (srl hi_32(x), C - 32), 0 4128 SDValue Zero = DAG.getConstant(0, SL, MVT::i32); 4129 4130 SDValue Hi = getHiHalf64(LHS, DAG); 4131 4132 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32); 4133 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); 4134 4135 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero}); 4136 4137 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); 4138 } 4139 4140 SDValue AMDGPUTargetLowering::performTruncateCombine( 4141 SDNode *N, DAGCombinerInfo &DCI) const { 4142 SDLoc SL(N); 4143 SelectionDAG &DAG = DCI.DAG; 4144 EVT VT = N->getValueType(0); 4145 SDValue Src = N->getOperand(0); 4146 4147 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x) 4148 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) { 4149 SDValue Vec = Src.getOperand(0); 4150 if (Vec.getOpcode() == ISD::BUILD_VECTOR) { 4151 SDValue Elt0 = Vec.getOperand(0); 4152 EVT EltVT = Elt0.getValueType(); 4153 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) { 4154 if (EltVT.isFloatingPoint()) { 4155 Elt0 = DAG.getNode(ISD::BITCAST, SL, 4156 EltVT.changeTypeToInteger(), Elt0); 4157 } 4158 4159 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0); 4160 } 4161 } 4162 } 4163 4164 // Equivalent of above for accessing the high element of a vector as an 4165 // integer operation. 4166 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y) 4167 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) { 4168 if (auto K = isConstOrConstSplat(Src.getOperand(1))) { 4169 if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) { 4170 SDValue BV = stripBitcast(Src.getOperand(0)); 4171 if (BV.getOpcode() == ISD::BUILD_VECTOR && 4172 BV.getValueType().getVectorNumElements() == 2) { 4173 SDValue SrcElt = BV.getOperand(1); 4174 EVT SrcEltVT = SrcElt.getValueType(); 4175 if (SrcEltVT.isFloatingPoint()) { 4176 SrcElt = DAG.getNode(ISD::BITCAST, SL, 4177 SrcEltVT.changeTypeToInteger(), SrcElt); 4178 } 4179 4180 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt); 4181 } 4182 } 4183 } 4184 } 4185 4186 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit. 4187 // 4188 // i16 (trunc (srl i64:x, K)), K <= 16 -> 4189 // i16 (trunc (srl (i32 (trunc x), K))) 4190 if (VT.getScalarSizeInBits() < 32) { 4191 EVT SrcVT = Src.getValueType(); 4192 if (SrcVT.getScalarSizeInBits() > 32 && 4193 (Src.getOpcode() == ISD::SRL || 4194 Src.getOpcode() == ISD::SRA || 4195 Src.getOpcode() == ISD::SHL)) { 4196 SDValue Amt = Src.getOperand(1); 4197 KnownBits Known = DAG.computeKnownBits(Amt); 4198 4199 // - For left shifts, do the transform as long as the shift 4200 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31) 4201 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid 4202 // losing information stored in the high bits when truncating. 4203 const unsigned MaxCstSize = 4204 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits()); 4205 if (Known.getMaxValue().ule(MaxCstSize)) { 4206 EVT MidVT = VT.isVector() ? 4207 EVT::getVectorVT(*DAG.getContext(), MVT::i32, 4208 VT.getVectorNumElements()) : MVT::i32; 4209 4210 EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout()); 4211 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT, 4212 Src.getOperand(0)); 4213 DCI.AddToWorklist(Trunc.getNode()); 4214 4215 if (Amt.getValueType() != NewShiftVT) { 4216 Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT); 4217 DCI.AddToWorklist(Amt.getNode()); 4218 } 4219 4220 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT, 4221 Trunc, Amt); 4222 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift); 4223 } 4224 } 4225 } 4226 4227 return SDValue(); 4228 } 4229 4230 // We need to specifically handle i64 mul here to avoid unnecessary conversion 4231 // instructions. If we only match on the legalized i64 mul expansion, 4232 // SimplifyDemandedBits will be unable to remove them because there will be 4233 // multiple uses due to the separate mul + mulh[su]. 4234 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, 4235 SDValue N0, SDValue N1, unsigned Size, bool Signed) { 4236 if (Size <= 32) { 4237 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24; 4238 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); 4239 } 4240 4241 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24; 4242 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24; 4243 4244 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); 4245 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); 4246 4247 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi); 4248 } 4249 4250 /// If \p V is an add of a constant 1, returns the other operand. Otherwise 4251 /// return SDValue(). 4252 static SDValue getAddOneOp(const SDNode *V) { 4253 if (V->getOpcode() != ISD::ADD) 4254 return SDValue(); 4255 4256 return isOneConstant(V->getOperand(1)) ? V->getOperand(0) : SDValue(); 4257 } 4258 4259 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N, 4260 DAGCombinerInfo &DCI) const { 4261 assert(N->getOpcode() == ISD::MUL); 4262 EVT VT = N->getValueType(0); 4263 4264 // Don't generate 24-bit multiplies on values that are in SGPRs, since 4265 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs 4266 // unnecessarily). isDivergent() is used as an approximation of whether the 4267 // value is in an SGPR. 4268 if (!N->isDivergent()) 4269 return SDValue(); 4270 4271 unsigned Size = VT.getSizeInBits(); 4272 if (VT.isVector() || Size > 64) 4273 return SDValue(); 4274 4275 SelectionDAG &DAG = DCI.DAG; 4276 SDLoc DL(N); 4277 4278 SDValue N0 = N->getOperand(0); 4279 SDValue N1 = N->getOperand(1); 4280 4281 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad 4282 // matching. 4283 4284 // mul x, (add y, 1) -> add (mul x, y), x 4285 auto IsFoldableAdd = [](SDValue V) -> SDValue { 4286 SDValue AddOp = getAddOneOp(V.getNode()); 4287 if (!AddOp) 4288 return SDValue(); 4289 4290 if (V.hasOneUse() || all_of(V->uses(), [](const SDNode *U) -> bool { 4291 return U->getOpcode() == ISD::MUL; 4292 })) 4293 return AddOp; 4294 4295 return SDValue(); 4296 }; 4297 4298 // FIXME: The selection pattern is not properly checking for commuted 4299 // operands, so we have to place the mul in the LHS 4300 if (SDValue MulOper = IsFoldableAdd(N0)) { 4301 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper); 4302 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1); 4303 } 4304 4305 if (SDValue MulOper = IsFoldableAdd(N1)) { 4306 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper); 4307 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0); 4308 } 4309 4310 // There are i16 integer mul/mad. 4311 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16)) 4312 return SDValue(); 4313 4314 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends 4315 // in the source into any_extends if the result of the mul is truncated. Since 4316 // we can assume the high bits are whatever we want, use the underlying value 4317 // to avoid the unknown high bits from interfering. 4318 if (N0.getOpcode() == ISD::ANY_EXTEND) 4319 N0 = N0.getOperand(0); 4320 4321 if (N1.getOpcode() == ISD::ANY_EXTEND) 4322 N1 = N1.getOperand(0); 4323 4324 SDValue Mul; 4325 4326 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { 4327 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); 4328 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); 4329 Mul = getMul24(DAG, DL, N0, N1, Size, false); 4330 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { 4331 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); 4332 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); 4333 Mul = getMul24(DAG, DL, N0, N1, Size, true); 4334 } else { 4335 return SDValue(); 4336 } 4337 4338 // We need to use sext even for MUL_U24, because MUL_U24 is used 4339 // for signed multiply of 8 and 16-bit types. 4340 return DAG.getSExtOrTrunc(Mul, DL, VT); 4341 } 4342 4343 SDValue 4344 AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N, 4345 DAGCombinerInfo &DCI) const { 4346 if (N->getValueType(0) != MVT::i32) 4347 return SDValue(); 4348 4349 SelectionDAG &DAG = DCI.DAG; 4350 SDLoc DL(N); 4351 4352 bool Signed = N->getOpcode() == ISD::SMUL_LOHI; 4353 SDValue N0 = N->getOperand(0); 4354 SDValue N1 = N->getOperand(1); 4355 4356 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends 4357 // in the source into any_extends if the result of the mul is truncated. Since 4358 // we can assume the high bits are whatever we want, use the underlying value 4359 // to avoid the unknown high bits from interfering. 4360 if (N0.getOpcode() == ISD::ANY_EXTEND) 4361 N0 = N0.getOperand(0); 4362 if (N1.getOpcode() == ISD::ANY_EXTEND) 4363 N1 = N1.getOperand(0); 4364 4365 // Try to use two fast 24-bit multiplies (one for each half of the result) 4366 // instead of one slow extending multiply. 4367 unsigned LoOpcode = 0; 4368 unsigned HiOpcode = 0; 4369 if (Signed) { 4370 if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { 4371 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); 4372 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); 4373 LoOpcode = AMDGPUISD::MUL_I24; 4374 HiOpcode = AMDGPUISD::MULHI_I24; 4375 } 4376 } else { 4377 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { 4378 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); 4379 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); 4380 LoOpcode = AMDGPUISD::MUL_U24; 4381 HiOpcode = AMDGPUISD::MULHI_U24; 4382 } 4383 } 4384 if (!LoOpcode) 4385 return SDValue(); 4386 4387 SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1); 4388 SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1); 4389 DCI.CombineTo(N, Lo, Hi); 4390 return SDValue(N, 0); 4391 } 4392 4393 SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N, 4394 DAGCombinerInfo &DCI) const { 4395 EVT VT = N->getValueType(0); 4396 4397 if (!Subtarget->hasMulI24() || VT.isVector()) 4398 return SDValue(); 4399 4400 // Don't generate 24-bit multiplies on values that are in SGPRs, since 4401 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs 4402 // unnecessarily). isDivergent() is used as an approximation of whether the 4403 // value is in an SGPR. 4404 // This doesn't apply if no s_mul_hi is available (since we'll end up with a 4405 // valu op anyway) 4406 if (Subtarget->hasSMulHi() && !N->isDivergent()) 4407 return SDValue(); 4408 4409 SelectionDAG &DAG = DCI.DAG; 4410 SDLoc DL(N); 4411 4412 SDValue N0 = N->getOperand(0); 4413 SDValue N1 = N->getOperand(1); 4414 4415 if (!isI24(N0, DAG) || !isI24(N1, DAG)) 4416 return SDValue(); 4417 4418 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); 4419 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); 4420 4421 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1); 4422 DCI.AddToWorklist(Mulhi.getNode()); 4423 return DAG.getSExtOrTrunc(Mulhi, DL, VT); 4424 } 4425 4426 SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N, 4427 DAGCombinerInfo &DCI) const { 4428 EVT VT = N->getValueType(0); 4429 4430 if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32) 4431 return SDValue(); 4432 4433 // Don't generate 24-bit multiplies on values that are in SGPRs, since 4434 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs 4435 // unnecessarily). isDivergent() is used as an approximation of whether the 4436 // value is in an SGPR. 4437 // This doesn't apply if no s_mul_hi is available (since we'll end up with a 4438 // valu op anyway) 4439 if (Subtarget->hasSMulHi() && !N->isDivergent()) 4440 return SDValue(); 4441 4442 SelectionDAG &DAG = DCI.DAG; 4443 SDLoc DL(N); 4444 4445 SDValue N0 = N->getOperand(0); 4446 SDValue N1 = N->getOperand(1); 4447 4448 if (!isU24(N0, DAG) || !isU24(N1, DAG)) 4449 return SDValue(); 4450 4451 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); 4452 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); 4453 4454 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1); 4455 DCI.AddToWorklist(Mulhi.getNode()); 4456 return DAG.getZExtOrTrunc(Mulhi, DL, VT); 4457 } 4458 4459 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG, 4460 SDValue Op, 4461 const SDLoc &DL, 4462 unsigned Opc) const { 4463 EVT VT = Op.getValueType(); 4464 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT); 4465 if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() && 4466 LegalVT != MVT::i16)) 4467 return SDValue(); 4468 4469 if (VT != MVT::i32) 4470 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); 4471 4472 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op); 4473 if (VT != MVT::i32) 4474 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX); 4475 4476 return FFBX; 4477 } 4478 4479 // The native instructions return -1 on 0 input. Optimize out a select that 4480 // produces -1 on 0. 4481 // 4482 // TODO: If zero is not undef, we could also do this if the output is compared 4483 // against the bitwidth. 4484 // 4485 // TODO: Should probably combine against FFBH_U32 instead of ctlz directly. 4486 SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, 4487 SDValue LHS, SDValue RHS, 4488 DAGCombinerInfo &DCI) const { 4489 if (!isNullConstant(Cond.getOperand(1))) 4490 return SDValue(); 4491 4492 SelectionDAG &DAG = DCI.DAG; 4493 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 4494 SDValue CmpLHS = Cond.getOperand(0); 4495 4496 // select (setcc x, 0, eq), -1, (ctlz_zero_undef x) -> ffbh_u32 x 4497 // select (setcc x, 0, eq), -1, (cttz_zero_undef x) -> ffbl_u32 x 4498 if (CCOpcode == ISD::SETEQ && 4499 (isCtlzOpc(RHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) && 4500 RHS.getOperand(0) == CmpLHS && isAllOnesConstant(LHS)) { 4501 unsigned Opc = 4502 isCttzOpc(RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32; 4503 return getFFBX_U32(DAG, CmpLHS, SL, Opc); 4504 } 4505 4506 // select (setcc x, 0, ne), (ctlz_zero_undef x), -1 -> ffbh_u32 x 4507 // select (setcc x, 0, ne), (cttz_zero_undef x), -1 -> ffbl_u32 x 4508 if (CCOpcode == ISD::SETNE && 4509 (isCtlzOpc(LHS.getOpcode()) || isCttzOpc(LHS.getOpcode())) && 4510 LHS.getOperand(0) == CmpLHS && isAllOnesConstant(RHS)) { 4511 unsigned Opc = 4512 isCttzOpc(LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32; 4513 4514 return getFFBX_U32(DAG, CmpLHS, SL, Opc); 4515 } 4516 4517 return SDValue(); 4518 } 4519 4520 static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI, 4521 unsigned Op, 4522 const SDLoc &SL, 4523 SDValue Cond, 4524 SDValue N1, 4525 SDValue N2) { 4526 SelectionDAG &DAG = DCI.DAG; 4527 EVT VT = N1.getValueType(); 4528 4529 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, 4530 N1.getOperand(0), N2.getOperand(0)); 4531 DCI.AddToWorklist(NewSelect.getNode()); 4532 return DAG.getNode(Op, SL, VT, NewSelect); 4533 } 4534 4535 // Pull a free FP operation out of a select so it may fold into uses. 4536 // 4537 // select c, (fneg x), (fneg y) -> fneg (select c, x, y) 4538 // select c, (fneg x), k -> fneg (select c, x, (fneg k)) 4539 // 4540 // select c, (fabs x), (fabs y) -> fabs (select c, x, y) 4541 // select c, (fabs x), +k -> fabs (select c, x, k) 4542 SDValue 4543 AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI, 4544 SDValue N) const { 4545 SelectionDAG &DAG = DCI.DAG; 4546 SDValue Cond = N.getOperand(0); 4547 SDValue LHS = N.getOperand(1); 4548 SDValue RHS = N.getOperand(2); 4549 4550 EVT VT = N.getValueType(); 4551 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) || 4552 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) { 4553 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N.getNode())) 4554 return SDValue(); 4555 4556 return distributeOpThroughSelect(DCI, LHS.getOpcode(), 4557 SDLoc(N), Cond, LHS, RHS); 4558 } 4559 4560 bool Inv = false; 4561 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) { 4562 std::swap(LHS, RHS); 4563 Inv = true; 4564 } 4565 4566 // TODO: Support vector constants. 4567 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS); 4568 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS && 4569 !selectSupportsSourceMods(N.getNode())) { 4570 SDLoc SL(N); 4571 // If one side is an fneg/fabs and the other is a constant, we can push the 4572 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative. 4573 SDValue NewLHS = LHS.getOperand(0); 4574 SDValue NewRHS = RHS; 4575 4576 // Careful: if the neg can be folded up, don't try to pull it back down. 4577 bool ShouldFoldNeg = true; 4578 4579 if (NewLHS.hasOneUse()) { 4580 unsigned Opc = NewLHS.getOpcode(); 4581 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(NewLHS.getNode())) 4582 ShouldFoldNeg = false; 4583 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL) 4584 ShouldFoldNeg = false; 4585 } 4586 4587 if (ShouldFoldNeg) { 4588 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative()) 4589 return SDValue(); 4590 4591 // We're going to be forced to use a source modifier anyway, there's no 4592 // point to pulling the negate out unless we can get a size reduction by 4593 // negating the constant. 4594 // 4595 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know 4596 // about cheaper constants. 4597 if (NewLHS.getOpcode() == ISD::FABS && 4598 getConstantNegateCost(CRHS) != NegatibleCost::Cheaper) 4599 return SDValue(); 4600 4601 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N.getNode())) 4602 return SDValue(); 4603 4604 if (LHS.getOpcode() == ISD::FNEG) 4605 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); 4606 4607 if (Inv) 4608 std::swap(NewLHS, NewRHS); 4609 4610 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, 4611 Cond, NewLHS, NewRHS); 4612 DCI.AddToWorklist(NewSelect.getNode()); 4613 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect); 4614 } 4615 } 4616 4617 return SDValue(); 4618 } 4619 4620 SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N, 4621 DAGCombinerInfo &DCI) const { 4622 if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0))) 4623 return Folded; 4624 4625 SDValue Cond = N->getOperand(0); 4626 if (Cond.getOpcode() != ISD::SETCC) 4627 return SDValue(); 4628 4629 EVT VT = N->getValueType(0); 4630 SDValue LHS = Cond.getOperand(0); 4631 SDValue RHS = Cond.getOperand(1); 4632 SDValue CC = Cond.getOperand(2); 4633 4634 SDValue True = N->getOperand(1); 4635 SDValue False = N->getOperand(2); 4636 4637 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses. 4638 SelectionDAG &DAG = DCI.DAG; 4639 if (DAG.isConstantValueOfAnyType(True) && 4640 !DAG.isConstantValueOfAnyType(False)) { 4641 // Swap cmp + select pair to move constant to false input. 4642 // This will allow using VOPC cndmasks more often. 4643 // select (setcc x, y), k, x -> select (setccinv x, y), x, k 4644 4645 SDLoc SL(N); 4646 ISD::CondCode NewCC = 4647 getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), LHS.getValueType()); 4648 4649 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC); 4650 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); 4651 } 4652 4653 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) { 4654 SDValue MinMax 4655 = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI); 4656 // Revisit this node so we can catch min3/max3/med3 patterns. 4657 //DCI.AddToWorklist(MinMax.getNode()); 4658 return MinMax; 4659 } 4660 } 4661 4662 // There's no reason to not do this if the condition has other uses. 4663 return performCtlz_CttzCombine(SDLoc(N), Cond, True, False, DCI); 4664 } 4665 4666 static bool isInv2Pi(const APFloat &APF) { 4667 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118)); 4668 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983)); 4669 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882)); 4670 4671 return APF.bitwiseIsEqual(KF16) || 4672 APF.bitwiseIsEqual(KF32) || 4673 APF.bitwiseIsEqual(KF64); 4674 } 4675 4676 // 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an 4677 // additional cost to negate them. 4678 TargetLowering::NegatibleCost 4679 AMDGPUTargetLowering::getConstantNegateCost(const ConstantFPSDNode *C) const { 4680 if (C->isZero()) 4681 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive; 4682 4683 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(C->getValueAPF())) 4684 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive; 4685 4686 return NegatibleCost::Neutral; 4687 } 4688 4689 bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const { 4690 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N)) 4691 return getConstantNegateCost(C) == NegatibleCost::Expensive; 4692 return false; 4693 } 4694 4695 bool AMDGPUTargetLowering::isConstantCheaperToNegate(SDValue N) const { 4696 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N)) 4697 return getConstantNegateCost(C) == NegatibleCost::Cheaper; 4698 return false; 4699 } 4700 4701 static unsigned inverseMinMax(unsigned Opc) { 4702 switch (Opc) { 4703 case ISD::FMAXNUM: 4704 return ISD::FMINNUM; 4705 case ISD::FMINNUM: 4706 return ISD::FMAXNUM; 4707 case ISD::FMAXNUM_IEEE: 4708 return ISD::FMINNUM_IEEE; 4709 case ISD::FMINNUM_IEEE: 4710 return ISD::FMAXNUM_IEEE; 4711 case ISD::FMAXIMUM: 4712 return ISD::FMINIMUM; 4713 case ISD::FMINIMUM: 4714 return ISD::FMAXIMUM; 4715 case AMDGPUISD::FMAX_LEGACY: 4716 return AMDGPUISD::FMIN_LEGACY; 4717 case AMDGPUISD::FMIN_LEGACY: 4718 return AMDGPUISD::FMAX_LEGACY; 4719 default: 4720 llvm_unreachable("invalid min/max opcode"); 4721 } 4722 } 4723 4724 /// \return true if it's profitable to try to push an fneg into its source 4725 /// instruction. 4726 bool AMDGPUTargetLowering::shouldFoldFNegIntoSrc(SDNode *N, SDValue N0) { 4727 // If the input has multiple uses and we can either fold the negate down, or 4728 // the other uses cannot, give up. This both prevents unprofitable 4729 // transformations and infinite loops: we won't repeatedly try to fold around 4730 // a negate that has no 'good' form. 4731 if (N0.hasOneUse()) { 4732 // This may be able to fold into the source, but at a code size cost. Don't 4733 // fold if the fold into the user is free. 4734 if (allUsesHaveSourceMods(N, 0)) 4735 return false; 4736 } else { 4737 if (fnegFoldsIntoOp(N0.getNode()) && 4738 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N0.getNode()))) 4739 return false; 4740 } 4741 4742 return true; 4743 } 4744 4745 SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N, 4746 DAGCombinerInfo &DCI) const { 4747 SelectionDAG &DAG = DCI.DAG; 4748 SDValue N0 = N->getOperand(0); 4749 EVT VT = N->getValueType(0); 4750 4751 unsigned Opc = N0.getOpcode(); 4752 4753 if (!shouldFoldFNegIntoSrc(N, N0)) 4754 return SDValue(); 4755 4756 SDLoc SL(N); 4757 switch (Opc) { 4758 case ISD::FADD: { 4759 if (!mayIgnoreSignedZero(N0)) 4760 return SDValue(); 4761 4762 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y)) 4763 SDValue LHS = N0.getOperand(0); 4764 SDValue RHS = N0.getOperand(1); 4765 4766 if (LHS.getOpcode() != ISD::FNEG) 4767 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); 4768 else 4769 LHS = LHS.getOperand(0); 4770 4771 if (RHS.getOpcode() != ISD::FNEG) 4772 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); 4773 else 4774 RHS = RHS.getOperand(0); 4775 4776 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags()); 4777 if (Res.getOpcode() != ISD::FADD) 4778 return SDValue(); // Op got folded away. 4779 if (!N0.hasOneUse()) 4780 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); 4781 return Res; 4782 } 4783 case ISD::FMUL: 4784 case AMDGPUISD::FMUL_LEGACY: { 4785 // (fneg (fmul x, y)) -> (fmul x, (fneg y)) 4786 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y)) 4787 SDValue LHS = N0.getOperand(0); 4788 SDValue RHS = N0.getOperand(1); 4789 4790 if (LHS.getOpcode() == ISD::FNEG) 4791 LHS = LHS.getOperand(0); 4792 else if (RHS.getOpcode() == ISD::FNEG) 4793 RHS = RHS.getOperand(0); 4794 else 4795 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); 4796 4797 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags()); 4798 if (Res.getOpcode() != Opc) 4799 return SDValue(); // Op got folded away. 4800 if (!N0.hasOneUse()) 4801 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); 4802 return Res; 4803 } 4804 case ISD::FMA: 4805 case ISD::FMAD: { 4806 // TODO: handle llvm.amdgcn.fma.legacy 4807 if (!mayIgnoreSignedZero(N0)) 4808 return SDValue(); 4809 4810 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z)) 4811 SDValue LHS = N0.getOperand(0); 4812 SDValue MHS = N0.getOperand(1); 4813 SDValue RHS = N0.getOperand(2); 4814 4815 if (LHS.getOpcode() == ISD::FNEG) 4816 LHS = LHS.getOperand(0); 4817 else if (MHS.getOpcode() == ISD::FNEG) 4818 MHS = MHS.getOperand(0); 4819 else 4820 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS); 4821 4822 if (RHS.getOpcode() != ISD::FNEG) 4823 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); 4824 else 4825 RHS = RHS.getOperand(0); 4826 4827 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS); 4828 if (Res.getOpcode() != Opc) 4829 return SDValue(); // Op got folded away. 4830 if (!N0.hasOneUse()) 4831 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); 4832 return Res; 4833 } 4834 case ISD::FMAXNUM: 4835 case ISD::FMINNUM: 4836 case ISD::FMAXNUM_IEEE: 4837 case ISD::FMINNUM_IEEE: 4838 case ISD::FMINIMUM: 4839 case ISD::FMAXIMUM: 4840 case AMDGPUISD::FMAX_LEGACY: 4841 case AMDGPUISD::FMIN_LEGACY: { 4842 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y) 4843 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y) 4844 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y) 4845 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y) 4846 4847 SDValue LHS = N0.getOperand(0); 4848 SDValue RHS = N0.getOperand(1); 4849 4850 // 0 doesn't have a negated inline immediate. 4851 // TODO: This constant check should be generalized to other operations. 4852 if (isConstantCostlierToNegate(RHS)) 4853 return SDValue(); 4854 4855 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); 4856 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); 4857 unsigned Opposite = inverseMinMax(Opc); 4858 4859 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags()); 4860 if (Res.getOpcode() != Opposite) 4861 return SDValue(); // Op got folded away. 4862 if (!N0.hasOneUse()) 4863 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); 4864 return Res; 4865 } 4866 case AMDGPUISD::FMED3: { 4867 SDValue Ops[3]; 4868 for (unsigned I = 0; I < 3; ++I) 4869 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags()); 4870 4871 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags()); 4872 if (Res.getOpcode() != AMDGPUISD::FMED3) 4873 return SDValue(); // Op got folded away. 4874 4875 if (!N0.hasOneUse()) { 4876 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res); 4877 DAG.ReplaceAllUsesWith(N0, Neg); 4878 4879 for (SDNode *U : Neg->uses()) 4880 DCI.AddToWorklist(U); 4881 } 4882 4883 return Res; 4884 } 4885 case ISD::FP_EXTEND: 4886 case ISD::FTRUNC: 4887 case ISD::FRINT: 4888 case ISD::FNEARBYINT: // XXX - Should fround be handled? 4889 case ISD::FROUNDEVEN: 4890 case ISD::FSIN: 4891 case ISD::FCANONICALIZE: 4892 case AMDGPUISD::RCP: 4893 case AMDGPUISD::RCP_LEGACY: 4894 case AMDGPUISD::RCP_IFLAG: 4895 case AMDGPUISD::SIN_HW: { 4896 SDValue CvtSrc = N0.getOperand(0); 4897 if (CvtSrc.getOpcode() == ISD::FNEG) { 4898 // (fneg (fp_extend (fneg x))) -> (fp_extend x) 4899 // (fneg (rcp (fneg x))) -> (rcp x) 4900 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0)); 4901 } 4902 4903 if (!N0.hasOneUse()) 4904 return SDValue(); 4905 4906 // (fneg (fp_extend x)) -> (fp_extend (fneg x)) 4907 // (fneg (rcp x)) -> (rcp (fneg x)) 4908 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); 4909 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags()); 4910 } 4911 case ISD::FP_ROUND: { 4912 SDValue CvtSrc = N0.getOperand(0); 4913 4914 if (CvtSrc.getOpcode() == ISD::FNEG) { 4915 // (fneg (fp_round (fneg x))) -> (fp_round x) 4916 return DAG.getNode(ISD::FP_ROUND, SL, VT, 4917 CvtSrc.getOperand(0), N0.getOperand(1)); 4918 } 4919 4920 if (!N0.hasOneUse()) 4921 return SDValue(); 4922 4923 // (fneg (fp_round x)) -> (fp_round (fneg x)) 4924 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); 4925 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1)); 4926 } 4927 case ISD::FP16_TO_FP: { 4928 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal 4929 // f16, but legalization of f16 fneg ends up pulling it out of the source. 4930 // Put the fneg back as a legal source operation that can be matched later. 4931 SDLoc SL(N); 4932 4933 SDValue Src = N0.getOperand(0); 4934 EVT SrcVT = Src.getValueType(); 4935 4936 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000) 4937 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src, 4938 DAG.getConstant(0x8000, SL, SrcVT)); 4939 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg); 4940 } 4941 case ISD::SELECT: { 4942 // fneg (select c, a, b) -> select c, (fneg a), (fneg b) 4943 // TODO: Invert conditions of foldFreeOpFromSelect 4944 return SDValue(); 4945 } 4946 case ISD::BITCAST: { 4947 SDLoc SL(N); 4948 SDValue BCSrc = N0.getOperand(0); 4949 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) { 4950 SDValue HighBits = BCSrc.getOperand(BCSrc.getNumOperands() - 1); 4951 if (HighBits.getValueType().getSizeInBits() != 32 || 4952 !fnegFoldsIntoOp(HighBits.getNode())) 4953 return SDValue(); 4954 4955 // f64 fneg only really needs to operate on the high half of of the 4956 // register, so try to force it to an f32 operation to help make use of 4957 // source modifiers. 4958 // 4959 // 4960 // fneg (f64 (bitcast (build_vector x, y))) -> 4961 // f64 (bitcast (build_vector (bitcast i32:x to f32), 4962 // (fneg (bitcast i32:y to f32))) 4963 4964 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::f32, HighBits); 4965 SDValue NegHi = DAG.getNode(ISD::FNEG, SL, MVT::f32, CastHi); 4966 SDValue CastBack = 4967 DAG.getNode(ISD::BITCAST, SL, HighBits.getValueType(), NegHi); 4968 4969 SmallVector<SDValue, 8> Ops(BCSrc->op_begin(), BCSrc->op_end()); 4970 Ops.back() = CastBack; 4971 DCI.AddToWorklist(NegHi.getNode()); 4972 SDValue Build = 4973 DAG.getNode(ISD::BUILD_VECTOR, SL, BCSrc.getValueType(), Ops); 4974 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, Build); 4975 4976 if (!N0.hasOneUse()) 4977 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Result)); 4978 return Result; 4979 } 4980 4981 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 && 4982 BCSrc.hasOneUse()) { 4983 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) -> 4984 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32) 4985 4986 // TODO: Cast back result for multiple uses is beneficial in some cases. 4987 4988 SDValue LHS = 4989 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(1)); 4990 SDValue RHS = 4991 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(2)); 4992 4993 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, LHS); 4994 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHS); 4995 4996 return DAG.getNode(ISD::SELECT, SL, MVT::f32, BCSrc.getOperand(0), NegLHS, 4997 NegRHS); 4998 } 4999 5000 return SDValue(); 5001 } 5002 default: 5003 return SDValue(); 5004 } 5005 } 5006 5007 SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N, 5008 DAGCombinerInfo &DCI) const { 5009 SelectionDAG &DAG = DCI.DAG; 5010 SDValue N0 = N->getOperand(0); 5011 5012 if (!N0.hasOneUse()) 5013 return SDValue(); 5014 5015 switch (N0.getOpcode()) { 5016 case ISD::FP16_TO_FP: { 5017 assert(!Subtarget->has16BitInsts() && "should only see if f16 is illegal"); 5018 SDLoc SL(N); 5019 SDValue Src = N0.getOperand(0); 5020 EVT SrcVT = Src.getValueType(); 5021 5022 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff) 5023 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src, 5024 DAG.getConstant(0x7fff, SL, SrcVT)); 5025 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs); 5026 } 5027 default: 5028 return SDValue(); 5029 } 5030 } 5031 5032 SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N, 5033 DAGCombinerInfo &DCI) const { 5034 const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0)); 5035 if (!CFP) 5036 return SDValue(); 5037 5038 // XXX - Should this flush denormals? 5039 const APFloat &Val = CFP->getValueAPF(); 5040 APFloat One(Val.getSemantics(), "1.0"); 5041 return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0)); 5042 } 5043 5044 SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, 5045 DAGCombinerInfo &DCI) const { 5046 SelectionDAG &DAG = DCI.DAG; 5047 SDLoc DL(N); 5048 5049 switch(N->getOpcode()) { 5050 default: 5051 break; 5052 case ISD::BITCAST: { 5053 EVT DestVT = N->getValueType(0); 5054 5055 // Push casts through vector builds. This helps avoid emitting a large 5056 // number of copies when materializing floating point vector constants. 5057 // 5058 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) => 5059 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y)) 5060 if (DestVT.isVector()) { 5061 SDValue Src = N->getOperand(0); 5062 if (Src.getOpcode() == ISD::BUILD_VECTOR && 5063 (DCI.getDAGCombineLevel() < AfterLegalizeDAG || 5064 isOperationLegal(ISD::BUILD_VECTOR, DestVT))) { 5065 EVT SrcVT = Src.getValueType(); 5066 unsigned NElts = DestVT.getVectorNumElements(); 5067 5068 if (SrcVT.getVectorNumElements() == NElts) { 5069 EVT DestEltVT = DestVT.getVectorElementType(); 5070 5071 SmallVector<SDValue, 8> CastedElts; 5072 SDLoc SL(N); 5073 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) { 5074 SDValue Elt = Src.getOperand(I); 5075 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt)); 5076 } 5077 5078 return DAG.getBuildVector(DestVT, SL, CastedElts); 5079 } 5080 } 5081 } 5082 5083 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector()) 5084 break; 5085 5086 // Fold bitcasts of constants. 5087 // 5088 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k) 5089 // TODO: Generalize and move to DAGCombiner 5090 SDValue Src = N->getOperand(0); 5091 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src)) { 5092 SDLoc SL(N); 5093 uint64_t CVal = C->getZExtValue(); 5094 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, 5095 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), 5096 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); 5097 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV); 5098 } 5099 5100 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Src)) { 5101 const APInt &Val = C->getValueAPF().bitcastToAPInt(); 5102 SDLoc SL(N); 5103 uint64_t CVal = Val.getZExtValue(); 5104 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, 5105 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), 5106 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); 5107 5108 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec); 5109 } 5110 5111 break; 5112 } 5113 case ISD::SHL: { 5114 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG) 5115 break; 5116 5117 return performShlCombine(N, DCI); 5118 } 5119 case ISD::SRL: { 5120 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG) 5121 break; 5122 5123 return performSrlCombine(N, DCI); 5124 } 5125 case ISD::SRA: { 5126 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG) 5127 break; 5128 5129 return performSraCombine(N, DCI); 5130 } 5131 case ISD::TRUNCATE: 5132 return performTruncateCombine(N, DCI); 5133 case ISD::MUL: 5134 return performMulCombine(N, DCI); 5135 case AMDGPUISD::MUL_U24: 5136 case AMDGPUISD::MUL_I24: { 5137 if (SDValue Simplified = simplifyMul24(N, DCI)) 5138 return Simplified; 5139 break; 5140 } 5141 case AMDGPUISD::MULHI_I24: 5142 case AMDGPUISD::MULHI_U24: 5143 return simplifyMul24(N, DCI); 5144 case ISD::SMUL_LOHI: 5145 case ISD::UMUL_LOHI: 5146 return performMulLoHiCombine(N, DCI); 5147 case ISD::MULHS: 5148 return performMulhsCombine(N, DCI); 5149 case ISD::MULHU: 5150 return performMulhuCombine(N, DCI); 5151 case ISD::SELECT: 5152 return performSelectCombine(N, DCI); 5153 case ISD::FNEG: 5154 return performFNegCombine(N, DCI); 5155 case ISD::FABS: 5156 return performFAbsCombine(N, DCI); 5157 case AMDGPUISD::BFE_I32: 5158 case AMDGPUISD::BFE_U32: { 5159 assert(!N->getValueType(0).isVector() && 5160 "Vector handling of BFE not implemented"); 5161 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 5162 if (!Width) 5163 break; 5164 5165 uint32_t WidthVal = Width->getZExtValue() & 0x1f; 5166 if (WidthVal == 0) 5167 return DAG.getConstant(0, DL, MVT::i32); 5168 5169 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 5170 if (!Offset) 5171 break; 5172 5173 SDValue BitsFrom = N->getOperand(0); 5174 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f; 5175 5176 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32; 5177 5178 if (OffsetVal == 0) { 5179 // This is already sign / zero extended, so try to fold away extra BFEs. 5180 unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal); 5181 5182 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom); 5183 if (OpSignBits >= SignBits) 5184 return BitsFrom; 5185 5186 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal); 5187 if (Signed) { 5188 // This is a sign_extend_inreg. Replace it to take advantage of existing 5189 // DAG Combines. If not eliminated, we will match back to BFE during 5190 // selection. 5191 5192 // TODO: The sext_inreg of extended types ends, although we can could 5193 // handle them in a single BFE. 5194 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, 5195 DAG.getValueType(SmallVT)); 5196 } 5197 5198 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT); 5199 } 5200 5201 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) { 5202 if (Signed) { 5203 return constantFoldBFE<int32_t>(DAG, 5204 CVal->getSExtValue(), 5205 OffsetVal, 5206 WidthVal, 5207 DL); 5208 } 5209 5210 return constantFoldBFE<uint32_t>(DAG, 5211 CVal->getZExtValue(), 5212 OffsetVal, 5213 WidthVal, 5214 DL); 5215 } 5216 5217 if ((OffsetVal + WidthVal) >= 32 && 5218 !(Subtarget->hasSDWA() && OffsetVal == 16 && WidthVal == 16)) { 5219 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32); 5220 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, 5221 BitsFrom, ShiftVal); 5222 } 5223 5224 if (BitsFrom.hasOneUse()) { 5225 APInt Demanded = APInt::getBitsSet(32, 5226 OffsetVal, 5227 OffsetVal + WidthVal); 5228 5229 KnownBits Known; 5230 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), 5231 !DCI.isBeforeLegalizeOps()); 5232 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 5233 if (TLI.ShrinkDemandedConstant(BitsFrom, Demanded, TLO) || 5234 TLI.SimplifyDemandedBits(BitsFrom, Demanded, Known, TLO)) { 5235 DCI.CommitTargetLoweringOpt(TLO); 5236 } 5237 } 5238 5239 break; 5240 } 5241 case ISD::LOAD: 5242 return performLoadCombine(N, DCI); 5243 case ISD::STORE: 5244 return performStoreCombine(N, DCI); 5245 case AMDGPUISD::RCP: 5246 case AMDGPUISD::RCP_IFLAG: 5247 return performRcpCombine(N, DCI); 5248 case ISD::AssertZext: 5249 case ISD::AssertSext: 5250 return performAssertSZExtCombine(N, DCI); 5251 case ISD::INTRINSIC_WO_CHAIN: 5252 return performIntrinsicWOChainCombine(N, DCI); 5253 case AMDGPUISD::FMAD_FTZ: { 5254 SDValue N0 = N->getOperand(0); 5255 SDValue N1 = N->getOperand(1); 5256 SDValue N2 = N->getOperand(2); 5257 EVT VT = N->getValueType(0); 5258 5259 // FMAD_FTZ is a FMAD + flush denormals to zero. 5260 // We flush the inputs, the intermediate step, and the output. 5261 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 5262 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 5263 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 5264 if (N0CFP && N1CFP && N2CFP) { 5265 const auto FTZ = [](const APFloat &V) { 5266 if (V.isDenormal()) { 5267 APFloat Zero(V.getSemantics(), 0); 5268 return V.isNegative() ? -Zero : Zero; 5269 } 5270 return V; 5271 }; 5272 5273 APFloat V0 = FTZ(N0CFP->getValueAPF()); 5274 APFloat V1 = FTZ(N1CFP->getValueAPF()); 5275 APFloat V2 = FTZ(N2CFP->getValueAPF()); 5276 V0.multiply(V1, APFloat::rmNearestTiesToEven); 5277 V0 = FTZ(V0); 5278 V0.add(V2, APFloat::rmNearestTiesToEven); 5279 return DAG.getConstantFP(FTZ(V0), DL, VT); 5280 } 5281 break; 5282 } 5283 } 5284 return SDValue(); 5285 } 5286 5287 //===----------------------------------------------------------------------===// 5288 // Helper functions 5289 //===----------------------------------------------------------------------===// 5290 5291 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, 5292 const TargetRegisterClass *RC, 5293 Register Reg, EVT VT, 5294 const SDLoc &SL, 5295 bool RawReg) const { 5296 MachineFunction &MF = DAG.getMachineFunction(); 5297 MachineRegisterInfo &MRI = MF.getRegInfo(); 5298 Register VReg; 5299 5300 if (!MRI.isLiveIn(Reg)) { 5301 VReg = MRI.createVirtualRegister(RC); 5302 MRI.addLiveIn(Reg, VReg); 5303 } else { 5304 VReg = MRI.getLiveInVirtReg(Reg); 5305 } 5306 5307 if (RawReg) 5308 return DAG.getRegister(VReg, VT); 5309 5310 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT); 5311 } 5312 5313 // This may be called multiple times, and nothing prevents creating multiple 5314 // objects at the same offset. See if we already defined this object. 5315 static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size, 5316 int64_t Offset) { 5317 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) { 5318 if (MFI.getObjectOffset(I) == Offset) { 5319 assert(MFI.getObjectSize(I) == Size); 5320 return I; 5321 } 5322 } 5323 5324 return MFI.CreateFixedObject(Size, Offset, true); 5325 } 5326 5327 SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG, 5328 EVT VT, 5329 const SDLoc &SL, 5330 int64_t Offset) const { 5331 MachineFunction &MF = DAG.getMachineFunction(); 5332 MachineFrameInfo &MFI = MF.getFrameInfo(); 5333 int FI = getOrCreateFixedStackObject(MFI, VT.getStoreSize(), Offset); 5334 5335 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset); 5336 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32); 5337 5338 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4), 5339 MachineMemOperand::MODereferenceable | 5340 MachineMemOperand::MOInvariant); 5341 } 5342 5343 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG, 5344 const SDLoc &SL, 5345 SDValue Chain, 5346 SDValue ArgVal, 5347 int64_t Offset) const { 5348 MachineFunction &MF = DAG.getMachineFunction(); 5349 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset); 5350 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 5351 5352 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32); 5353 // Stores to the argument stack area are relative to the stack pointer. 5354 SDValue SP = 5355 DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32); 5356 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr); 5357 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, Align(4), 5358 MachineMemOperand::MODereferenceable); 5359 return Store; 5360 } 5361 5362 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG, 5363 const TargetRegisterClass *RC, 5364 EVT VT, const SDLoc &SL, 5365 const ArgDescriptor &Arg) const { 5366 assert(Arg && "Attempting to load missing argument"); 5367 5368 SDValue V = Arg.isRegister() ? 5369 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) : 5370 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset()); 5371 5372 if (!Arg.isMasked()) 5373 return V; 5374 5375 unsigned Mask = Arg.getMask(); 5376 unsigned Shift = llvm::countr_zero<unsigned>(Mask); 5377 V = DAG.getNode(ISD::SRL, SL, VT, V, 5378 DAG.getShiftAmountConstant(Shift, VT, SL)); 5379 return DAG.getNode(ISD::AND, SL, VT, V, 5380 DAG.getConstant(Mask >> Shift, SL, VT)); 5381 } 5382 5383 uint32_t AMDGPUTargetLowering::getImplicitParameterOffset( 5384 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const { 5385 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset(); 5386 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr(); 5387 uint64_t ArgOffset = 5388 alignTo(ExplicitKernArgSize, Alignment) + ExplicitArgOffset; 5389 switch (Param) { 5390 case FIRST_IMPLICIT: 5391 return ArgOffset; 5392 case PRIVATE_BASE: 5393 return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET; 5394 case SHARED_BASE: 5395 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET; 5396 case QUEUE_PTR: 5397 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET; 5398 } 5399 llvm_unreachable("unexpected implicit parameter type"); 5400 } 5401 5402 uint32_t AMDGPUTargetLowering::getImplicitParameterOffset( 5403 const MachineFunction &MF, const ImplicitParameter Param) const { 5404 const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>(); 5405 return getImplicitParameterOffset(MFI->getExplicitKernArgSize(), Param); 5406 } 5407 5408 #define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node; 5409 5410 const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { 5411 switch ((AMDGPUISD::NodeType)Opcode) { 5412 case AMDGPUISD::FIRST_NUMBER: break; 5413 // AMDIL DAG nodes 5414 NODE_NAME_CASE(UMUL); 5415 NODE_NAME_CASE(BRANCH_COND); 5416 5417 // AMDGPU DAG nodes 5418 NODE_NAME_CASE(IF) 5419 NODE_NAME_CASE(ELSE) 5420 NODE_NAME_CASE(LOOP) 5421 NODE_NAME_CASE(CALL) 5422 NODE_NAME_CASE(TC_RETURN) 5423 NODE_NAME_CASE(TC_RETURN_GFX) 5424 NODE_NAME_CASE(TC_RETURN_CHAIN) 5425 NODE_NAME_CASE(TRAP) 5426 NODE_NAME_CASE(RET_GLUE) 5427 NODE_NAME_CASE(WAVE_ADDRESS) 5428 NODE_NAME_CASE(RETURN_TO_EPILOG) 5429 NODE_NAME_CASE(ENDPGM) 5430 NODE_NAME_CASE(ENDPGM_TRAP) 5431 NODE_NAME_CASE(SIMULATED_TRAP) 5432 NODE_NAME_CASE(DWORDADDR) 5433 NODE_NAME_CASE(FRACT) 5434 NODE_NAME_CASE(SETCC) 5435 NODE_NAME_CASE(SETREG) 5436 NODE_NAME_CASE(DENORM_MODE) 5437 NODE_NAME_CASE(FMA_W_CHAIN) 5438 NODE_NAME_CASE(FMUL_W_CHAIN) 5439 NODE_NAME_CASE(CLAMP) 5440 NODE_NAME_CASE(COS_HW) 5441 NODE_NAME_CASE(SIN_HW) 5442 NODE_NAME_CASE(FMAX_LEGACY) 5443 NODE_NAME_CASE(FMIN_LEGACY) 5444 NODE_NAME_CASE(FMAX3) 5445 NODE_NAME_CASE(SMAX3) 5446 NODE_NAME_CASE(UMAX3) 5447 NODE_NAME_CASE(FMIN3) 5448 NODE_NAME_CASE(SMIN3) 5449 NODE_NAME_CASE(UMIN3) 5450 NODE_NAME_CASE(FMED3) 5451 NODE_NAME_CASE(SMED3) 5452 NODE_NAME_CASE(UMED3) 5453 NODE_NAME_CASE(FMAXIMUM3) 5454 NODE_NAME_CASE(FMINIMUM3) 5455 NODE_NAME_CASE(FDOT2) 5456 NODE_NAME_CASE(URECIP) 5457 NODE_NAME_CASE(DIV_SCALE) 5458 NODE_NAME_CASE(DIV_FMAS) 5459 NODE_NAME_CASE(DIV_FIXUP) 5460 NODE_NAME_CASE(FMAD_FTZ) 5461 NODE_NAME_CASE(RCP) 5462 NODE_NAME_CASE(RSQ) 5463 NODE_NAME_CASE(RCP_LEGACY) 5464 NODE_NAME_CASE(RCP_IFLAG) 5465 NODE_NAME_CASE(LOG) 5466 NODE_NAME_CASE(EXP) 5467 NODE_NAME_CASE(FMUL_LEGACY) 5468 NODE_NAME_CASE(RSQ_CLAMP) 5469 NODE_NAME_CASE(FP_CLASS) 5470 NODE_NAME_CASE(DOT4) 5471 NODE_NAME_CASE(CARRY) 5472 NODE_NAME_CASE(BORROW) 5473 NODE_NAME_CASE(BFE_U32) 5474 NODE_NAME_CASE(BFE_I32) 5475 NODE_NAME_CASE(BFI) 5476 NODE_NAME_CASE(BFM) 5477 NODE_NAME_CASE(FFBH_U32) 5478 NODE_NAME_CASE(FFBH_I32) 5479 NODE_NAME_CASE(FFBL_B32) 5480 NODE_NAME_CASE(MUL_U24) 5481 NODE_NAME_CASE(MUL_I24) 5482 NODE_NAME_CASE(MULHI_U24) 5483 NODE_NAME_CASE(MULHI_I24) 5484 NODE_NAME_CASE(MAD_U24) 5485 NODE_NAME_CASE(MAD_I24) 5486 NODE_NAME_CASE(MAD_I64_I32) 5487 NODE_NAME_CASE(MAD_U64_U32) 5488 NODE_NAME_CASE(PERM) 5489 NODE_NAME_CASE(TEXTURE_FETCH) 5490 NODE_NAME_CASE(R600_EXPORT) 5491 NODE_NAME_CASE(CONST_ADDRESS) 5492 NODE_NAME_CASE(REGISTER_LOAD) 5493 NODE_NAME_CASE(REGISTER_STORE) 5494 NODE_NAME_CASE(SAMPLE) 5495 NODE_NAME_CASE(SAMPLEB) 5496 NODE_NAME_CASE(SAMPLED) 5497 NODE_NAME_CASE(SAMPLEL) 5498 NODE_NAME_CASE(CVT_F32_UBYTE0) 5499 NODE_NAME_CASE(CVT_F32_UBYTE1) 5500 NODE_NAME_CASE(CVT_F32_UBYTE2) 5501 NODE_NAME_CASE(CVT_F32_UBYTE3) 5502 NODE_NAME_CASE(CVT_PKRTZ_F16_F32) 5503 NODE_NAME_CASE(CVT_PKNORM_I16_F32) 5504 NODE_NAME_CASE(CVT_PKNORM_U16_F32) 5505 NODE_NAME_CASE(CVT_PK_I16_I32) 5506 NODE_NAME_CASE(CVT_PK_U16_U32) 5507 NODE_NAME_CASE(FP_TO_FP16) 5508 NODE_NAME_CASE(BUILD_VERTICAL_VECTOR) 5509 NODE_NAME_CASE(CONST_DATA_PTR) 5510 NODE_NAME_CASE(PC_ADD_REL_OFFSET) 5511 NODE_NAME_CASE(LDS) 5512 NODE_NAME_CASE(FPTRUNC_ROUND_UPWARD) 5513 NODE_NAME_CASE(FPTRUNC_ROUND_DOWNWARD) 5514 NODE_NAME_CASE(DUMMY_CHAIN) 5515 case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break; 5516 NODE_NAME_CASE(LOAD_D16_HI) 5517 NODE_NAME_CASE(LOAD_D16_LO) 5518 NODE_NAME_CASE(LOAD_D16_HI_I8) 5519 NODE_NAME_CASE(LOAD_D16_HI_U8) 5520 NODE_NAME_CASE(LOAD_D16_LO_I8) 5521 NODE_NAME_CASE(LOAD_D16_LO_U8) 5522 NODE_NAME_CASE(STORE_MSKOR) 5523 NODE_NAME_CASE(LOAD_CONSTANT) 5524 NODE_NAME_CASE(TBUFFER_STORE_FORMAT) 5525 NODE_NAME_CASE(TBUFFER_STORE_FORMAT_D16) 5526 NODE_NAME_CASE(TBUFFER_LOAD_FORMAT) 5527 NODE_NAME_CASE(TBUFFER_LOAD_FORMAT_D16) 5528 NODE_NAME_CASE(DS_ORDERED_COUNT) 5529 NODE_NAME_CASE(ATOMIC_CMP_SWAP) 5530 NODE_NAME_CASE(BUFFER_LOAD) 5531 NODE_NAME_CASE(BUFFER_LOAD_UBYTE) 5532 NODE_NAME_CASE(BUFFER_LOAD_USHORT) 5533 NODE_NAME_CASE(BUFFER_LOAD_BYTE) 5534 NODE_NAME_CASE(BUFFER_LOAD_SHORT) 5535 NODE_NAME_CASE(BUFFER_LOAD_TFE) 5536 NODE_NAME_CASE(BUFFER_LOAD_UBYTE_TFE) 5537 NODE_NAME_CASE(BUFFER_LOAD_USHORT_TFE) 5538 NODE_NAME_CASE(BUFFER_LOAD_BYTE_TFE) 5539 NODE_NAME_CASE(BUFFER_LOAD_SHORT_TFE) 5540 NODE_NAME_CASE(BUFFER_LOAD_FORMAT) 5541 NODE_NAME_CASE(BUFFER_LOAD_FORMAT_TFE) 5542 NODE_NAME_CASE(BUFFER_LOAD_FORMAT_D16) 5543 NODE_NAME_CASE(SBUFFER_LOAD) 5544 NODE_NAME_CASE(SBUFFER_LOAD_BYTE) 5545 NODE_NAME_CASE(SBUFFER_LOAD_UBYTE) 5546 NODE_NAME_CASE(SBUFFER_LOAD_SHORT) 5547 NODE_NAME_CASE(SBUFFER_LOAD_USHORT) 5548 NODE_NAME_CASE(BUFFER_STORE) 5549 NODE_NAME_CASE(BUFFER_STORE_BYTE) 5550 NODE_NAME_CASE(BUFFER_STORE_SHORT) 5551 NODE_NAME_CASE(BUFFER_STORE_FORMAT) 5552 NODE_NAME_CASE(BUFFER_STORE_FORMAT_D16) 5553 NODE_NAME_CASE(BUFFER_ATOMIC_SWAP) 5554 NODE_NAME_CASE(BUFFER_ATOMIC_ADD) 5555 NODE_NAME_CASE(BUFFER_ATOMIC_SUB) 5556 NODE_NAME_CASE(BUFFER_ATOMIC_SMIN) 5557 NODE_NAME_CASE(BUFFER_ATOMIC_UMIN) 5558 NODE_NAME_CASE(BUFFER_ATOMIC_SMAX) 5559 NODE_NAME_CASE(BUFFER_ATOMIC_UMAX) 5560 NODE_NAME_CASE(BUFFER_ATOMIC_AND) 5561 NODE_NAME_CASE(BUFFER_ATOMIC_OR) 5562 NODE_NAME_CASE(BUFFER_ATOMIC_XOR) 5563 NODE_NAME_CASE(BUFFER_ATOMIC_INC) 5564 NODE_NAME_CASE(BUFFER_ATOMIC_DEC) 5565 NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP) 5566 NODE_NAME_CASE(BUFFER_ATOMIC_CSUB) 5567 NODE_NAME_CASE(BUFFER_ATOMIC_FADD) 5568 NODE_NAME_CASE(BUFFER_ATOMIC_FMIN) 5569 NODE_NAME_CASE(BUFFER_ATOMIC_FMAX) 5570 NODE_NAME_CASE(BUFFER_ATOMIC_COND_SUB_U32) 5571 5572 case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break; 5573 } 5574 return nullptr; 5575 } 5576 5577 SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand, 5578 SelectionDAG &DAG, int Enabled, 5579 int &RefinementSteps, 5580 bool &UseOneConstNR, 5581 bool Reciprocal) const { 5582 EVT VT = Operand.getValueType(); 5583 5584 if (VT == MVT::f32) { 5585 RefinementSteps = 0; 5586 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand); 5587 } 5588 5589 // TODO: There is also f64 rsq instruction, but the documentation is less 5590 // clear on its precision. 5591 5592 return SDValue(); 5593 } 5594 5595 SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand, 5596 SelectionDAG &DAG, int Enabled, 5597 int &RefinementSteps) const { 5598 EVT VT = Operand.getValueType(); 5599 5600 if (VT == MVT::f32) { 5601 // Reciprocal, < 1 ulp error. 5602 // 5603 // This reciprocal approximation converges to < 0.5 ulp error with one 5604 // newton rhapson performed with two fused multiple adds (FMAs). 5605 5606 RefinementSteps = 0; 5607 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand); 5608 } 5609 5610 // TODO: There is also f64 rcp instruction, but the documentation is less 5611 // clear on its precision. 5612 5613 return SDValue(); 5614 } 5615 5616 static unsigned workitemIntrinsicDim(unsigned ID) { 5617 switch (ID) { 5618 case Intrinsic::amdgcn_workitem_id_x: 5619 return 0; 5620 case Intrinsic::amdgcn_workitem_id_y: 5621 return 1; 5622 case Intrinsic::amdgcn_workitem_id_z: 5623 return 2; 5624 default: 5625 llvm_unreachable("not a workitem intrinsic"); 5626 } 5627 } 5628 5629 void AMDGPUTargetLowering::computeKnownBitsForTargetNode( 5630 const SDValue Op, KnownBits &Known, 5631 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const { 5632 5633 Known.resetAll(); // Don't know anything. 5634 5635 unsigned Opc = Op.getOpcode(); 5636 5637 switch (Opc) { 5638 default: 5639 break; 5640 case AMDGPUISD::CARRY: 5641 case AMDGPUISD::BORROW: { 5642 Known.Zero = APInt::getHighBitsSet(32, 31); 5643 break; 5644 } 5645 5646 case AMDGPUISD::BFE_I32: 5647 case AMDGPUISD::BFE_U32: { 5648 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 5649 if (!CWidth) 5650 return; 5651 5652 uint32_t Width = CWidth->getZExtValue() & 0x1f; 5653 5654 if (Opc == AMDGPUISD::BFE_U32) 5655 Known.Zero = APInt::getHighBitsSet(32, 32 - Width); 5656 5657 break; 5658 } 5659 case AMDGPUISD::FP_TO_FP16: { 5660 unsigned BitWidth = Known.getBitWidth(); 5661 5662 // High bits are zero. 5663 Known.Zero = APInt::getHighBitsSet(BitWidth, BitWidth - 16); 5664 break; 5665 } 5666 case AMDGPUISD::MUL_U24: 5667 case AMDGPUISD::MUL_I24: { 5668 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); 5669 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1); 5670 unsigned TrailZ = LHSKnown.countMinTrailingZeros() + 5671 RHSKnown.countMinTrailingZeros(); 5672 Known.Zero.setLowBits(std::min(TrailZ, 32u)); 5673 // Skip extra check if all bits are known zeros. 5674 if (TrailZ >= 32) 5675 break; 5676 5677 // Truncate to 24 bits. 5678 LHSKnown = LHSKnown.trunc(24); 5679 RHSKnown = RHSKnown.trunc(24); 5680 5681 if (Opc == AMDGPUISD::MUL_I24) { 5682 unsigned LHSValBits = LHSKnown.countMaxSignificantBits(); 5683 unsigned RHSValBits = RHSKnown.countMaxSignificantBits(); 5684 unsigned MaxValBits = LHSValBits + RHSValBits; 5685 if (MaxValBits > 32) 5686 break; 5687 unsigned SignBits = 32 - MaxValBits + 1; 5688 bool LHSNegative = LHSKnown.isNegative(); 5689 bool LHSNonNegative = LHSKnown.isNonNegative(); 5690 bool LHSPositive = LHSKnown.isStrictlyPositive(); 5691 bool RHSNegative = RHSKnown.isNegative(); 5692 bool RHSNonNegative = RHSKnown.isNonNegative(); 5693 bool RHSPositive = RHSKnown.isStrictlyPositive(); 5694 5695 if ((LHSNonNegative && RHSNonNegative) || (LHSNegative && RHSNegative)) 5696 Known.Zero.setHighBits(SignBits); 5697 else if ((LHSNegative && RHSPositive) || (LHSPositive && RHSNegative)) 5698 Known.One.setHighBits(SignBits); 5699 } else { 5700 unsigned LHSValBits = LHSKnown.countMaxActiveBits(); 5701 unsigned RHSValBits = RHSKnown.countMaxActiveBits(); 5702 unsigned MaxValBits = LHSValBits + RHSValBits; 5703 if (MaxValBits >= 32) 5704 break; 5705 Known.Zero.setBitsFrom(MaxValBits); 5706 } 5707 break; 5708 } 5709 case AMDGPUISD::PERM: { 5710 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 5711 if (!CMask) 5712 return; 5713 5714 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); 5715 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1); 5716 unsigned Sel = CMask->getZExtValue(); 5717 5718 for (unsigned I = 0; I < 32; I += 8) { 5719 unsigned SelBits = Sel & 0xff; 5720 if (SelBits < 4) { 5721 SelBits *= 8; 5722 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I; 5723 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I; 5724 } else if (SelBits < 7) { 5725 SelBits = (SelBits & 3) * 8; 5726 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I; 5727 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I; 5728 } else if (SelBits == 0x0c) { 5729 Known.Zero |= 0xFFull << I; 5730 } else if (SelBits > 0x0c) { 5731 Known.One |= 0xFFull << I; 5732 } 5733 Sel >>= 8; 5734 } 5735 break; 5736 } 5737 case AMDGPUISD::BUFFER_LOAD_UBYTE: { 5738 Known.Zero.setHighBits(24); 5739 break; 5740 } 5741 case AMDGPUISD::BUFFER_LOAD_USHORT: { 5742 Known.Zero.setHighBits(16); 5743 break; 5744 } 5745 case AMDGPUISD::LDS: { 5746 auto GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode()); 5747 Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout()); 5748 5749 Known.Zero.setHighBits(16); 5750 Known.Zero.setLowBits(Log2(Alignment)); 5751 break; 5752 } 5753 case AMDGPUISD::SMIN3: 5754 case AMDGPUISD::SMAX3: 5755 case AMDGPUISD::SMED3: 5756 case AMDGPUISD::UMIN3: 5757 case AMDGPUISD::UMAX3: 5758 case AMDGPUISD::UMED3: { 5759 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(2), Depth + 1); 5760 if (Known2.isUnknown()) 5761 break; 5762 5763 KnownBits Known1 = DAG.computeKnownBits(Op.getOperand(1), Depth + 1); 5764 if (Known1.isUnknown()) 5765 break; 5766 5767 KnownBits Known0 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); 5768 if (Known0.isUnknown()) 5769 break; 5770 5771 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling. 5772 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero; 5773 Known.One = Known0.One & Known1.One & Known2.One; 5774 break; 5775 } 5776 case ISD::INTRINSIC_WO_CHAIN: { 5777 unsigned IID = Op.getConstantOperandVal(0); 5778 switch (IID) { 5779 case Intrinsic::amdgcn_workitem_id_x: 5780 case Intrinsic::amdgcn_workitem_id_y: 5781 case Intrinsic::amdgcn_workitem_id_z: { 5782 unsigned MaxValue = Subtarget->getMaxWorkitemID( 5783 DAG.getMachineFunction().getFunction(), workitemIntrinsicDim(IID)); 5784 Known.Zero.setHighBits(llvm::countl_zero(MaxValue)); 5785 break; 5786 } 5787 default: 5788 break; 5789 } 5790 } 5791 } 5792 } 5793 5794 unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode( 5795 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, 5796 unsigned Depth) const { 5797 switch (Op.getOpcode()) { 5798 case AMDGPUISD::BFE_I32: { 5799 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 5800 if (!Width) 5801 return 1; 5802 5803 unsigned SignBits = 32 - Width->getZExtValue() + 1; 5804 if (!isNullConstant(Op.getOperand(1))) 5805 return SignBits; 5806 5807 // TODO: Could probably figure something out with non-0 offsets. 5808 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1); 5809 return std::max(SignBits, Op0SignBits); 5810 } 5811 5812 case AMDGPUISD::BFE_U32: { 5813 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 5814 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1; 5815 } 5816 5817 case AMDGPUISD::CARRY: 5818 case AMDGPUISD::BORROW: 5819 return 31; 5820 case AMDGPUISD::BUFFER_LOAD_BYTE: 5821 return 25; 5822 case AMDGPUISD::BUFFER_LOAD_SHORT: 5823 return 17; 5824 case AMDGPUISD::BUFFER_LOAD_UBYTE: 5825 return 24; 5826 case AMDGPUISD::BUFFER_LOAD_USHORT: 5827 return 16; 5828 case AMDGPUISD::FP_TO_FP16: 5829 return 16; 5830 case AMDGPUISD::SMIN3: 5831 case AMDGPUISD::SMAX3: 5832 case AMDGPUISD::SMED3: 5833 case AMDGPUISD::UMIN3: 5834 case AMDGPUISD::UMAX3: 5835 case AMDGPUISD::UMED3: { 5836 unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(2), Depth + 1); 5837 if (Tmp2 == 1) 5838 return 1; // Early out. 5839 5840 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth + 1); 5841 if (Tmp1 == 1) 5842 return 1; // Early out. 5843 5844 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1); 5845 if (Tmp0 == 1) 5846 return 1; // Early out. 5847 5848 return std::min({Tmp0, Tmp1, Tmp2}); 5849 } 5850 default: 5851 return 1; 5852 } 5853 } 5854 5855 unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr( 5856 GISelKnownBits &Analysis, Register R, 5857 const APInt &DemandedElts, const MachineRegisterInfo &MRI, 5858 unsigned Depth) const { 5859 const MachineInstr *MI = MRI.getVRegDef(R); 5860 if (!MI) 5861 return 1; 5862 5863 // TODO: Check range metadata on MMO. 5864 switch (MI->getOpcode()) { 5865 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE: 5866 return 25; 5867 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT: 5868 return 17; 5869 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: 5870 return 24; 5871 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: 5872 return 16; 5873 case AMDGPU::G_AMDGPU_SMED3: 5874 case AMDGPU::G_AMDGPU_UMED3: { 5875 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs(); 5876 unsigned Tmp2 = Analysis.computeNumSignBits(Src2, DemandedElts, Depth + 1); 5877 if (Tmp2 == 1) 5878 return 1; 5879 unsigned Tmp1 = Analysis.computeNumSignBits(Src1, DemandedElts, Depth + 1); 5880 if (Tmp1 == 1) 5881 return 1; 5882 unsigned Tmp0 = Analysis.computeNumSignBits(Src0, DemandedElts, Depth + 1); 5883 if (Tmp0 == 1) 5884 return 1; 5885 return std::min({Tmp0, Tmp1, Tmp2}); 5886 } 5887 default: 5888 return 1; 5889 } 5890 } 5891 5892 bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op, 5893 const SelectionDAG &DAG, 5894 bool SNaN, 5895 unsigned Depth) const { 5896 unsigned Opcode = Op.getOpcode(); 5897 switch (Opcode) { 5898 case AMDGPUISD::FMIN_LEGACY: 5899 case AMDGPUISD::FMAX_LEGACY: { 5900 if (SNaN) 5901 return true; 5902 5903 // TODO: Can check no nans on one of the operands for each one, but which 5904 // one? 5905 return false; 5906 } 5907 case AMDGPUISD::FMUL_LEGACY: 5908 case AMDGPUISD::CVT_PKRTZ_F16_F32: { 5909 if (SNaN) 5910 return true; 5911 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && 5912 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); 5913 } 5914 case AMDGPUISD::FMED3: 5915 case AMDGPUISD::FMIN3: 5916 case AMDGPUISD::FMAX3: 5917 case AMDGPUISD::FMINIMUM3: 5918 case AMDGPUISD::FMAXIMUM3: 5919 case AMDGPUISD::FMAD_FTZ: { 5920 if (SNaN) 5921 return true; 5922 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && 5923 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && 5924 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); 5925 } 5926 case AMDGPUISD::CVT_F32_UBYTE0: 5927 case AMDGPUISD::CVT_F32_UBYTE1: 5928 case AMDGPUISD::CVT_F32_UBYTE2: 5929 case AMDGPUISD::CVT_F32_UBYTE3: 5930 return true; 5931 5932 case AMDGPUISD::RCP: 5933 case AMDGPUISD::RSQ: 5934 case AMDGPUISD::RCP_LEGACY: 5935 case AMDGPUISD::RSQ_CLAMP: { 5936 if (SNaN) 5937 return true; 5938 5939 // TODO: Need is known positive check. 5940 return false; 5941 } 5942 case ISD::FLDEXP: 5943 case AMDGPUISD::FRACT: { 5944 if (SNaN) 5945 return true; 5946 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); 5947 } 5948 case AMDGPUISD::DIV_SCALE: 5949 case AMDGPUISD::DIV_FMAS: 5950 case AMDGPUISD::DIV_FIXUP: 5951 // TODO: Refine on operands. 5952 return SNaN; 5953 case AMDGPUISD::SIN_HW: 5954 case AMDGPUISD::COS_HW: { 5955 // TODO: Need check for infinity 5956 return SNaN; 5957 } 5958 case ISD::INTRINSIC_WO_CHAIN: { 5959 unsigned IntrinsicID = Op.getConstantOperandVal(0); 5960 // TODO: Handle more intrinsics 5961 switch (IntrinsicID) { 5962 case Intrinsic::amdgcn_cubeid: 5963 return true; 5964 5965 case Intrinsic::amdgcn_frexp_mant: { 5966 if (SNaN) 5967 return true; 5968 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); 5969 } 5970 case Intrinsic::amdgcn_cvt_pkrtz: { 5971 if (SNaN) 5972 return true; 5973 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && 5974 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); 5975 } 5976 case Intrinsic::amdgcn_rcp: 5977 case Intrinsic::amdgcn_rsq: 5978 case Intrinsic::amdgcn_rcp_legacy: 5979 case Intrinsic::amdgcn_rsq_legacy: 5980 case Intrinsic::amdgcn_rsq_clamp: { 5981 if (SNaN) 5982 return true; 5983 5984 // TODO: Need is known positive check. 5985 return false; 5986 } 5987 case Intrinsic::amdgcn_trig_preop: 5988 case Intrinsic::amdgcn_fdot2: 5989 // TODO: Refine on operand 5990 return SNaN; 5991 case Intrinsic::amdgcn_fma_legacy: 5992 if (SNaN) 5993 return true; 5994 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && 5995 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1) && 5996 DAG.isKnownNeverNaN(Op.getOperand(3), SNaN, Depth + 1); 5997 default: 5998 return false; 5999 } 6000 } 6001 default: 6002 return false; 6003 } 6004 } 6005 6006 bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI, 6007 Register N0, Register N1) const { 6008 return MRI.hasOneNonDBGUse(N0); // FIXME: handle regbanks 6009 } 6010 6011 TargetLowering::AtomicExpansionKind 6012 AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { 6013 switch (RMW->getOperation()) { 6014 case AtomicRMWInst::Nand: 6015 case AtomicRMWInst::FAdd: 6016 case AtomicRMWInst::FSub: 6017 case AtomicRMWInst::FMax: 6018 case AtomicRMWInst::FMin: 6019 return AtomicExpansionKind::CmpXChg; 6020 case AtomicRMWInst::Xchg: { 6021 const DataLayout &DL = RMW->getFunction()->getDataLayout(); 6022 unsigned ValSize = DL.getTypeSizeInBits(RMW->getType()); 6023 if (ValSize == 32 || ValSize == 64) 6024 return AtomicExpansionKind::None; 6025 return AtomicExpansionKind::CmpXChg; 6026 } 6027 default: { 6028 if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) { 6029 unsigned Size = IntTy->getBitWidth(); 6030 if (Size == 32 || Size == 64) 6031 return AtomicExpansionKind::None; 6032 } 6033 6034 return AtomicExpansionKind::CmpXChg; 6035 } 6036 } 6037 } 6038 6039 /// Whether it is profitable to sink the operands of an 6040 /// Instruction I to the basic block of I. 6041 /// This helps using several modifiers (like abs and neg) more often. 6042 bool AMDGPUTargetLowering::shouldSinkOperands( 6043 Instruction *I, SmallVectorImpl<Use *> &Ops) const { 6044 using namespace PatternMatch; 6045 6046 for (auto &Op : I->operands()) { 6047 // Ensure we are not already sinking this operand. 6048 if (any_of(Ops, [&](Use *U) { return U->get() == Op.get(); })) 6049 continue; 6050 6051 if (match(&Op, m_FAbs(m_Value())) || match(&Op, m_FNeg(m_Value()))) 6052 Ops.push_back(&Op); 6053 } 6054 6055 return !Ops.empty(); 6056 } 6057